diff --git a/.codecov.yml b/.codecov.yml index b2bc67814e3..191144aae16 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,5 +1,6 @@ - -codecov: - ci: - - !appveyor - - travis +coverage: + status: + project: + default: + target: 60% + threshold: 2% diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index d7a92b3b409..79bbe9af075 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -6,6 +6,8 @@ inputs: required: true cmake-args: default: null + cmake-target: + default: all # An implicit input is the environment variable `build_dir`. runs: using: composite @@ -26,4 +28,5 @@ runs: cmake \ --build ${build_dir} \ --config ${{ inputs.configuration }} \ - --parallel ${NUM_PROCESSORS:-$(nproc)} + --parallel ${NUM_PROCESSORS:-$(nproc)} \ + --target ${{ inputs.cmake-target }} diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 3147e8774f2..d9687ff121e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -12,8 +12,36 @@ runs: - name: export custom recipes shell: bash run: | + conan config set general.revisions_enabled=1 conan export external/snappy snappy/1.1.10@ + conan export external/rocksdb rocksdb/6.29.5@ conan export external/soci soci/4.0.3@ + - name: add Ripple Conan remote + shell: bash + run: | + conan remote list + conan remote remove ripple || true + # Do not quote the URL. An empty string will be accepted (with + # a non-fatal warning), but a missing argument will not. + conan remote add ripple ${{ env.CONAN_URL }} --insert 0 + - name: try to authenticate to Ripple Conan remote + id: remote + shell: bash + run: | + # `conan user` implicitly uses the environment variables + # CONAN_LOGIN_USERNAME_ and CONAN_PASSWORD_. + # https://docs.conan.io/1/reference/commands/misc/user.html#using-environment-variables + # https://docs.conan.io/1/reference/env_vars.html#conan-login-username-conan-login-username-remote-name + # https://docs.conan.io/1/reference/env_vars.html#conan-password-conan-password-remote-name + echo outcome=$(conan user --remote ripple --password >&2 \ + && echo success || echo failure) | tee ${GITHUB_OUTPUT} + - name: list missing binaries + id: binaries + shell: bash + # Print the list of dependencies that would need to be built locally. + # A non-empty list means we have "failed" to cache binaries remotely. + run: | + echo missing=$(conan info . --build missing --settings build_type=${{ inputs.configuration }} --json 2>/dev/null | grep '^\[') | tee ${GITHUB_OUTPUT} - name: install dependencies shell: bash run: | @@ -24,3 +52,7 @@ runs: --build missing \ --settings build_type=${{ inputs.configuration }} \ .. + - name: upload dependencies to remote + if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success') + shell: bash + run: conan upload --remote ripple '*' --all --parallel --confirm diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f3c5edd0b0b..3ab3a38807f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,12 @@ ## High Level Overview of Change @@ -33,6 +39,7 @@ Please check [x] relevant options, delete irrelevant ones. - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Refactor (non-breaking change that only restructures code) +- [ ] Performance (increase or change in throughput and/or latency) - [ ] Tests (you added tests for code that already exists, or your new feature included in this PR) - [ ] Documentation update - [ ] Chore (no impact to binary, e.g. `.gitignore`, formatting, dropping support for older tooling) @@ -58,6 +65,12 @@ Please check [x] relevant options, delete irrelevant ones. ## Before / After If relevant, use this section for an English description of the change at a technical level. If this change affects an API, examples should be included here. + +For performance-impacting changes, please provide these details: +1. Is this a new feature, bug fix, or improvement to existing functionality? +2. What behavior/functionality does the change impact? +3. In what processing can the impact be measured? Be as specific as possible - e.g. RPC client call, payment transaction that involves LOB, AMM, caching, DB operations, etc. +4. Does this change affect concurrent processing - e.g. does it involve acquiring locks, multi-threaded processing, or async processing? --> + + +## Action Required + +One new amendment is now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators. + +If you operate an XRP Ledger server, upgrade to version 2.1.1 by April 8, 2024 to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network. + +## Changelog + +### Amendments + +- **fixAMMOverflowOffer**: Fix improper handling of large synthetic AMM offers in the payment engine. Due to the importance of this fix, the default vote in the source code has been set to YES. For information on how to configure your validator's amendment voting, see [Configure Amendment Voting](https://xrpl.org/docs/infrastructure/configuration/configure-amendment-voting). + +# Introducing XRP Ledger version 2.1.0 + +Version 2.1.0 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release adds a bug fix, build improvements, and introduces the `fixNFTokenReserve` and `fixInnerObjTemplate` amendments. + +[Sign Up for Future Release Announcements](https://groups.google.com/g/ripple-server) + + + + +## Action Required + +Two new amendments are now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators. + +If you operate an XRP Ledger server, upgrade to version 2.1.0 by March 5, 2024 to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network. + +## Changelog + +### Amendments +(These are changes which may impact or be useful to end users. For example, you may be able to update your code/workflow to take advantage of these changes.) + +- **fixNFTokenReserve**: Adds a check to the `NFTokenAcceptOffer` transactor to see if the `OwnerCount` changed. If it did, it checks that the reserve requirement is met. [#4767](https://github.com/XRPLF/rippled/pull/4767) + +- **fixInnerObjTemplate**: Adds an `STObject` constructor overload that includes an additional boolean argument to set the inner object template; currently, the inner object template isn't set upon object creation. In some circumstances, this causes a `tefEXCEPTION` error when trying to access the AMM `sfTradingFee` and `sfDiscountedFee` fields in the inner objects of `sfVoteEntry` and `sfAuctionSlot`. [#4906](https://github.com/XRPLF/rippled/pull/4906) + + +### Bug Fixes and Performance Improvements +(These are behind-the-scenes improvements, such as internal changes to the code, which are not expected to impact end users.) + +- Fixed a bug that prevented the gRPC port info from being specified in the `rippled` config file. [#4728](https://github.com/XRPLF/rippled/pull/4728) + + +### Docs and Build System + +- Added unit tests to check that payees and payers aren't the same account. [#4860](https://github.com/XRPLF/rippled/pull/4860) + +- Removed a workaround that bypassed Windows CI unit test failures. [#4871](https://github.com/XRPLF/rippled/pull/4871) + +- Updated library names to be platform-agnostic in Conan recipes. [#4831](https://github.com/XRPLF/rippled/pull/4831) + +- Added headers required in the Conan package to build xbridge witness servers. [#4885](https://github.com/XRPLF/rippled/pull/4885) + +- Improved object lifetime management when creating a temporary `Rules` object, fixing a crash in Windows unit tests. [#4917](https://github.com/XRPLF/rippled/pull/4917) + +### GitHub + +The public source code repository for `rippled` is hosted on GitHub at . + +We welcome all contributions and invite everyone to join the community of XRP Ledger developers to help build the Internet of Value. + + +## Credits + +The following people contributed directly to this release: + +- Bronek Kozicki +- CJ Cobb +- Chenna Keshava B S <21219765+ckeshava@users.noreply.github.com> +- Ed Hennis +- Elliot Lee +- Gregory Tsipenyuk +- John Freeman +- Michael Legleux +- Ryan Molley +- Shawn Xie <35279399+shawnxie999@users.noreply.github.com> + + +Bug Bounties and Responsible Disclosures: + +We welcome reviews of the `rippled` code and urge researchers to responsibly disclose any issues they may find. + +To report a bug, please send a detailed report to: + +# Introducing XRP Ledger version 2.0.1 + +Version 2.0.1 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release includes minor fixes, unit test improvements, and doc updates. + +[Sign Up for Future Release Announcements](https://groups.google.com/g/ripple-server) + + + + +## Action Required + +If you operate an XRP Ledger server, upgrade to version 2.0.1 to take advantage of the changes included in this update. Nodes on version 1.12 should upgrade as soon as possible. + + +## Changelog + + +### Changes +(These are changes which may impact or be useful to end users. For example, you may be able to update your code/workflow to take advantage of these changes.) + +- Updated the `send_queue_limit` to 500 in the default `rippled` config to handle increased transaction loads. [#4867](https://github.com/XRPLF/rippled/pull/4867) + + +### Bug Fixes and Performance Improvements +(These are behind-the-scenes improvements, such as internal changes to the code, which are not expected to impact end users.) + +- Fixed an assertion that occurred when `rippled` was under heavy websocket client load. [#4848](https://github.com/XRPLF/rippled/pull/4848) + +- Improved lifetime management of serialized type ledger entries to improve memory usage. [#4822](https://github.com/XRPLF/rippled/pull/4822) + +- Fixed a clang warning about deprecated sprintf usage. [#4747](https://github.com/XRPLF/rippled/pull/4747) + + +### Docs and Build System + +- Added `DeliverMax` to more JSONRPC tests. [#4826](https://github.com/XRPLF/rippled/pull/4826) + +- Updated the pull request template to include a `Type of Change` checkbox and additional contextual questions. [#4875](https://github.com/XRPLF/rippled/pull/4875) + +- Updated help messages for unit tests pattern matching. [#4846](https://github.com/XRPLF/rippled/pull/4846) + +- Improved the time it take to generate coverage reports. [#4849](https://github.com/XRPLF/rippled/pull/4849) + +- Fixed broken links in the Conan build docs. [#4699](https://github.com/XRPLF/rippled/pull/4699) + +- Spurious codecov uploads are now retried if there's an error uploading them the first time. [#4896](https://github.com/XRPLF/rippled/pull/4896) + + +### GitHub + +The public source code repository for `rippled` is hosted on GitHub at . + +We welcome all contributions and invite everyone to join the community of XRP Ledger developers to help build the Internet of Value. + + +## Credits + +The following people contributed directly to this release: + +- Bronek Kozicki +- Chenna Keshava B S <21219765+ckeshava@users.noreply.github.com> +- Ed Hennis +- Elliot Lee +- Lathan Britz +- Mark Travis +- nixer89 + +Bug Bounties and Responsible Disclosures: + +We welcome reviews of the `rippled` code and urge researchers to responsibly disclose any issues they may find. + +To report a bug, please send a detailed report to: + +# Introducing XRP Ledger version 2.0.0 + +Version 2.0.0 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release adds new features and bug fixes, and introduces these amendments: + +- `DID` +- `XChainBridge` +- `fixDisallowIncomingV1` +- `fixFillOrKill` + +[Sign Up for Future Release Announcements](https://groups.google.com/g/ripple-server) + + + + +## Action Required + +Four new amendments are now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators. + +If you operate an XRP Ledger server, upgrade to version 2.0.0 by January 22, 2024 to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network. + + +## Changelog + + +### Amendments, New Features, and Changes +(These are changes which may impact or be useful to end users. For example, you may be able to update your code/workflow to take advantage of these changes.) + +- **XChainBridge**: Introduces cross-chain bridges, enabling interoperability between the XRP Ledger and sidechains. [#4292](https://github.com/XRPLF/rippled/pull/4292) + +- **DID**: Introduces decentralized identifiers. [#4636](https://github.com/XRPLF/rippled/pull/4636) + +- **fixDisallowIncomingV1**: Fixes an issue that occurs when users try to authorize a trustline while the `lsfDisallowIncomingTrustline` flag is enabled on their account. [#4721](https://github.com/XRPLF/rippled/pull/4721) + +- **fixFillOrKill**: Fixes an issue introduced in the `flowCross` amendment. The `tfFillOrKill` and `tfSell` flags are now properly handled to allow offers to cross in certain scenarios. [#4694](https://github.com/XRPLF/rippled/pull/4694) + +- **API v2 released with these changes:** + + - Accepts currency codes in ASCII, using the full alphabet. [#4566](https://github.com/XRPLF/rippled/pull/4566) + - Added test to verify the `check` field is a string. [#4630](https://github.com/XRPLF/rippled/pull/4630) + - Added errors for malformed `account_tx` and `noripple_check` fields. [#4620](https://github.com/XRPLF/rippled/pull/4620) + - Added errors for malformed `gateway_balances` and `channel_authorize` requests. [#4618](https://github.com/XRPLF/rippled/pull/4618) + - Added a `DeliverMax` alias to `Amount` and removed `Amount`. [#4733](https://github.com/XRPLF/rippled/pull/4733) + - Removed `tx_history` and `ledger_header` methods. Also updated `RPC::Handler` to allow for version-specific methods. [#4759](https://github.com/XRPLF/rippled/pull/4759) + - Standardized the JSON serialization format of transactions. [#4727](https://github.com/XRPLF/rippled/issues/4727) + - Bumped API support to v2, but kept the command-line interface for `rippled` and unit tests at v1. [#4803](https://github.com/XRPLF/rippled/pull/4803) + - Standardized `ledger_index` to return as a number. [#4820](https://github.com/XRPLF/rippled/pull/4820) + +- Added a `server_definitions` command that returns an SDK-compatible `definitions.json` file, generated from the `rippled` instance currently running. [#4703](https://github.com/XRPLF/rippled/pull/4703) + +- Improved unit test command line input and run times. [#4634](https://github.com/XRPLF/rippled/pull/4634) + +- Added the link compression setting to the the `rippled-example.cfg` file. [#4753](https://github.com/XRPLF/rippled/pull/4753) + +- Changed the reserved hook error code name from `tecHOOK_ERROR` to `tecHOOK_REJECTED`. [#4559](https://github.com/XRPLF/rippled/pull/4559) + + +### Bug Fixes and Performance Improvements +(These are behind-the-scenes improvements, such as internal changes to the code, which are not expected to impact end users.) + +- Simplified `TxFormats` common fields logic. [#4637](https://github.com/XRPLF/rippled/pull/4637) + +- Improved transaction throughput by asynchronously writing batches to *NuDB*. [#4503](https://github.com/XRPLF/rippled/pull/4503) + +- Removed 2 unused functions. [#4708](https://github.com/XRPLF/rippled/pull/4708) + +- Removed an unused variable that caused clang 14 build errors. [#4672](https://github.com/XRPLF/rippled/pull/4672) + +- Fixed comment about return value of `LedgerHistory::fixIndex`. [#4574](https://github.com/XRPLF/rippled/pull/4574) + +- Updated `secp256k1` to 0.3.2. [#4653](https://github.com/XRPLF/rippled/pull/4653) + +- Removed built-in SNTP clock issues. [#4628](https://github.com/XRPLF/rippled/pull/4628) + +- Fixed amendment flapping. This issue usually occurred when an amendment was on the verge of gaining majority, but a validator not in favor of the amendment went offline. [#4410](https://github.com/XRPLF/rippled/pull/4410) + +- Fixed asan stack-use-after-scope issue. [#4676](https://github.com/XRPLF/rippled/pull/4676) + +- Transactions and pseudo-transactions share the same `commonFields` again. [#4715](https://github.com/XRPLF/rippled/pull/4715) + +- Reduced boilerplate in `applySteps.cpp`. When a new transactor is added, only one function needs to be modified now. [#4710](https://github.com/XRPLF/rippled/pull/4710) + +- Removed an incorrect assert. [#4743](https://github.com/XRPLF/rippled/pull/4743) + +- Replaced some asserts in `PeerFinder::Logic` with `LogicError` to better indicate the nature of server crashes. [#4562](https://github.com/XRPLF/rippled/pull/4562) + +- Fixed an issue with enabling new amendments on a network with an ID greater than 1024. [#4737](https://github.com/XRPLF/rippled/pull/4737) + + +### Docs and Build System + +- Updated `rippled-example.cfg` docs to clarify usage of *ssl_cert* vs *ssl_chain*. [#4667](https://github.com/XRPLF/rippled/pull/4667) + +- Updated `BUILD.md`: + - Made the `environment.md` link easier to find. Also made it easier to find platform-specific info. [#4507](https://github.com/XRPLF/rippled/pull/4507) + - Fixed typo. [#4718](https://github.com/XRPLF/rippled/pull/4718) + - Updated the minimum compiler requirements. [#4700](https://github.com/XRPLF/rippled/pull/4700) + - Added note about enabling `XRPFees`. [#4741](https://github.com/XRPLF/rippled/pull/4741) + +- Updated `API-CHANGELOG.md`: + - Explained API v2 is releasing with `rippled` 2.0.0. [#4633](https://github.com/XRPLF/rippled/pull/4633) + - Clarified the location of the `signer_lists` field in the `account_info` response for API v2. [#4724](https://github.com/XRPLF/rippled/pull/4724) + - Added documentation for the new `DeliverMax` field. [#4784](https://github.com/XRPLF/rippled/pull/4784) + - Removed references to API v2 being "in progress" and "in beta". [#4828](https://github.com/XRPLF/rippled/pull/4828) + - Clarified that all breaking API changes will now occur in API v3 or later. [#4773](https://github.com/XRPLF/rippled/pull/4773) + +- Fixed a mistake in the overlay README. [#4635](https://github.com/XRPLF/rippled/pull/4635) + +- Fixed an early return from `RippledRelease.cmake` that prevented targets from being created during packaging. [#4707](https://github.com/XRPLF/rippled/pull/4707) + +- Fixed a build error with Intel Macs. [#4632](https://github.com/XRPLF/rippled/pull/4632) + +- Added `.build` to `.gitignore`. [#4722](https://github.com/XRPLF/rippled/pull/4722) + +- Fixed a `uint is not universally defined` Windows build error. [#4731](https://github.com/XRPLF/rippled/pull/4731) + +- Reenabled Windows CI build with Artifactory support. [#4596](https://github.com/XRPLF/rippled/pull/4596) + +- Fixed output of remote step in Nix workflow. [#4746](https://github.com/XRPLF/rippled/pull/4746) + +- Fixed a broken link in `conan.md`. [#4740](https://github.com/XRPLF/rippled/pull/4740) + +- Added a `python` call to fix the `pip` upgrade command in Windows CI. [#4768](https://github.com/XRPLF/rippled/pull/4768) + +- Added an API Impact section to `pull_request_template.md`. [#4757](https://github.com/XRPLF/rippled/pull/4757) + +- Set permissions for the Doxygen workflow. [#4756](https://github.com/XRPLF/rippled/pull/4756) + +- Switched to Unity builds to speed up Windows CI. [#4780](https://github.com/XRPLF/rippled/pull/4780) + +- Clarified what makes concensus healthy in `FeeEscalation.md`. [#4729](https://github.com/XRPLF/rippled/pull/4729) + +- Removed a dependency on the header for unit tests. [#4788](https://github.com/XRPLF/rippled/pull/4788) + +- Fixed a clang `unused-but-set-variable` warning. [#4677](https://github.com/XRPLF/rippled/pull/4677) + +- Removed an unused Dockerfile. [#4791](https://github.com/XRPLF/rippled/pull/4791) + +- Fixed unit tests to work with API v2. [#4785](https://github.com/XRPLF/rippled/pull/4785) + +- Added support for the mold linker on Linux. [#4807](https://github.com/XRPLF/rippled/pull/4807) + +- Updated Linux distribtuions `rippled` smoke tests run on. [#4813](https://github.com/XRPLF/rippled/pull/4813) + +- Added codename `bookworm` to the distribution matrix during Artifactory uploads, enabling Debian 12 clients to install `rippled` packages. [#4836](https://github.com/XRPLF/rippled/pull/4836) + +- Added a workaround for compilation errors with GCC 13 and other compilers relying on libstdc++ version 13. [#4817](https://github.com/XRPLF/rippled/pull/4817) + +- Fixed a minor typo in the code comments of `AMMCreate.h`. [4821](https://github.com/XRPLF/rippled/pull/4821) + + +### GitHub + +The public source code repository for `rippled` is hosted on GitHub at . + +We welcome all contributions and invite everyone to join the community of XRP Ledger developers to help build the Internet of Value. + + +## Credits + +The following people contributed directly to this release: + +- Bronek Kozicki +- Chenna Keshava B S <21219765+ckeshava@users.noreply.github.com> +- Denis Angell +- Ed Hennis +- Elliot Lee +- Florent <36513774+florent-uzio@users.noreply.github.com> +- ForwardSlashBack <142098649+ForwardSlashBack@users.noreply.github.com> +- Gregory Tsipenyuk +- Howard Hinnant +- Hussein Badakhchani +- Jackson Mills +- John Freeman +- Manoj Doshi +- Mark Pevec +- Mark Travis +- Mayukha Vadari +- Michael Legleux +- Nik Bougalis +- Peter Chen <34582813+PeterChen13579@users.noreply.github.com> +- Rome Reginelli +- Scott Determan +- Scott Schurr +- Sophia Xie <106177003+sophiax851@users.noreply.github.com> +- Stefan van Kessel +- pwang200 <354723+pwang200@users.noreply.github.com> +- shichengsg002 <147461171+shichengsg002@users.noreply.github.com> +- sokkaofthewatertribe <140777955+sokkaofthewatertribe@users.noreply.github.com> + +Bug Bounties and Responsible Disclosures: + +We welcome reviews of the rippled code and urge researchers to responsibly disclose any issues they may find. + +To report a bug, please send a detailed report to: + + # Introducing XRP Ledger version 1.12.0 Version 1.12.0 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release adds new features and bug fixes, and introduces these amendments: diff --git a/bin/ci/ubuntu/build-and-test.sh b/bin/ci/ubuntu/build-and-test.sh index 7ae75f2b16a..2c1734863fb 100755 --- a/bin/ci/ubuntu/build-and-test.sh +++ b/bin/ci/ubuntu/build-and-test.sh @@ -44,7 +44,7 @@ if [[ ${NINJA_BUILD:-} == true ]]; then fi coverage=false -if [[ "${TARGET}" == "coverage_report" ]] ; then +if [[ "${TARGET}" == "coverage" ]] ; then echo "coverage option detected." coverage=true fi diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index a3bcf0056be..2ba2afa727d 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -416,7 +416,6 @@ # # The default list of entries is: # - r.ripple.com 51235 -# - zaphod.alloy.ee 51235 # - sahyadri.isrdc.in 51235 # # Examples: @@ -482,7 +481,7 @@ # # Configure the maximum number of transactions to have in the job queue # -# Must be a number between 1000 and 100000, defaults to 10000 +# Must be a number between 100 and 1000, defaults to 250 # # # [overlay] @@ -1641,6 +1640,7 @@ port = 6006 ip = 127.0.0.1 admin = 127.0.0.1 protocol = ws +send_queue_limit = 500 [port_grpc] port = 50051 @@ -1651,6 +1651,7 @@ secure_gateway = 127.0.0.1 #port = 6005 #ip = 127.0.0.1 #protocol = wss +#send_queue_limit = 500 #------------------------------------------------------------------------------- diff --git a/cfg/rippled-reporting.cfg b/cfg/rippled-reporting.cfg index 632a8a7800e..290bcc5418a 100644 --- a/cfg/rippled-reporting.cfg +++ b/cfg/rippled-reporting.cfg @@ -388,7 +388,6 @@ # # The default list of entries is: # - r.ripple.com 51235 -# - zaphod.alloy.ee 51235 # - sahyadri.isrdc.in 51235 # # Examples: @@ -454,7 +453,7 @@ # # Configure the maximum number of transactions to have in the job queue # -# Must be a number between 1000 and 100000, defaults to 10000 +# Must be a number between 100 and 1000, defaults to 250 # # # [overlay] diff --git a/conanfile.py b/conanfile.py index c789cacf568..d1d24a4876a 100644 --- a/conanfile.py +++ b/conanfile.py @@ -36,6 +36,7 @@ class Xrpl(ConanFile): 'soci/4.0.3', 'sqlite3/3.42.0', 'zlib/1.2.13', + 'xxhash/0.8.2', ] default_options = { @@ -86,6 +87,7 @@ class Xrpl(ConanFile): 'soci:shared': False, 'soci:with_sqlite3': True, 'soci:with_boost': True, + 'xxhash:shared': False, } def set_version(self): @@ -147,9 +149,9 @@ def package(self): def package_info(self): libxrpl = self.cpp_info.components['libxrpl'] libxrpl.libs = [ - 'libxrpl_core.a', - 'libed25519.a', - 'libsecp256k1.a', + 'xrpl_core', + 'ed25519', + 'secp256k1', ] # TODO: Fix the protobufs to include each other relative to # `include/`, not `include/ripple/proto/`. @@ -159,4 +161,5 @@ def package_info(self): 'openssl::crypto', 'date::date', 'grpc::grpc++', + 'xxHash::xxhash', ] diff --git a/docs/build/conan.md b/docs/build/conan.md index 210db18a5df..5f1ff7ae983 100644 --- a/docs/build/conan.md +++ b/docs/build/conan.md @@ -109,6 +109,16 @@ For options, each package recipe defines its own defaults. You can pass every parameter to Conan on the command line, but it is more convenient to put them in a configuration file, once, that Conan can read every time it is configured. -For Conan, that file is a [profile][profile]. +For Conan, that file is a [profile][]. **All we must do to properly configure Conan is edit and pass the profile.** By default, Conan will use the profile named "default". + +[build_type]: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html +[find_package]: https://cmake.org/cmake/help/latest/command/find_package.html +[pcf]: https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#package-configuration-file +[prefix_path]: https://cmake.org/cmake/help/latest/variable/CMAKE_PREFIX_PATH.html +[profile]: https://docs.conan.io/en/latest/reference/profiles.html +[pvf]: https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#package-version-file +[runtime]: https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html +[search]: https://cmake.org/cmake/help/latest/command/find_package.html#search-procedure +[toolchain]: https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html diff --git a/external/rocksdb/conandata.yml b/external/rocksdb/conandata.yml new file mode 100644 index 00000000000..86b42f79f0f --- /dev/null +++ b/external/rocksdb/conandata.yml @@ -0,0 +1,27 @@ +sources: + "6.29.5": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.29.5.tar.gz" + sha256: "ddbf84791f0980c0bbce3902feb93a2c7006f6f53bfd798926143e31d4d756f0" + "6.27.3": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz" + sha256: "ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58" + "6.20.3": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.20.3.tar.gz" + sha256: "c6502c7aae641b7e20fafa6c2b92273d935d2b7b2707135ebd9a67b092169dca" + "8.8.1": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.8.1.tar.gz" + sha256: "056c7e21ad8ae36b026ac3b94b9d6e0fcc60e1d937fc80330921e4181be5c36e" +patches: + "6.29.5": + - patch_file: "patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch" + patch_description: "Fix build with gcc 13 by including cstdint" + patch_type: "portability" + patch_source: "https://github.com/facebook/rocksdb/pull/11118" + - patch_file: "patches/6.29.5-0002-exclude-thirdparty.patch" + patch_description: "Do not include thirdparty.inc" + patch_type: "portability" + "6.27.3": + - patch_file: "patches/6.27.3-0001-add-include-cstdint-for-gcc-13.patch" + patch_description: "Fix build with gcc 13 by including cstdint" + patch_type: "portability" + patch_source: "https://github.com/facebook/rocksdb/pull/11118" diff --git a/external/rocksdb/conanfile.py b/external/rocksdb/conanfile.py new file mode 100644 index 00000000000..09425b9f863 --- /dev/null +++ b/external/rocksdb/conanfile.py @@ -0,0 +1,233 @@ +import os +import glob +import shutil + +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.build import check_min_cppstd +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import apply_conandata_patches, collect_libs, copy, export_conandata_patches, get, rm, rmdir +from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime +from conan.tools.scm import Version + +required_conan_version = ">=1.53.0" + + +class RocksDBConan(ConanFile): + name = "rocksdb" + homepage = "https://github.com/facebook/rocksdb" + license = ("GPL-2.0-only", "Apache-2.0") + url = "https://github.com/conan-io/conan-center-index" + description = "A library that provides an embeddable, persistent key-value store for fast storage" + topics = ("database", "leveldb", "facebook", "key-value") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "lite": [True, False], + "with_gflags": [True, False], + "with_snappy": [True, False], + "with_lz4": [True, False], + "with_zlib": [True, False], + "with_zstd": [True, False], + "with_tbb": [True, False], + "with_jemalloc": [True, False], + "enable_sse": [False, "sse42", "avx2"], + "use_rtti": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "lite": False, + "with_snappy": False, + "with_lz4": False, + "with_zlib": False, + "with_zstd": False, + "with_gflags": False, + "with_tbb": False, + "with_jemalloc": False, + "enable_sse": False, + "use_rtti": False, + } + + @property + def _min_cppstd(self): + return "11" if Version(self.version) < "8.8.1" else "17" + + @property + def _compilers_minimum_version(self): + return {} if self._min_cppstd == "11" else { + "apple-clang": "10", + "clang": "7", + "gcc": "7", + "msvc": "191", + "Visual Studio": "15", + } + + def export_sources(self): + export_conandata_patches(self) + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + if self.settings.arch != "x86_64": + del self.options.with_tbb + if self.settings.build_type == "Debug": + self.options.use_rtti = True # Rtti are used in asserts for debug mode... + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self, src_folder="src") + + def requirements(self): + if self.options.with_gflags: + self.requires("gflags/2.2.2") + if self.options.with_snappy: + self.requires("snappy/1.1.10") + if self.options.with_lz4: + self.requires("lz4/1.9.4") + if self.options.with_zlib: + self.requires("zlib/[>=1.2.11 <2]") + if self.options.with_zstd: + self.requires("zstd/1.5.5") + if self.options.get_safe("with_tbb"): + self.requires("onetbb/2021.10.0") + if self.options.with_jemalloc: + self.requires("jemalloc/5.3.0") + + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, self._min_cppstd) + + minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) + if minimum_version and Version(self.settings.compiler.version) < minimum_version: + raise ConanInvalidConfiguration( + f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." + ) + + if self.settings.arch not in ["x86_64", "ppc64le", "ppc64", "mips64", "armv8"]: + raise ConanInvalidConfiguration("Rocksdb requires 64 bits") + + check_min_vs(self, "191") + + if self.version == "6.20.3" and \ + self.settings.os == "Linux" and \ + self.settings.compiler == "gcc" and \ + Version(self.settings.compiler.version) < "5": + raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522 + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["FAIL_ON_WARNINGS"] = False + tc.variables["WITH_TESTS"] = False + tc.variables["WITH_TOOLS"] = False + tc.variables["WITH_CORE_TOOLS"] = False + tc.variables["WITH_BENCHMARK_TOOLS"] = False + tc.variables["WITH_FOLLY_DISTRIBUTED_MUTEX"] = False + if is_msvc(self): + tc.variables["WITH_MD_LIBRARY"] = not is_msvc_static_runtime(self) + tc.variables["ROCKSDB_INSTALL_ON_WINDOWS"] = self.settings.os == "Windows" + tc.variables["ROCKSDB_LITE"] = self.options.lite + tc.variables["WITH_GFLAGS"] = self.options.with_gflags + tc.variables["WITH_SNAPPY"] = self.options.with_snappy + tc.variables["WITH_LZ4"] = self.options.with_lz4 + tc.variables["WITH_ZLIB"] = self.options.with_zlib + tc.variables["WITH_ZSTD"] = self.options.with_zstd + tc.variables["WITH_TBB"] = self.options.get_safe("with_tbb", False) + tc.variables["WITH_JEMALLOC"] = self.options.with_jemalloc + tc.variables["ROCKSDB_BUILD_SHARED"] = self.options.shared + tc.variables["ROCKSDB_LIBRARY_EXPORTS"] = self.settings.os == "Windows" and self.options.shared + tc.variables["ROCKSDB_DLL" ] = self.settings.os == "Windows" and self.options.shared + tc.variables["USE_RTTI"] = self.options.use_rtti + if not bool(self.options.enable_sse): + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + elif self.options.enable_sse == "sse42": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = True + elif self.options.enable_sse == "avx2": + tc.variables["PORTABLE"] = False + tc.variables["FORCE_SSE42"] = False + # not available yet in CCI + tc.variables["WITH_NUMA"] = False + tc.generate() + + deps = CMakeDeps(self) + if self.options.with_jemalloc: + deps.set_property("jemalloc", "cmake_file_name", "JeMalloc") + deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc") + deps.generate() + + def build(self): + apply_conandata_patches(self) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def _remove_static_libraries(self): + rm(self, "rocksdb.lib", os.path.join(self.package_folder, "lib")) + for lib in glob.glob(os.path.join(self.package_folder, "lib", "*.a")): + if not lib.endswith(".dll.a"): + os.remove(lib) + + def _remove_cpp_headers(self): + for path in glob.glob(os.path.join(self.package_folder, "include", "rocksdb", "*")): + if path != os.path.join(self.package_folder, "include", "rocksdb", "c.h"): + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + + def package(self): + copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "LICENSE*", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + cmake = CMake(self) + cmake.install() + if self.options.shared: + self._remove_static_libraries() + self._remove_cpp_headers() # Force stable ABI for shared libraries + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + + def package_info(self): + cmake_target = "rocksdb-shared" if self.options.shared else "rocksdb" + self.cpp_info.set_property("cmake_file_name", "RocksDB") + self.cpp_info.set_property("cmake_target_name", f"RocksDB::{cmake_target}") + # TODO: back to global scope in conan v2 once cmake_find_package* generators removed + self.cpp_info.components["librocksdb"].libs = collect_libs(self) + if self.settings.os == "Windows": + self.cpp_info.components["librocksdb"].system_libs = ["shlwapi", "rpcrt4"] + if self.options.shared: + self.cpp_info.components["librocksdb"].defines = ["ROCKSDB_DLL"] + elif self.settings.os in ["Linux", "FreeBSD"]: + self.cpp_info.components["librocksdb"].system_libs = ["pthread", "m"] + if self.options.lite: + self.cpp_info.components["librocksdb"].defines.append("ROCKSDB_LITE") + + # TODO: to remove in conan v2 once cmake_find_package* generators removed + self.cpp_info.names["cmake_find_package"] = "RocksDB" + self.cpp_info.names["cmake_find_package_multi"] = "RocksDB" + self.cpp_info.components["librocksdb"].names["cmake_find_package"] = cmake_target + self.cpp_info.components["librocksdb"].names["cmake_find_package_multi"] = cmake_target + self.cpp_info.components["librocksdb"].set_property("cmake_target_name", f"RocksDB::{cmake_target}") + if self.options.with_gflags: + self.cpp_info.components["librocksdb"].requires.append("gflags::gflags") + if self.options.with_snappy: + self.cpp_info.components["librocksdb"].requires.append("snappy::snappy") + if self.options.with_lz4: + self.cpp_info.components["librocksdb"].requires.append("lz4::lz4") + if self.options.with_zlib: + self.cpp_info.components["librocksdb"].requires.append("zlib::zlib") + if self.options.with_zstd: + self.cpp_info.components["librocksdb"].requires.append("zstd::zstd") + if self.options.get_safe("with_tbb"): + self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb") + if self.options.with_jemalloc: + self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc") diff --git a/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch b/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch new file mode 100644 index 00000000000..05725bf2c9a --- /dev/null +++ b/external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch @@ -0,0 +1,30 @@ +--- a/include/rocksdb/utilities/checkpoint.h ++++ b/include/rocksdb/utilities/checkpoint.h +@@ -8,6 +8,7 @@ + #pragma once + #ifndef ROCKSDB_LITE + ++#include + #include + #include + #include "rocksdb/status.h" +--- a/table/block_based/data_block_hash_index.h ++++ b/table/block_based/data_block_hash_index.h +@@ -5,6 +5,7 @@ + + #pragma once + ++#include + #include + #include + +--- a/util/string_util.h ++++ b/util/string_util.h +@@ -6,6 +6,7 @@ + + #pragma once + ++#include + #include + #include + #include diff --git a/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch new file mode 100644 index 00000000000..fb0dd0c46b4 --- /dev/null +++ b/external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch @@ -0,0 +1,16 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ec59d4491..35577c998 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -101 +100,0 @@ if(MSVC) +- option(WITH_GFLAGS "build with GFlags" OFF) +@@ -103,2 +102,2 @@ if(MSVC) +- include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc) +-else() ++endif() ++ +@@ -117 +116 @@ else() +- if(MINGW) ++ if(MINGW OR MSVC) +@@ -183 +181,0 @@ else() +-endif() diff --git a/src/ripple/app/consensus/RCLConsensus.cpp b/src/ripple/app/consensus/RCLConsensus.cpp index 4e973ef2bdf..c0ebe06dd7e 100644 --- a/src/ripple/app/consensus/RCLConsensus.cpp +++ b/src/ripple/app/consensus/RCLConsensus.cpp @@ -55,7 +55,7 @@ RCLConsensus::RCLConsensus( LedgerMaster& ledgerMaster, LocalTxs& localTxs, InboundTransactions& inboundTransactions, - Consensus::clock_type& clock, + Consensus::clock_type const& clock, ValidatorKeys const& validatorKeys, beast::Journal journal) : adaptor_( @@ -98,20 +98,22 @@ RCLConsensus::Adaptor::Adaptor( JLOG(j_.info()) << "Consensus engine started (cookie: " + std::to_string(valCookie_) + ")"; - if (validatorKeys_.nodeID != beast::zero) + if (validatorKeys_.nodeID != beast::zero && validatorKeys_.keys) { std::stringstream ss; JLOG(j_.info()) << "Validator identity: " << toBase58( TokenType::NodePublic, - validatorKeys_.masterPublicKey); + validatorKeys_.keys->masterPublicKey); - if (validatorKeys_.masterPublicKey != validatorKeys_.publicKey) + if (validatorKeys_.keys->masterPublicKey != + validatorKeys_.keys->publicKey) { JLOG(j_.debug()) << "Validator ephemeral signing key: " - << toBase58(TokenType::NodePublic, validatorKeys_.publicKey) + << toBase58( + TokenType::NodePublic, validatorKeys_.keys->publicKey) << " (seq: " << std::to_string(validatorKeys_.sequence) << ")"; } } @@ -171,9 +173,6 @@ RCLConsensus::Adaptor::share(RCLCxPeerPos const& peerPos) auto const sig = peerPos.signature(); prop.set_signature(sig.data(), sig.size()); - if (proposal.ledgerSeq().has_value()) - prop.set_ledgerseq(*proposal.ledgerSeq()); - app_.overlay().relay(prop, peerPos.suppressionID(), peerPos.publicKey()); } @@ -183,7 +182,7 @@ RCLConsensus::Adaptor::share(RCLCxTx const& tx) // If we didn't relay this transaction recently, relay it to all peers if (app_.getHashRouter().shouldRelay(tx.id())) { - JLOG(j_.trace()) << "Relaying disputed tx " << tx.id(); + JLOG(j_.debug()) << "Relaying disputed tx " << tx.id(); auto const slice = tx.tx_->slice(); protocol::TMTransaction msg; msg.set_rawtransaction(slice.data(), slice.size()); @@ -195,13 +194,13 @@ RCLConsensus::Adaptor::share(RCLCxTx const& tx) } else { - JLOG(j_.trace()) << "Not relaying disputed tx " << tx.id(); + JLOG(j_.debug()) << "Not relaying disputed tx " << tx.id(); } } void RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) { - JLOG(j_.debug()) << (proposal.isBowOut() ? "We bow out: " : "We propose: ") + JLOG(j_.trace()) << (proposal.isBowOut() ? "We bow out: " : "We propose: ") << ripple::to_string(proposal.prevLedger()) << " -> " << ripple::to_string(proposal.position()); @@ -213,14 +212,20 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) proposal.prevLedger().begin(), proposal.prevLedger().size()); prop.set_proposeseq(proposal.proposeSeq()); prop.set_closetime(proposal.closeTime().time_since_epoch().count()); - prop.set_nodepubkey( - validatorKeys_.publicKey.data(), validatorKeys_.publicKey.size()); - prop.set_ledgerseq(*proposal.ledgerSeq()); - auto sig = signDigest( - validatorKeys_.publicKey, - validatorKeys_.secretKey, - proposal.signingHash()); + if (!validatorKeys_.keys) + { + JLOG(j_.warn()) << "RCLConsensus::Adaptor::propose: ValidatorKeys " + "not set: \n"; + return; + } + + auto const& keys = *validatorKeys_.keys; + + prop.set_nodepubkey(keys.publicKey.data(), keys.publicKey.size()); + + auto sig = + signDigest(keys.publicKey, keys.secretKey, proposal.signingHash()); prop.set_signature(sig.data(), sig.size()); @@ -229,7 +234,7 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) proposal.prevLedger(), proposal.proposeSeq(), proposal.closeTime(), - validatorKeys_.publicKey, + keys.publicKey, sig); app_.getHashRouter().addSuppression(suppression); @@ -301,8 +306,7 @@ auto RCLConsensus::Adaptor::onClose( RCLCxLedger const& ledger, NetClock::time_point const& closeTime, - ConsensusMode mode, - clock_type& clock) -> Result + ConsensusMode mode) -> Result { const bool wrongLCL = mode == ConsensusMode::wrongLedger; const bool proposing = mode == ConsensusMode::proposing; @@ -384,6 +388,7 @@ RCLConsensus::Adaptor::onClose( // Needed because of the move below. auto const setHash = initialSet->getHash().as_uint256(); + return Result{ std::move(initialSet), RCLCxPeerPos::Proposal{ @@ -392,9 +397,7 @@ RCLConsensus::Adaptor::onClose( setHash, closeTime, app_.timeKeeper().closeTime(), - validatorKeys_.nodeID, - initialLedger->info().seq, - clock}}; + validatorKeys_.nodeID}}; } void @@ -406,43 +409,50 @@ RCLConsensus::Adaptor::onForceAccept( ConsensusMode const& mode, Json::Value&& consensusJson) { - auto txsBuilt = buildAndValidate( - result, prevLedger, closeResolution, mode, std::move(consensusJson)); - prepareOpenLedger(std::move(txsBuilt), result, rawCloseTimes, mode); + doAccept( + result, + prevLedger, + closeResolution, + rawCloseTimes, + mode, + std::move(consensusJson)); } void RCLConsensus::Adaptor::onAccept( Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration const& closeResolution, ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, - Json::Value&& consensusJson, - std::pair&& tb) + Json::Value&& consensusJson) { app_.getJobQueue().addJob( jtACCEPT, "acceptLedger", - [=, - this, - cj = std::move(consensusJson), - txsBuilt = std::move(tb)]() mutable { + [=, this, cj = std::move(consensusJson)]() mutable { // Note that no lock is held or acquired during this job. // This is because generic Consensus guarantees that once a ledger // is accepted, the consensus results and capture by reference state // will not change until startRound is called (which happens via // endConsensus). - prepareOpenLedger(std::move(txsBuilt), result, rawCloseTimes, mode); + this->doAccept( + result, + prevLedger, + closeResolution, + rawCloseTimes, + mode, + std::move(cj)); this->app_.getOPs().endConsensus(); }); } -std::pair< - RCLConsensus::Adaptor::CanonicalTxSet_t, - RCLConsensus::Adaptor::Ledger_t> -RCLConsensus::Adaptor::buildAndValidate( +void +RCLConsensus::Adaptor::doAccept( Result const& result, - Ledger_t const& prevLedger, - NetClock::duration const& closeResolution, + RCLCxLedger const& prevLedger, + NetClock::duration closeResolution, + ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, Json::Value&& consensusJson) { @@ -496,12 +506,12 @@ RCLConsensus::Adaptor::buildAndValidate( { retriableTxs.insert( std::make_shared(SerialIter{item.slice()})); - JLOG(j_.trace()) << " Tx: " << item.key(); + JLOG(j_.debug()) << " Tx: " << item.key(); } catch (std::exception const& ex) { failed.insert(item.key()); - JLOG(j_.trace()) + JLOG(j_.warn()) << " Tx: " << item.key() << " throws: " << ex.what(); } } @@ -578,19 +588,6 @@ RCLConsensus::Adaptor::buildAndValidate( ledgerMaster_.consensusBuilt( built.ledger_, result.txns.id(), std::move(consensusJson)); - return {retriableTxs, built}; -} - -void -RCLConsensus::Adaptor::prepareOpenLedger( - std::pair&& txsBuilt, - Result const& result, - ConsensusCloseTimes const& rawCloseTimes, - ConsensusMode const& mode) -{ - auto& retriableTxs = txsBuilt.first; - auto const& built = txsBuilt.second; - //------------------------------------------------------------------------- { // Apply disputed transactions that didn't get in @@ -613,7 +610,7 @@ RCLConsensus::Adaptor::prepareOpenLedger( // we voted NO try { - JLOG(j_.trace()) + JLOG(j_.debug()) << "Test applying disputed transaction that did" << " not get in " << dispute.tx().id(); @@ -631,7 +628,7 @@ RCLConsensus::Adaptor::prepareOpenLedger( } catch (std::exception const& ex) { - JLOG(j_.trace()) << "Failed to apply transaction we voted " + JLOG(j_.debug()) << "Failed to apply transaction we voted " "NO on. Exception: " << ex.what(); } @@ -681,7 +678,6 @@ RCLConsensus::Adaptor::prepareOpenLedger( // we entered the round with the network, // see how close our close time is to other node's // close time reports, and update our clock. - bool const consensusFail = result.state == ConsensusState::MovedOn; if ((mode == ConsensusMode::proposing || mode == ConsensusMode::observing) && !consensusFail) @@ -812,10 +808,19 @@ RCLConsensus::Adaptor::validate( validationTime = lastValidationTime_ + 1s; lastValidationTime_ = validationTime; + if (!validatorKeys_.keys) + { + JLOG(j_.warn()) << "RCLConsensus::Adaptor::validate: ValidatorKeys " + "not set\n"; + return; + } + + auto const& keys = *validatorKeys_.keys; + auto v = std::make_shared( lastValidationTime_, - validatorKeys_.publicKey, - validatorKeys_.secretKey, + keys.publicKey, + keys.secretKey, validatorKeys_.nodeID, [&](STValidation& v) { v.setFieldH256(sfLedgerHash, ledger.id()); @@ -902,30 +907,12 @@ RCLConsensus::Adaptor::onModeChange(ConsensusMode before, ConsensusMode after) mode_ = after; } -bool -RCLConsensus::Adaptor::retryAccept( - Ledger_t const& newLedger, - std::optional>& start) - const -{ - static bool const standalone = ledgerMaster_.standalone(); - auto const& validLedger = ledgerMaster_.getValidatedLedger(); - - return (app_.getOPs().isFull() && !standalone && - (validLedger && (newLedger.id() != validLedger->info().hash) && - (newLedger.seq() >= validLedger->info().seq))) && - (!start || - std::chrono::steady_clock::now() - *start < std::chrono::seconds{5}); -} - -//----------------------------------------------------------------------------- - Json::Value RCLConsensus::getJson(bool full) const { Json::Value ret; { - std::lock_guard _{adaptor_.peekMutex()}; + std::lock_guard _{mutex_}; ret = consensus_.getJson(full); } ret["validating"] = adaptor_.validating(); @@ -937,7 +924,7 @@ RCLConsensus::timerEntry(NetClock::time_point const& now) { try { - std::lock_guard _{adaptor_.peekMutex()}; + std::lock_guard _{mutex_}; consensus_.timerEntry(now); } catch (SHAMapMissingNode const& mn) @@ -953,7 +940,7 @@ RCLConsensus::gotTxSet(NetClock::time_point const& now, RCLTxSet const& txSet) { try { - std::lock_guard _{adaptor_.peekMutex()}; + std::lock_guard _{mutex_}; consensus_.gotTxSet(now, txSet); } catch (SHAMapMissingNode const& mn) @@ -971,7 +958,7 @@ RCLConsensus::simulate( NetClock::time_point const& now, std::optional consensusDelay) { - std::lock_guard _{adaptor_.peekMutex()}; + std::lock_guard _{mutex_}; consensus_.simulate(now, consensusDelay); } @@ -980,7 +967,7 @@ RCLConsensus::peerProposal( NetClock::time_point const& now, RCLCxPeerPos const& newProposal) { - std::lock_guard _{adaptor_.peekMutex()}; + std::lock_guard _{mutex_}; return consensus_.peerProposal(now, newProposal); } @@ -991,7 +978,7 @@ RCLConsensus::Adaptor::preStartRound( { // We have a key, we do not want out of sync validations after a restart // and are not amendment blocked. - validating_ = validatorKeys_.publicKey.size() != 0 && + validating_ = validatorKeys_.keys && prevLgr.seq() >= app_.getMaxDisallowedLedger() && !app_.getOPs().isBlocked(); @@ -1053,12 +1040,6 @@ RCLConsensus::Adaptor::getQuorumKeys() const return app_.validators().getQuorumKeys(); } -std::size_t -RCLConsensus::Adaptor::quorum() const -{ - return app_.validators().quorum(); -} - std::size_t RCLConsensus::Adaptor::laggards( Ledger_t::Seq const seq, @@ -1070,7 +1051,7 @@ RCLConsensus::Adaptor::laggards( bool RCLConsensus::Adaptor::validator() const { - return !validatorKeys_.publicKey.empty(); + return validatorKeys_.keys.has_value(); } void @@ -1088,7 +1069,7 @@ RCLConsensus::startRound( hash_set const& nowUntrusted, hash_set const& nowTrusted) { - std::lock_guard _{adaptor_.peekMutex()}; + std::lock_guard _{mutex_}; consensus_.startRound( now, prevLgrId, @@ -1096,5 +1077,4 @@ RCLConsensus::startRound( nowUntrusted, adaptor_.preStartRound(prevLgr, nowTrusted)); } - } // namespace ripple diff --git a/src/ripple/app/consensus/RCLConsensus.h b/src/ripple/app/consensus/RCLConsensus.h index 4e6dce1efd3..f8c01e93caa 100644 --- a/src/ripple/app/consensus/RCLConsensus.h +++ b/src/ripple/app/consensus/RCLConsensus.h @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -37,11 +36,8 @@ #include #include #include -#include #include -#include #include - namespace ripple { class InboundTransactions; @@ -63,7 +59,6 @@ class RCLConsensus Application& app_; std::unique_ptr feeVote_; LedgerMaster& ledgerMaster_; - LocalTxs& localTxs_; InboundTransactions& inboundTransactions_; beast::Journal const j_; @@ -83,6 +78,7 @@ class RCLConsensus // These members are queried via public accesors and are atomic for // thread safety. + std::atomic validating_{false}; std::atomic prevProposers_{0}; std::atomic prevRoundTime_{ std::chrono::milliseconds{0}}; @@ -91,25 +87,14 @@ class RCLConsensus RCLCensorshipDetector censorshipDetector_; NegativeUNLVote nUnlVote_; - // Since Consensus does not provide intrinsic thread-safety, this mutex - // needs to guard all calls to consensus_. One reason it is recursive - // is because logic in phaseEstablish() around buildAndValidate() - // needs to lock and unlock to protect Consensus data members. - mutable std::recursive_mutex mutex_; - std::optional validationDelay_; - std::optional timerDelay_; - std::atomic validating_{false}; - public: using Ledger_t = RCLCxLedger; using NodeID_t = NodeID; using NodeKey_t = PublicKey; using TxSet_t = RCLTxSet; - using CanonicalTxSet_t = CanonicalTXSet; using PeerPosition_t = RCLCxPeerPos; using Result = ConsensusResult; - using clock_type = Stopwatch; Adaptor( Application& app, @@ -164,9 +149,6 @@ class RCLConsensus std::pair> getQuorumKeys() const; - std::size_t - quorum() const; - std::size_t laggards(Ledger_t::Seq const seq, hash_set& trustedKeys) const; @@ -196,93 +178,6 @@ class RCLConsensus return parms_; } - std::recursive_mutex& - peekMutex() const - { - return mutex_; - } - - LedgerMaster& - getLedgerMaster() const - { - return ledgerMaster_; - } - - void - clearValidating() - { - validating_ = false; - } - - /** Whether to try building another ledger to validate. - * - * This should be called when a newly-created ledger hasn't been - * validated to avoid us forking to an invalid ledger. - * - * Retry only if all of the below are true: - * * We are synced to the network. - * * Not in standalone mode. - * * We have validated a ledger. - * * The latest validated ledger and the new ledger are different. - * * The new ledger sequence is >= the validated ledger. - * * Less than 5 seconds have elapsed retrying. - * - * @param newLedger The new ledger which we have created. - * @param start When we started possibly retrying ledgers. - * @return Whether to retry. - */ - bool - retryAccept( - Ledger_t const& newLedger, - std::optional>& - start) const; - - /** Amount of time delayed waiting to confirm validation. - * - * @return Time in milliseconds. - */ - std::optional - getValidationDelay() const - { - return validationDelay_; - } - - /** Set amount of time that has been delayed waiting for validation. - * - * Clear if nothing passed. - * - * @param vd Amount of time in milliseconds. - */ - void - setValidationDelay( - std::optional vd = std::nullopt) - { - validationDelay_ = vd; - } - - /** Amount of time to wait for heartbeat. - * - * @return Time in milliseconds. - */ - std::optional - getTimerDelay() const - { - return timerDelay_; - } - - /** Set amount of time to wait for next heartbeat. - * - * Clear if nothing passed. - * - * @param td Amount of time in milliseconds. - */ - void - setTimerDelay( - std::optional td = std::nullopt) - { - timerDelay_ = td; - } - private: //--------------------------------------------------------------------- // The following members implement the generic Consensus requirements @@ -402,34 +297,34 @@ class RCLConsensus @param ledger the ledger we are changing to @param closeTime When consensus closed the ledger @param mode Current consensus mode - @param clock Clock used for Consensus and testing. @return Tentative consensus result */ Result onClose( RCLCxLedger const& ledger, NetClock::time_point const& closeTime, - ConsensusMode mode, - clock_type& clock); + ConsensusMode mode); /** Process the accepted ledger. @param result The result of consensus + @param prevLedger The closed ledger consensus worked from + @param closeResolution The resolution used in agreeing on an + effective closeTime @param rawCloseTimes The unrounded closetimes of ourself and our peers @param mode Our participating mode at the time consensus was declared @param consensusJson Json representation of consensus state - @param txsBuilt The consensus transaction set and new ledger built - around it */ void onAccept( Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration const& closeResolution, ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, - Json::Value&& consensusJson, - std::pair&& txsBuilt); + Json::Value&& consensusJson); /** Process the accepted ledger that was a result of simulation/force accept. @@ -457,40 +352,18 @@ class RCLConsensus RCLCxLedger const& ledger, bool haveCorrectLCL); - /** Build and attempt to validate a new ledger. - * - * @param result The result of consensus. - * @param prevLedger The closed ledger from which this is to be based. - * @param closeResolution The resolution used in agreeing on an - * effective closeTime. - * @param mode Our participating mode at the time consensus was - * declared. - * @param consensusJson Json representation of consensus state. - * @return The consensus transaction set and resulting ledger. - */ - std::pair - buildAndValidate( - Result const& result, - Ledger_t const& prevLedger, - NetClock::duration const& closeResolution, - ConsensusMode const& mode, - Json::Value&& consensusJson); + /** Accept a new ledger based on the given transactions. - /** Prepare the next open ledger. - * - * @param txsBuilt The consensus transaction set and resulting ledger. - * @param result The result of consensus. - * @param rawCloseTimes The unrounded closetimes of our peers and - * ourself. - * @param mode Our participating mode at the time consensus was - declared. + @ref onAccept */ void - prepareOpenLedger( - std::pair&& txsBuilt, + doAccept( Result const& result, + RCLCxLedger const& prevLedger, + NetClock::duration closeResolution, ConsensusCloseTimes const& rawCloseTimes, - ConsensusMode const& mode); + ConsensusMode const& mode, + Json::Value&& consensusJson); /** Build the new last closed ledger. @@ -548,7 +421,7 @@ class RCLConsensus LedgerMaster& ledgerMaster, LocalTxs& localTxs, InboundTransactions& inboundTransactions, - Consensus::clock_type& clock, + Consensus::clock_type const& clock, ValidatorKeys const& validatorKeys, beast::Journal journal); @@ -625,7 +498,7 @@ class RCLConsensus RCLCxLedger::ID prevLedgerID() const { - std::lock_guard _{adaptor_.peekMutex()}; + std::lock_guard _{mutex_}; return consensus_.prevLedgerID(); } @@ -647,19 +520,12 @@ class RCLConsensus return adaptor_.parms(); } - std::optional - getTimerDelay() const - { - return adaptor_.getTimerDelay(); - } - - void - setTimerDelay(std::optional td = std::nullopt) - { - adaptor_.setTimerDelay(td); - } - private: + // Since Consensus does not provide intrinsic thread-safety, this mutex + // guards all calls to consensus_. adaptor_ uses atomics internally + // to allow concurrent access of its data members that have getters. + mutable std::recursive_mutex mutex_; + Adaptor adaptor_; Consensus consensus_; beast::Journal const j_; diff --git a/src/ripple/app/consensus/RCLCxPeerPos.h b/src/ripple/app/consensus/RCLCxPeerPos.h index f104299b770..e82a85d422b 100644 --- a/src/ripple/app/consensus/RCLCxPeerPos.h +++ b/src/ripple/app/consensus/RCLCxPeerPos.h @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -45,7 +44,7 @@ class RCLCxPeerPos { public: //< The type of the proposed position - using Proposal = ConsensusProposal; + using Proposal = ConsensusProposal; /** Constructor diff --git a/src/ripple/app/ledger/BookListeners.cpp b/src/ripple/app/ledger/BookListeners.cpp index bbc0058bc76..3c7e013e1dd 100644 --- a/src/ripple/app/ledger/BookListeners.cpp +++ b/src/ripple/app/ledger/BookListeners.cpp @@ -54,8 +54,9 @@ BookListeners::publish( // Only publish jvObj if this is the first occurence if (havePublished.emplace(p->getSeq()).second) { - p->send( - jvObj.select(apiVersionSelector(p->getApiVersion())), true); + jvObj.visit( + p->getApiVersion(), // + [&](Json::Value const& jv) { p->send(jv, true); }); } ++it; } diff --git a/src/ripple/app/ledger/BookListeners.h b/src/ripple/app/ledger/BookListeners.h index 748378a12b1..605cf6dc6af 100644 --- a/src/ripple/app/ledger/BookListeners.h +++ b/src/ripple/app/ledger/BookListeners.h @@ -20,8 +20,8 @@ #ifndef RIPPLE_APP_LEDGER_BOOKLISTENERS_H_INCLUDED #define RIPPLE_APP_LEDGER_BOOKLISTENERS_H_INCLUDED -#include #include +#include #include #include diff --git a/src/ripple/app/ledger/InboundLedgers.h b/src/ripple/app/ledger/InboundLedgers.h index dca4a80c54b..b12760153e2 100644 --- a/src/ripple/app/ledger/InboundLedgers.h +++ b/src/ripple/app/ledger/InboundLedgers.h @@ -83,6 +83,9 @@ class InboundLedgers virtual void stop() = 0; + + virtual std::size_t + cacheSize() = 0; }; std::unique_ptr diff --git a/src/ripple/app/ledger/LedgerMaster.h b/src/ripple/app/ledger/LedgerMaster.h index e2ca3039935..980506c2267 100644 --- a/src/ripple/app/ledger/LedgerMaster.h +++ b/src/ripple/app/ledger/LedgerMaster.h @@ -294,27 +294,6 @@ class LedgerMaster : public AbstractFetchPackContainer std::optional minSqlSeq(); - //! Whether we are in standalone mode. - bool - standalone() const - { - return standalone_; - } - - /** Wait up to a specified duration for the next validated ledger. - * - * @tparam Rep std::chrono duration Rep. - * @tparam Period std::chrono duration Period. - * @param dur Duration to wait. - */ - template - void - waitForValidated(std::chrono::duration const& dur) - { - std::unique_lock lock(validMutex_); - validCond_.wait_for(lock, dur); - } - // Iff a txn exists at the specified ledger and offset then return its txnid std::optional txnIdFromIndex(uint32_t ledgerSeq, uint32_t txnIndex); @@ -435,10 +414,7 @@ class LedgerMaster : public AbstractFetchPackContainer // Time that the previous upgrade warning was issued. TimeKeeper::time_point upgradeWarningPrevTime_{}; - // mutex and condition variable for waiting for next validated ledger - std::mutex validMutex_; - std::condition_variable validCond_; - +private: struct Stats { template @@ -460,6 +436,7 @@ class LedgerMaster : public AbstractFetchPackContainer Stats m_stats; +private: void collect_metrics() { diff --git a/src/ripple/app/ledger/LedgerReplayer.h b/src/ripple/app/ledger/LedgerReplayer.h index 6866250485d..b06dd2cc858 100644 --- a/src/ripple/app/ledger/LedgerReplayer.h +++ b/src/ripple/app/ledger/LedgerReplayer.h @@ -125,6 +125,27 @@ class LedgerReplayer final void stop(); + std::size_t + tasksSize() const + { + std::lock_guard lock(mtx_); + return tasks_.size(); + } + + std::size_t + deltasSize() const + { + std::lock_guard lock(mtx_); + return deltas_.size(); + } + + std::size_t + skipListsSize() const + { + std::lock_guard lock(mtx_); + return skipLists_.size(); + } + private: mutable std::mutex mtx_; std::vector> tasks_; diff --git a/src/ripple/app/ledger/OrderBookDB.cpp b/src/ripple/app/ledger/OrderBookDB.cpp index faa957203a8..e1fc8a248f0 100644 --- a/src/ripple/app/ledger/OrderBookDB.cpp +++ b/src/ripple/app/ledger/OrderBookDB.cpp @@ -47,7 +47,7 @@ OrderBookDB::setup(std::shared_ptr const& ledger) if (seq != 0) { - if ((seq > ledger->seq()) && ((ledger->seq() - seq) < 25600)) + if ((ledger->seq() > seq) && ((ledger->seq() - seq) < 25600)) return; if ((ledger->seq() <= seq) && ((seq - ledger->seq()) < 16)) diff --git a/src/ripple/app/ledger/OrderBookDB.h b/src/ripple/app/ledger/OrderBookDB.h index b072bafb0c3..45705a61572 100644 --- a/src/ripple/app/ledger/OrderBookDB.h +++ b/src/ripple/app/ledger/OrderBookDB.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include diff --git a/src/ripple/app/ledger/impl/InboundLedgers.cpp b/src/ripple/app/ledger/impl/InboundLedgers.cpp index ec422af131c..f20bc50738c 100644 --- a/src/ripple/app/ledger/impl/InboundLedgers.cpp +++ b/src/ripple/app/ledger/impl/InboundLedgers.cpp @@ -447,6 +447,13 @@ class InboundLedgersImp : public InboundLedgers mRecentFailures.clear(); } + std::size_t + cacheSize() override + { + ScopedLockType lock(mLock); + return mLedgers.size(); + } + private: clock_type& m_clock; diff --git a/src/ripple/app/ledger/impl/LedgerMaster.cpp b/src/ripple/app/ledger/impl/LedgerMaster.cpp index 857c0efcc28..9388a3005ba 100644 --- a/src/ripple/app/ledger/impl/LedgerMaster.cpp +++ b/src/ripple/app/ledger/impl/LedgerMaster.cpp @@ -367,8 +367,6 @@ LedgerMaster::setValidLedger(std::shared_ptr const& l) } mValidLedger.set(l); - // In case we're waiting for a valid before proceeding with Consensus. - validCond_.notify_one(); mValidLedgerSign = signTime.time_since_epoch().count(); assert( mValidLedgerSeq || !app_.getMaxDisallowedLedger() || @@ -551,25 +549,22 @@ void LedgerMaster::applyHeldTransactions() { std::lock_guard sl(m_mutex); - // It can be expensive to modify the open ledger even with no transactions - // to process. Regardless, make sure to reset held transactions with - // the parent. - if (mHeldTransactions.size()) - { - app_.openLedger().modify([&](OpenView& view, beast::Journal j) { - bool any = false; - for (auto const& it : mHeldTransactions) - { - ApplyFlags flags = tapNONE; - auto const result = - app_.getTxQ().apply(app_, view, it.second, flags, j); - if (result.second) - any = true; - } - return any; - }); - } + app_.openLedger().modify([&](OpenView& view, beast::Journal j) { + bool any = false; + for (auto const& it : mHeldTransactions) + { + ApplyFlags flags = tapNONE; + auto const result = + app_.getTxQ().apply(app_, view, it.second, flags, j); + if (result.second) + any = true; + } + return any; + }); + + // VFALCO TODO recreate the CanonicalTxSet object instead of resetting + // it. // VFALCO NOTE The hash for an open ledger is undefined so we use // something that is a reasonable substitute. mHeldTransactions.reset(app_.openLedger().current()->info().parentHash); @@ -1548,6 +1543,7 @@ LedgerMaster::updatePaths() if (app_.getOPs().isNeedNetworkLedger()) { --mPathFindThread; + mPathLedger.reset(); JLOG(m_journal.debug()) << "Need network ledger for updating paths"; return; } @@ -1573,6 +1569,7 @@ LedgerMaster::updatePaths() else { // Nothing to do --mPathFindThread; + mPathLedger.reset(); JLOG(m_journal.debug()) << "Nothing to do for updating paths"; return; } @@ -1589,6 +1586,7 @@ LedgerMaster::updatePaths() << "Published ledger too old for updating paths"; std::lock_guard ml(m_mutex); --mPathFindThread; + mPathLedger.reset(); return; } } @@ -1601,6 +1599,7 @@ LedgerMaster::updatePaths() if (!pathRequests.requestsPending()) { --mPathFindThread; + mPathLedger.reset(); JLOG(m_journal.debug()) << "No path requests found. Nothing to do for updating " "paths. " @@ -1618,6 +1617,7 @@ LedgerMaster::updatePaths() << "No path requests left. No need for further updating " "paths"; --mPathFindThread; + mPathLedger.reset(); return; } } diff --git a/src/ripple/app/ledger/impl/LedgerToJson.cpp b/src/ripple/app/ledger/impl/LedgerToJson.cpp index d22cc7cd487..1310dd13a65 100644 --- a/src/ripple/app/ledger/impl/LedgerToJson.cpp +++ b/src/ripple/app/ledger/impl/LedgerToJson.cpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace ripple { @@ -52,10 +53,17 @@ isBinary(LedgerFill const& fill) template void -fillJson(Object& json, bool closed, LedgerInfo const& info, bool bFull) +fillJson( + Object& json, + bool closed, + LedgerInfo const& info, + bool bFull, + unsigned apiVersion) { json[jss::parent_hash] = to_string(info.parentHash); - json[jss::ledger_index] = to_string(info.seq); + json[jss::ledger_index] = (apiVersion > 1) + ? Json::Value(info.seq) + : Json::Value(std::to_string(info.seq)); if (closed) { @@ -159,7 +167,8 @@ fillJsonTx( txJson[jss::validated] = validated; if (validated) { - txJson[jss::ledger_index] = to_string(fill.ledger.seq()); + auto const seq = fill.ledger.seq(); + txJson[jss::ledger_index] = seq; if (fill.closeTime) txJson[jss::close_time_iso] = to_string_iso(*fill.closeTime); } @@ -315,7 +324,13 @@ fillJson(Object& json, LedgerFill const& fill) if (isBinary(fill)) fillJsonBinary(json, !fill.ledger.open(), fill.ledger.info()); else - fillJson(json, !fill.ledger.open(), fill.ledger.info(), bFull); + fillJson( + json, + !fill.ledger.open(), + fill.ledger.info(), + bFull, + (fill.context ? fill.context->apiVersion + : RPC::apiMaximumSupportedVersion)); if (bFull || fill.options & LedgerFill::dumpTxrp) fillJsonTx(json, fill); diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 08ba296b271..55300a390c9 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -185,7 +185,7 @@ class ApplicationImp : public Application, public BasicApp NodeCache m_tempNodeCache; CachedSLEs cachedSLEs_; - std::pair nodeIdentity_; + std::optional> nodeIdentity_; ValidatorKeys const validatorKeys_; std::unique_ptr m_resourceManager; @@ -587,13 +587,20 @@ class ApplicationImp : public Application, public BasicApp std::pair const& nodeIdentity() override { - return nodeIdentity_; + if (nodeIdentity_) + return *nodeIdentity_; + + LogicError( + "Accessing Application::nodeIdentity() before it is initialized."); } - PublicKey const& + std::optional getValidationPublicKey() const override { - return validatorKeys_.publicKey; + if (!validatorKeys_.keys) + return {}; + + return validatorKeys_.keys->publicKey; } NetworkOPs& @@ -1065,20 +1072,172 @@ class ApplicationImp : public Application, public BasicApp // VFALCO TODO fix the dependency inversion using an observer, // have listeners register for "onSweep ()" notification. - nodeFamily_.sweep(); + { + std::shared_ptr const fullBelowCache = + nodeFamily_.getFullBelowCache(0); + + std::shared_ptr const treeNodeCache = + nodeFamily_.getTreeNodeCache(0); + + std::size_t const oldFullBelowSize = fullBelowCache->size(); + std::size_t const oldTreeNodeSize = treeNodeCache->size(); + + nodeFamily_.sweep(); + + JLOG(m_journal.debug()) + << "NodeFamily::FullBelowCache sweep. Size before: " + << oldFullBelowSize + << "; size after: " << fullBelowCache->size(); + + JLOG(m_journal.debug()) + << "NodeFamily::TreeNodeCache sweep. Size before: " + << oldTreeNodeSize << "; size after: " << treeNodeCache->size(); + } if (shardFamily_) + { + std::size_t const oldFullBelowSize = + shardFamily_->getFullBelowCacheSize(); + std::size_t const oldTreeNodeSize = + shardFamily_->getTreeNodeCacheSize().second; + shardFamily_->sweep(); - getMasterTransaction().sweep(); - getNodeStore().sweep(); + + JLOG(m_journal.debug()) + << "ShardFamily::FullBelowCache sweep. Size before: " + << oldFullBelowSize + << "; size after: " << shardFamily_->getFullBelowCacheSize(); + + JLOG(m_journal.debug()) + << "ShardFamily::TreeNodeCache sweep. Size before: " + << oldTreeNodeSize << "; size after: " + << shardFamily_->getTreeNodeCacheSize().second; + } + { + TaggedCache const& masterTxCache = + getMasterTransaction().getCache(); + + std::size_t const oldMasterTxSize = masterTxCache.size(); + + getMasterTransaction().sweep(); + + JLOG(m_journal.debug()) + << "MasterTransaction sweep. Size before: " << oldMasterTxSize + << "; size after: " << masterTxCache.size(); + } + { + // Does not appear to have an associated cache. + getNodeStore().sweep(); + } if (shardStore_) + { + // Does not appear to have an associated cache. shardStore_->sweep(); - getLedgerMaster().sweep(); - getTempNodeCache().sweep(); - getValidations().expire(m_journal); - getInboundLedgers().sweep(); - getLedgerReplayer().sweep(); - m_acceptedLedgerCache.sweep(); - cachedSLEs_.sweep(); + } + { + std::size_t const oldLedgerMasterCacheSize = + getLedgerMaster().getFetchPackCacheSize(); + + getLedgerMaster().sweep(); + + JLOG(m_journal.debug()) + << "LedgerMaster sweep. Size before: " + << oldLedgerMasterCacheSize << "; size after: " + << getLedgerMaster().getFetchPackCacheSize(); + } + { + // NodeCache == TaggedCache + std::size_t const oldTempNodeCacheSize = getTempNodeCache().size(); + + getTempNodeCache().sweep(); + + JLOG(m_journal.debug()) + << "TempNodeCache sweep. Size before: " << oldTempNodeCacheSize + << "; size after: " << getTempNodeCache().size(); + } + { + std::size_t const oldCurrentCacheSize = + getValidations().sizeOfCurrentCache(); + std::size_t const oldSizeSeqEnforcesSize = + getValidations().sizeOfSeqEnforcersCache(); + std::size_t const oldByLedgerSize = + getValidations().sizeOfByLedgerCache(); + std::size_t const oldBySequenceSize = + getValidations().sizeOfBySequenceCache(); + + getValidations().expire(m_journal); + + JLOG(m_journal.debug()) + << "Validations Current expire. Size before: " + << oldCurrentCacheSize + << "; size after: " << getValidations().sizeOfCurrentCache(); + + JLOG(m_journal.debug()) + << "Validations SeqEnforcer expire. Size before: " + << oldSizeSeqEnforcesSize << "; size after: " + << getValidations().sizeOfSeqEnforcersCache(); + + JLOG(m_journal.debug()) + << "Validations ByLedger expire. Size before: " + << oldByLedgerSize + << "; size after: " << getValidations().sizeOfByLedgerCache(); + + JLOG(m_journal.debug()) + << "Validations BySequence expire. Size before: " + << oldBySequenceSize + << "; size after: " << getValidations().sizeOfBySequenceCache(); + } + { + std::size_t const oldInboundLedgersSize = + getInboundLedgers().cacheSize(); + + getInboundLedgers().sweep(); + + JLOG(m_journal.debug()) + << "InboundLedgers sweep. Size before: " + << oldInboundLedgersSize + << "; size after: " << getInboundLedgers().cacheSize(); + } + { + size_t const oldTasksSize = getLedgerReplayer().tasksSize(); + size_t const oldDeltasSize = getLedgerReplayer().deltasSize(); + size_t const oldSkipListsSize = getLedgerReplayer().skipListsSize(); + + getLedgerReplayer().sweep(); + + JLOG(m_journal.debug()) + << "LedgerReplayer tasks sweep. Size before: " << oldTasksSize + << "; size after: " << getLedgerReplayer().tasksSize(); + + JLOG(m_journal.debug()) + << "LedgerReplayer deltas sweep. Size before: " + << oldDeltasSize + << "; size after: " << getLedgerReplayer().deltasSize(); + + JLOG(m_journal.debug()) + << "LedgerReplayer skipLists sweep. Size before: " + << oldSkipListsSize + << "; size after: " << getLedgerReplayer().skipListsSize(); + } + { + std::size_t const oldAcceptedLedgerSize = + m_acceptedLedgerCache.size(); + + m_acceptedLedgerCache.sweep(); + + JLOG(m_journal.debug()) + << "AcceptedLedgerCache sweep. Size before: " + << oldAcceptedLedgerSize + << "; size after: " << m_acceptedLedgerCache.size(); + } + { + std::size_t const oldCachedSLEsSize = cachedSLEs_.size(); + + cachedSLEs_.sweep(); + + JLOG(m_journal.debug()) + << "CachedSLEs sweep. Size before: " << oldCachedSLEsSize + << "; size after: " << cachedSLEs_.size(); + } #ifdef RIPPLED_REPORTING if (auto pg = dynamic_cast(&*mRelationalDatabase)) @@ -1193,7 +1352,7 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) return false; } - if (validatorKeys_.publicKey.size()) + if (validatorKeys_.keys) setMaxDisallowedLedger(); // Configure the amendments the server supports @@ -1314,9 +1473,19 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) publisherManifests_->load(getWalletDB(), "PublisherManifests"); + // It is possible to have a valid ValidatorKeys object without + // setting the signingKey or masterKey. This occurs if the + // configuration file does not have either + // SECTION_VALIDATOR_TOKEN or SECTION_VALIDATION_SEED section. + + // masterKey for the configuration-file specified validator keys + std::optional localSigningKey; + if (validatorKeys_.keys) + localSigningKey = validatorKeys_.keys->publicKey; + // Setup trusted validators if (!validators_->load( - validatorKeys_.publicKey, + localSigningKey, config().section(SECTION_VALIDATORS).values(), config().section(SECTION_VALIDATOR_LIST_KEYS).values())) { @@ -1533,7 +1702,6 @@ ApplicationImp::start(bool withTimers) { setSweepTimer(); setEntropyTimer(); - m_networkOPs->setBatchApplyTimer(); } m_io_latency_sampler.start(); diff --git a/src/ripple/app/main/Application.h b/src/ripple/app/main/Application.h index 67343852b66..3fa8d13e870 100644 --- a/src/ripple/app/main/Application.h +++ b/src/ripple/app/main/Application.h @@ -242,7 +242,7 @@ class Application : public beast::PropertyStream::Source virtual std::pair const& nodeIdentity() = 0; - virtual PublicKey const& + virtual std::optional getValidationPublicKey() const = 0; virtual Resource::Manager& diff --git a/src/ripple/app/main/GRPCServer.cpp b/src/ripple/app/main/GRPCServer.cpp index 3a5e96b0ed9..a535a4a1a53 100644 --- a/src/ripple/app/main/GRPCServer.cpp +++ b/src/ripple/app/main/GRPCServer.cpp @@ -23,6 +23,7 @@ #include #include +#include namespace ripple { @@ -427,9 +428,9 @@ GRPCServerImpl::GRPCServerImpl(Application& app) : app_(app), journal_(app_.journal("gRPC Server")) { // if present, get endpoint from config - if (app_.config().exists("port_grpc")) + if (app_.config().exists(SECTION_PORT_GRPC)) { - const auto& section = app_.config().section("port_grpc"); + Section const& section = app_.config().section(SECTION_PORT_GRPC); auto const optIp = section.get("ip"); if (!optIp) @@ -659,7 +660,7 @@ GRPCServerImpl::setupListeners() secureGatewayIPs_)); } return requests; -}; +} bool GRPCServerImpl::start() diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 2ddb771b1e1..c806bfb7da5 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -431,9 +431,8 @@ run(int argc, char** argv) po::value()->implicit_value(""), "Perform unit tests. The optional argument specifies one or " "more comma-separated selectors. Each selector specifies a suite name, " - "full-name (lib.module.suite), module, or library " - "(checked in that " - "order).")( + "suite name prefix, full-name (lib.module.suite), module, or library " + "(checked in that order).")( "unittest-arg", po::value()->implicit_value(""), "Supplies an argument string to unit tests. If provided, this argument " diff --git a/src/ripple/app/misc/AMMHelpers.h b/src/ripple/app/misc/AMMHelpers.h index 24c25922800..e84c6c535ea 100644 --- a/src/ripple/app/misc/AMMHelpers.h +++ b/src/ripple/app/misc/AMMHelpers.h @@ -134,7 +134,7 @@ withinRelativeDistance( template requires( std::is_same_v || std::is_same_v || - std::is_same_v) + std::is_same_v || std::is_same_v) bool withinRelativeDistance(Amt const& calc, Amt const& req, Number const& dist) { diff --git a/src/ripple/app/misc/AmendmentTable.h b/src/ripple/app/misc/AmendmentTable.h index d7f7c8b26bb..fef13d50ddb 100644 --- a/src/ripple/app/misc/AmendmentTable.h +++ b/src/ripple/app/misc/AmendmentTable.h @@ -81,11 +81,11 @@ class AmendmentTable firstUnsupportedExpected() const = 0; virtual Json::Value - getJson() const = 0; + getJson(bool isAdmin) const = 0; /** Returns a Json::objectValue. */ virtual Json::Value - getJson(uint256 const& amendment) const = 0; + getJson(uint256 const& amendment, bool isAdmin) const = 0; /** Called when a new fully-validated ledger is accepted. */ void diff --git a/src/ripple/app/misc/Manifest.h b/src/ripple/app/misc/Manifest.h index 88452e5e0cc..b1cb5d2f325 100644 --- a/src/ripple/app/misc/Manifest.h +++ b/src/ripple/app/misc/Manifest.h @@ -86,7 +86,10 @@ struct Manifest PublicKey masterKey; /// The ephemeral key associated with this manifest. - PublicKey signingKey; + // A revoked manifest does not have a signingKey + // This field is specified as "optional" in manifestFormat's + // SOTemplate + std::optional signingKey; /// The sequence number of this manifest. std::uint32_t sequence = 0; @@ -94,7 +97,22 @@ struct Manifest /// The domain, if one was specified in the manifest; empty otherwise. std::string domain; - Manifest() = default; + Manifest() = delete; + + Manifest( + std::string const& serialized_, + PublicKey const& masterKey_, + std::optional const& signingKey_, + std::uint32_t seq, + std::string const& domain_) + : serialized(serialized_) + , masterKey(masterKey_) + , signingKey(signingKey_) + , sequence(seq) + , domain(domain_) + { + } + Manifest(Manifest const& other) = delete; Manifest& operator=(Manifest const& other) = delete; @@ -110,6 +128,12 @@ struct Manifest uint256 hash() const; + /// Returns `true` if manifest revokes master key + // The maximum possible sequence number means that the master key has + // been revoked + static bool + revoked(std::uint32_t sequence); + /// Returns `true` if manifest revokes master key bool revoked() const; @@ -266,7 +290,7 @@ class ManifestCache May be called concurrently */ - PublicKey + std::optional getSigningKey(PublicKey const& pk) const; /** Returns ephemeral signing key's master public key. diff --git a/src/ripple/app/misc/NegativeUNLVote.cpp b/src/ripple/app/misc/NegativeUNLVote.cpp index aa9db60c33d..9b616be6ce1 100644 --- a/src/ripple/app/misc/NegativeUNLVote.cpp +++ b/src/ripple/app/misc/NegativeUNLVote.cpp @@ -90,7 +90,7 @@ NegativeUNLVote::doVoting( auto n = choose(prevLedger->info().hash, candidates.toDisableCandidates); assert(nidToKeyMap.count(n)); - addTx(seq, nidToKeyMap[n], ToDisable, initialSet); + addTx(seq, nidToKeyMap.at(n), ToDisable, initialSet); } if (!candidates.toReEnableCandidates.empty()) @@ -98,7 +98,7 @@ NegativeUNLVote::doVoting( auto n = choose( prevLedger->info().hash, candidates.toReEnableCandidates); assert(nidToKeyMap.count(n)); - addTx(seq, nidToKeyMap[n], ToReEnable, initialSet); + addTx(seq, nidToKeyMap.at(n), ToReEnable, initialSet); } } } diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index 7d53384c755..0f4c339da65 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -53,15 +52,15 @@ #include #include #include -#include #include -#include #include #include #include #include #include #include +#include +#include #include #include #include @@ -69,7 +68,6 @@ #include #include #include -#include #include #include @@ -238,7 +236,6 @@ class NetworkOPsImp final : public NetworkOPs , heartbeatTimer_(io_svc) , clusterTimer_(io_svc) , accountHistoryTxTimer_(io_svc) - , batchApplyTimer_(io_svc) , mConsensus( app, make_FeeVote( @@ -288,12 +285,43 @@ class NetworkOPsImp final : public NetworkOPs processTransaction( std::shared_ptr& transaction, bool bUnlimited, - RPC::SubmitSync sync, bool bLocal, FailHard failType) override; - bool - transactionBatch(bool drain) override; + /** + * For transactions submitted directly by a client, apply batch of + * transactions and wait for this transaction to complete. + * + * @param transaction Transaction object. + * @param bUnliimited Whether a privileged client connection submitted it. + * @param failType fail_hard setting from transaction submission. + */ + void + doTransactionSync( + std::shared_ptr transaction, + bool bUnlimited, + FailHard failType); + + /** + * For transactions not submitted by a locally connected client, fire and + * forget. Add to batch and trigger it to be processed if there's no batch + * currently being applied. + * + * @param transaction Transaction object + * @param bUnlimited Whether a privileged client connection submitted it. + * @param failType fail_hard setting from transaction submission. + */ + void + doTransactionAsync( + std::shared_ptr transaction, + bool bUnlimited, + FailHard failtype); + + /** + * Apply transactions in batches. Continue until none are queued. + */ + void + transactionBatch(); /** * Attempt to apply transactions and post-process based on the results. @@ -569,15 +597,6 @@ class NetworkOPsImp final : public NetworkOPs << "NetworkOPs: accountHistoryTxTimer cancel error: " << ec.message(); } - - ec.clear(); - batchApplyTimer_.cancel(ec); - if (ec) - { - JLOG(m_journal.error()) - << "NetworkOPs: batchApplyTimer cancel error: " - << ec.message(); - } } // Make sure that any waitHandlers pending in our timers are done. using namespace std::chrono_literals; @@ -699,9 +718,6 @@ class NetworkOPsImp final : public NetworkOPs void setAccountHistoryJobTimer(SubAccountHistoryInfoWeak subInfo); - void - setBatchApplyTimer() override; - Application& app_; beast::Journal m_journal; @@ -721,8 +737,6 @@ class NetworkOPsImp final : public NetworkOPs boost::asio::steady_timer heartbeatTimer_; boost::asio::steady_timer clusterTimer_; boost::asio::steady_timer accountHistoryTxTimer_; - //! This timer is for applying transaction batches. - boost::asio::steady_timer batchApplyTimer_; RCLConsensus mConsensus; @@ -956,24 +970,9 @@ NetworkOPsImp::setTimer( void NetworkOPsImp::setHeartbeatTimer() { - // timerDelay is to optimize the timer interval such as for phase establish. - // Setting a max of ledgerGRANULARITY allows currently in-flight proposals - // to be accounted for at the very beginning of the phase. - std::chrono::milliseconds timerDelay; - auto td = mConsensus.getTimerDelay(); - if (td) - { - timerDelay = std::min(*td, mConsensus.parms().ledgerGRANULARITY); - mConsensus.setTimerDelay(); - } - else - { - timerDelay = mConsensus.parms().ledgerGRANULARITY; - } - setTimer( heartbeatTimer_, - timerDelay, + mConsensus.parms().ledgerGRANULARITY, [this]() { m_job_queue.addJob(jtNETOP_TIMER, "NetOPs.heartbeat", [this]() { processHeartbeatTimer(); @@ -1011,42 +1010,6 @@ NetworkOPsImp::setAccountHistoryJobTimer(SubAccountHistoryInfoWeak subInfo) [this, subInfo]() { setAccountHistoryJobTimer(subInfo); }); } -void -NetworkOPsImp::setBatchApplyTimer() -{ - using namespace std::chrono_literals; - // 100ms lag between batch intervals provides significant throughput gains - // with little increased latency. Tuning this figure further will - // require further testing. In general, increasing this figure will - // also increase theoretical throughput, but with diminishing returns. - auto constexpr batchInterval = 100ms; - - setTimer( - batchApplyTimer_, - batchInterval, - [this]() { - { - std::lock_guard lock(mMutex); - // Only do the job if there's work to do and it's not currently - // being done. - if (mTransactions.size() && - mDispatchState == DispatchState::none) - { - if (m_job_queue.addJob( - jtBATCH, "transactionBatch", [this]() { - transactionBatch(false); - })) - { - mDispatchState = DispatchState::scheduled; - } - return; - } - } - setBatchApplyTimer(); - }, - [this]() { setBatchApplyTimer(); }); -} - void NetworkOPsImp::processHeartbeatTimer() { @@ -1223,8 +1186,7 @@ NetworkOPsImp::submitTransaction(std::shared_ptr const& iTrans) m_job_queue.addJob(jtTRANSACTION, "submitTxn", [this, tx]() { auto t = tx; - processTransaction( - t, false, RPC::SubmitSync::async, false, FailHard::no); + processTransaction(t, false, false, FailHard::no); }); } @@ -1232,7 +1194,6 @@ void NetworkOPsImp::processTransaction( std::shared_ptr& transaction, bool bUnlimited, - RPC::SubmitSync sync, bool bLocal, FailHard failType) { @@ -1262,7 +1223,7 @@ NetworkOPsImp::processTransaction( // Not concerned with local checks at this point. if (validity == Validity::SigBad) { - JLOG(m_journal.trace()) << "Transaction has bad signature: " << reason; + JLOG(m_journal.info()) << "Transaction has bad signature: " << reason; transaction->setStatus(INVALID); transaction->setResult(temBAD_SIGNATURE); app_.getHashRouter().setFlags(transaction->getID(), SF_BAD); @@ -1272,72 +1233,100 @@ NetworkOPsImp::processTransaction( // canonicalize can change our pointer app_.getMasterTransaction().canonicalize(&transaction); - std::unique_lock lock(mMutex); + if (bLocal) + doTransactionSync(transaction, bUnlimited, failType); + else + doTransactionAsync(transaction, bUnlimited, failType); +} + +void +NetworkOPsImp::doTransactionAsync( + std::shared_ptr transaction, + bool bUnlimited, + FailHard failType) +{ + std::lock_guard lock(mMutex); + + if (transaction->getApplying()) + return; + + mTransactions.push_back( + TransactionStatus(transaction, bUnlimited, false, failType)); + transaction->setApplying(); + + if (mDispatchState == DispatchState::none) + { + if (m_job_queue.addJob( + jtBATCH, "transactionBatch", [this]() { transactionBatch(); })) + { + mDispatchState = DispatchState::scheduled; + } + } +} + +void +NetworkOPsImp::doTransactionSync( + std::shared_ptr transaction, + bool bUnlimited, + FailHard failType) +{ + std::unique_lock lock(mMutex); + if (!transaction->getApplying()) { - transaction->setApplying(); mTransactions.push_back( - TransactionStatus(transaction, bUnlimited, bLocal, failType)); + TransactionStatus(transaction, bUnlimited, true, failType)); + transaction->setApplying(); } - switch (sync) + + do { - using enum RPC::SubmitSync; - case sync: - do + if (mDispatchState == DispatchState::running) + { + // A batch processing job is already running, so wait. + mCond.wait(lock); + } + else + { + apply(lock); + + if (mTransactions.size()) { - // If a batch is being processed, then wait. Otherwise, - // process a batch. - if (mDispatchState == DispatchState::running) - mCond.wait(lock); - else - apply(lock); - } while (transaction->getApplying()); - break; - - case async: - // It's conceivable for the submitted transaction to be - // processed and its result to be modified before being returned - // to the client. Make a copy of the transaction and set its - // status to guarantee that the client gets the terSUBMITTED - // result in all cases. - transaction = std::make_shared(*transaction); - transaction->setResult(terSUBMITTED); - break; - - case wait: - mCond.wait( - lock, [&transaction] { return !transaction->getApplying(); }); - break; - - default: - assert(false); - } + // More transactions need to be applied, but by another job. + if (m_job_queue.addJob(jtBATCH, "transactionBatch", [this]() { + transactionBatch(); + })) + { + mDispatchState = DispatchState::scheduled; + } + } + } + } while (transaction->getApplying()); } -bool -NetworkOPsImp::transactionBatch(bool const drain) +void +NetworkOPsImp::transactionBatch() { - { - std::unique_lock lock(mMutex); - if (mDispatchState == DispatchState::running || mTransactions.empty()) - return false; + std::unique_lock lock(mMutex); - do - apply(lock); - while (drain && mTransactions.size()); + if (mDispatchState == DispatchState::running) + return; + + while (mTransactions.size()) + { + apply(lock); } - setBatchApplyTimer(); - return true; } void NetworkOPsImp::apply(std::unique_lock& batchLock) { - assert(!mTransactions.empty()); - assert(mDispatchState != DispatchState::running); std::vector submit_held; std::vector transactions; mTransactions.swap(transactions); + assert(!transactions.empty()); + + assert(mDispatchState != DispatchState::running); mDispatchState = DispatchState::running; batchLock.unlock(); @@ -1721,9 +1710,7 @@ NetworkOPsImp::checkLastClosedLedger( switchLedgers = false; } else - { networkClosed = closedLedger; - } if (!switchLedgers) return false; @@ -1997,9 +1984,9 @@ NetworkOPsImp::pubManifest(Manifest const& mo) jvObj[jss::type] = "manifestReceived"; jvObj[jss::master_key] = toBase58(TokenType::NodePublic, mo.masterKey); - if (!mo.signingKey.empty()) + if (mo.signingKey) jvObj[jss::signing_key] = - toBase58(TokenType::NodePublic, mo.signingKey); + toBase58(TokenType::NodePublic, *mo.signingKey); jvObj[jss::seq] = Json::UInt(mo.sequence); if (auto sig = mo.getSignature()) jvObj[jss::signature] = strHex(*sig); @@ -2196,8 +2183,10 @@ NetworkOPsImp::pubValidation(std::shared_ptr const& val) if (masterKey != signerPublic) jvObj[jss::master_key] = toBase58(TokenType::NodePublic, masterKey); + // NOTE *seq is a number, but old API versions used string. We replace + // number with a string using MultiApiJson near end of this function if (auto const seq = (*val)[~sfLedgerSequence]) - jvObj[jss::ledger_index] = to_string(*seq); + jvObj[jss::ledger_index] = *seq; if (val->isFieldPresent(sfAmendments)) { @@ -2235,12 +2224,28 @@ NetworkOPsImp::pubValidation(std::shared_ptr const& val) reserveIncXRP && reserveIncXRP->native()) jvObj[jss::reserve_inc] = reserveIncXRP->xrp().jsonClipped(); + // NOTE Use MultiApiJson to publish two slightly different JSON objects + // for consumers supporting different API versions + MultiApiJson multiObj{jvObj}; + multiObj.visit( + RPC::apiVersion<1>, // + [](Json::Value& jvTx) { + // Type conversion for older API versions to string + if (jvTx.isMember(jss::ledger_index)) + { + jvTx[jss::ledger_index] = + std::to_string(jvTx[jss::ledger_index].asUInt()); + } + }); + for (auto i = mStreamMaps[sValidations].begin(); i != mStreamMaps[sValidations].end();) { if (auto p = i->second.lock()) { - p->send(jvObj, true); + multiObj.visit( + p->getApiVersion(), // + [&](Json::Value const& jv) { p->send(jv, true); }); ++i; } else @@ -2465,10 +2470,11 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters) if (admin) { - if (!app_.getValidationPublicKey().empty()) + if (auto const localPubKey = app_.validators().localPublicKey(); + localPubKey && app_.getValidationPublicKey()) { - info[jss::pubkey_validator] = toBase58( - TokenType::NodePublic, app_.validators().localPublicKey()); + info[jss::pubkey_validator] = + toBase58(TokenType::NodePublic, localPubKey.value()); } else { @@ -2728,9 +2734,9 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters) } } - if (app_.config().exists("port_grpc")) + if (app_.config().exists(SECTION_PORT_GRPC)) { - auto const& grpcSection = app_.config().section("port_grpc"); + auto const& grpcSection = app_.config().section(SECTION_PORT_GRPC); auto const optPort = grpcSection.get("port"); if (optPort && grpcSection.get("ip")) { @@ -2783,8 +2789,9 @@ NetworkOPsImp::pubProposedTransaction( if (p) { - p->send( - jvObj.select(apiVersionSelector(p->getApiVersion())), true); + jvObj.visit( + p->getApiVersion(), // + [&](Json::Value const& jv) { p->send(jv, true); }); ++it; } else @@ -3180,29 +3187,15 @@ NetworkOPsImp::transJson( } std::string const hash = to_string(transaction->getTransactionID()); - MultiApiJson multiObj({jvObj, jvObj}); - // Minimum supported API version must match index 0 in MultiApiJson - static_assert(apiVersionSelector(RPC::apiMinimumSupportedVersion)() == 0); - // Last valid (possibly beta) API ver. must match last index in MultiApiJson - static_assert( - apiVersionSelector(RPC::apiMaximumValidVersion)() + 1 // - == MultiApiJson::size); - for (unsigned apiVersion = RPC::apiMinimumSupportedVersion, - lastIndex = MultiApiJson::size; - apiVersion <= RPC::apiMaximumValidVersion; - ++apiVersion) - { - unsigned const index = apiVersionSelector(apiVersion)(); - assert(index < MultiApiJson::size); - if (index != lastIndex) - { - lastIndex = index; - - Json::Value& jvTx = multiObj.val[index]; + MultiApiJson multiObj{jvObj}; + forAllApiVersions( + multiObj.visit(), // + [&]( + Json::Value& jvTx, std::integral_constant) { RPC::insertDeliverMax( - jvTx[jss::transaction], transaction->getTxnType(), apiVersion); + jvTx[jss::transaction], transaction->getTxnType(), Version); - if (apiVersion > 1) + if constexpr (Version > 1) { jvTx[jss::tx_json] = jvTx.removeMember(jss::transaction); jvTx[jss::hash] = hash; @@ -3211,8 +3204,7 @@ NetworkOPsImp::transJson( { jvTx[jss::transaction][jss::hash] = hash; } - } - } + }); return multiObj; } @@ -3240,8 +3232,9 @@ NetworkOPsImp::pubValidatedTransaction( if (p) { - p->send( - jvObj.select(apiVersionSelector(p->getApiVersion())), true); + jvObj.visit( + p->getApiVersion(), // + [&](Json::Value const& jv) { p->send(jv, true); }); ++it; } else @@ -3256,8 +3249,9 @@ NetworkOPsImp::pubValidatedTransaction( if (p) { - p->send( - jvObj.select(apiVersionSelector(p->getApiVersion())), true); + jvObj.visit( + p->getApiVersion(), // + [&](Json::Value const& jv) { p->send(jv, true); }); ++it; } else @@ -3377,9 +3371,9 @@ NetworkOPsImp::pubAccountTransaction( for (InfoSub::ref isrListener : notify) { - isrListener->send( - jvObj.select(apiVersionSelector(isrListener->getApiVersion())), - true); + jvObj.visit( + isrListener->getApiVersion(), // + [&](Json::Value const& jv) { isrListener->send(jv, true); }); } if (last) @@ -3396,9 +3390,9 @@ NetworkOPsImp::pubAccountTransaction( jvObj.set(jss::account_history_tx_index, index->forwardTxIndex_++); - info.sink_->send( - jvObj.select(apiVersionSelector(info.sink_->getApiVersion())), - true); + jvObj.visit( + info.sink_->getApiVersion(), // + [&](Json::Value const& jv) { info.sink_->send(jv, true); }); } } } @@ -3456,9 +3450,9 @@ NetworkOPsImp::pubProposedAccountTransaction( MultiApiJson jvObj = transJson(tx, result, false, ledger, std::nullopt); for (InfoSub::ref isrListener : notify) - isrListener->send( - jvObj.select(apiVersionSelector(isrListener->getApiVersion())), - true); + jvObj.visit( + isrListener->getApiVersion(), // + [&](Json::Value const& jv) { isrListener->send(jv, true); }); assert( jvObj.isMember(jss::account_history_tx_stream) == @@ -3469,9 +3463,9 @@ NetworkOPsImp::pubProposedAccountTransaction( if (index->forwardTxIndex_ == 0 && !index->haveHistorical_) jvObj.set(jss::account_history_tx_first, true); jvObj.set(jss::account_history_tx_index, index->forwardTxIndex_++); - info.sink_->send( - jvObj.select(apiVersionSelector(info.sink_->getApiVersion())), - true); + jvObj.visit( + info.sink_->getApiVersion(), // + [&](Json::Value const& jv) { info.sink_->send(jv, true); }); } } } @@ -3677,9 +3671,9 @@ NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo) bool unsubscribe) -> bool { if (auto sptr = subInfo.sinkWptr_.lock()) { - sptr->send( - jvObj.select(apiVersionSelector(sptr->getApiVersion())), - true); + jvObj.visit( + sptr->getApiVersion(), // + [&](Json::Value const& jv) { sptr->send(jv, true); }); if (unsubscribe) unsubAccountHistory(sptr, accountId, false); diff --git a/src/ripple/app/misc/NetworkOPs.h b/src/ripple/app/misc/NetworkOPs.h index d388ab310a0..344b9e26086 100644 --- a/src/ripple/app/misc/NetworkOPs.h +++ b/src/ripple/app/misc/NetworkOPs.h @@ -71,10 +71,6 @@ enum class OperatingMode { FULL = 4 //!< we have the ledger and can even validate }; -namespace RPC { -enum class SubmitSync; -} - /** Provides server functionality for clients. Clients include backend applications, local commands, and connected @@ -127,47 +123,22 @@ class NetworkOPs : public InfoSub::Source virtual void submitTransaction(std::shared_ptr const&) = 0; - /** Process a transaction. - * - * The transaction has been submitted either from the peer network or - * from a client. For client submissions, there are 3 distinct behaviors: - * 1) sync (default): process transactions in a batch immediately, - * and return only once the transaction has been processed. - * 2) async: Put transaction into the batch for the next processing - * interval and return immediately. - * 3) wait: Put transaction into the batch for the next processing - * interval and return only after it is processed. + /** + * Process transactions as they arrive from the network or which are + * submitted by clients. Process local transactions synchronously * - * @param transaction Transaction object. + * @param transaction Transaction object * @param bUnlimited Whether a privileged client connection submitted it. - * @param sync Client submission synchronous behavior type requested. - * @param bLocal Whether submitted by client (local) or peer. - * @param failType Whether to fail hard or not. + * @param bLocal Client submission. + * @param failType fail_hard setting from transaction submission. */ virtual void processTransaction( std::shared_ptr& transaction, bool bUnlimited, - RPC::SubmitSync sync, bool bLocal, FailHard failType) = 0; - /** Apply transactions in batches. - * - * Only a single batch unless drain is set. This is to optimize performance - * because there is significant overhead in applying each batch, whereas - * processing an individual transaction is fast. - * - * Setting the drain parameter is relevant for some transaction - * processing unit tests that expect all submitted transactions to - * be processed synchronously. - * - * @param drain Whether to process batches until none remain. - * @return Whether any transactions were processed. - */ - virtual bool - transactionBatch(bool drain) = 0; - //-------------------------------------------------------------------------- // // Owner functions @@ -216,8 +187,6 @@ class NetworkOPs : public InfoSub::Source setStandAlone() = 0; virtual void setStateTimer() = 0; - virtual void - setBatchApplyTimer() = 0; virtual void setNeedNetworkLedger() = 0; diff --git a/src/ripple/app/misc/ValidatorKeys.h b/src/ripple/app/misc/ValidatorKeys.h index c58c95f8cc7..a6b53841739 100644 --- a/src/ripple/app/misc/ValidatorKeys.h +++ b/src/ripple/app/misc/ValidatorKeys.h @@ -36,13 +36,35 @@ class Config; class ValidatorKeys { public: - PublicKey masterPublicKey; - PublicKey publicKey; - SecretKey secretKey; + // Group all keys in a struct. Either all keys are valid or none are. + struct Keys + { + PublicKey masterPublicKey; + PublicKey publicKey; + SecretKey secretKey; + + Keys() = delete; + Keys( + PublicKey const& masterPublic_, + PublicKey const& public_, + SecretKey const& secret_) + : masterPublicKey(masterPublic_) + , publicKey(public_) + , secretKey(secret_) + { + } + }; + + // Note: The existence of keys cannot be used as a proxy for checking the + // validity of a configuration. It is possible to have a valid + // configuration while not setting the keys, as per the constructor of + // the ValidatorKeys class. + std::optional keys; NodeID nodeID; std::string manifest; std::uint32_t sequence = 0; + ValidatorKeys() = delete; ValidatorKeys(Config const& config, beast::Journal j); bool diff --git a/src/ripple/app/misc/ValidatorList.h b/src/ripple/app/misc/ValidatorList.h index 0d7605fc09d..280818abd35 100644 --- a/src/ripple/app/misc/ValidatorList.h +++ b/src/ripple/app/misc/ValidatorList.h @@ -247,7 +247,17 @@ class ValidatorList // a seed, the signing key is the same as the master key. hash_set trustedSigningKeys_; - PublicKey localPubKey_; + std::optional localPubKey_; + + // The below variable contains the Publisher list specified in the local + // config file under the title of SECTION_VALIDATORS or [validators]. + // This list is not associated with the masterKey of any publisher. + + // Appropos PublisherListCollection fields, localPublisherList does not + // have any "remaining" manifests. It is assumed to be perennially + // "available". The "validUntil" field is set to the highest possible + // value of the field, hence this list is always valid. + PublisherList localPublisherList; // The master public keys of the current negative UNL hash_set negativeUNL_; @@ -331,7 +341,7 @@ class ValidatorList */ bool load( - PublicKey const& localSigningKey, + std::optional const& localSigningKey, std::vector const& configKeys, std::vector const& publisherKeys); @@ -553,13 +563,14 @@ class ValidatorList bool trustedPublisher(PublicKey const& identity) const; - /** Returns local validator public key + /** This function returns the local validator public key + * or a std::nullopt @par Thread Safety May be called concurrently */ - PublicKey + std::optional localPublicKey() const; /** Invokes the callback once for every listed validation public key. @@ -766,6 +777,8 @@ class ValidatorList std::optional const& hash, lock_guard const&); + // This function updates the keyListings_ counts for all the trusted + // master keys void updatePublisherList( PublicKey const& pubKey, @@ -849,11 +862,10 @@ class ValidatorList Calling public member function is expected to lock mutex */ - ListDisposition + std::pair> verify( lock_guard const&, Json::Value& list, - PublicKey& pubKey, std::string const& manifest, std::string const& blob, std::string const& signature); diff --git a/src/ripple/app/misc/detail/Work.h b/src/ripple/app/misc/detail/Work.h index 15be569566e..afd49adf326 100644 --- a/src/ripple/app/misc/detail/Work.h +++ b/src/ripple/app/misc/detail/Work.h @@ -20,6 +20,10 @@ #ifndef RIPPLE_APP_MISC_DETAIL_WORK_H_INCLUDED #define RIPPLE_APP_MISC_DETAIL_WORK_H_INCLUDED +// TODO: This include is a workaround for beast compilation bug. +// Remove when fix https://github.com/boostorg/beast/pull/2682/ is available. +#include + #include #include diff --git a/src/ripple/app/misc/impl/AMMUtils.cpp b/src/ripple/app/misc/impl/AMMUtils.cpp index 1d787dbe4ca..d766bc508b6 100644 --- a/src/ripple/app/misc/impl/AMMUtils.cpp +++ b/src/ripple/app/misc/impl/AMMUtils.cpp @@ -138,6 +138,9 @@ std::uint16_t getTradingFee(ReadView const& view, SLE const& ammSle, AccountID const& account) { using namespace std::chrono; + assert( + !view.rules().enabled(fixInnerObjTemplate) || + ammSle.isFieldPresent(sfAuctionSlot)); if (ammSle.isFieldPresent(sfAuctionSlot)) { auto const& auctionSlot = @@ -287,9 +290,10 @@ initializeFeeAuctionVote( Issue const& lptIssue, std::uint16_t tfee) { + auto const& rules = view.rules(); // AMM creator gets the voting slot. STArray voteSlots; - STObject voteEntry{sfVoteEntry}; + STObject voteEntry = STObject::makeInnerObject(sfVoteEntry, rules); if (tfee != 0) voteEntry.setFieldU16(sfTradingFee, tfee); voteEntry.setFieldU32(sfVoteWeight, VOTE_WEIGHT_SCALE_FACTOR); @@ -297,7 +301,15 @@ initializeFeeAuctionVote( voteSlots.push_back(voteEntry); ammSle->setFieldArray(sfVoteSlots, voteSlots); // AMM creator gets the auction slot for free. - auto& auctionSlot = ammSle->peekFieldObject(sfAuctionSlot); + // AuctionSlot is created on AMMCreate and updated on AMMDeposit + // when AMM is in an empty state + if (rules.enabled(fixInnerObjTemplate) && + !ammSle->isFieldPresent(sfAuctionSlot)) + { + STObject auctionSlot = STObject::makeInnerObject(sfAuctionSlot, rules); + ammSle->set(std::move(auctionSlot)); + } + STObject& auctionSlot = ammSle->peekFieldObject(sfAuctionSlot); auctionSlot.setAccountID(sfAccount, account); // current + sec in 24h auto const expiration = std::chrono::duration_cast( diff --git a/src/ripple/app/misc/impl/AmendmentTable.cpp b/src/ripple/app/misc/impl/AmendmentTable.cpp index 8f4f0321992..8f9556e0714 100644 --- a/src/ripple/app/misc/impl/AmendmentTable.cpp +++ b/src/ripple/app/misc/impl/AmendmentTable.cpp @@ -388,6 +388,7 @@ class AmendmentTableImpl final : public AmendmentTable Json::Value& v, uint256 const& amendment, AmendmentState const& state, + bool isAdmin, std::lock_guard const& lock) const; void @@ -428,9 +429,9 @@ class AmendmentTableImpl final : public AmendmentTable firstUnsupportedExpected() const override; Json::Value - getJson() const override; + getJson(bool isAdmin) const override; Json::Value - getJson(uint256 const&) const override; + getJson(uint256 const&, bool isAdmin) const override; bool needValidatedLedger(LedgerIndex seq) const override; @@ -906,13 +907,14 @@ AmendmentTableImpl::injectJson( Json::Value& v, const uint256& id, const AmendmentState& fs, + bool isAdmin, std::lock_guard const&) const { if (!fs.name.empty()) v[jss::name] = fs.name; v[jss::supported] = fs.supported; - if (!fs.enabled) + if (!fs.enabled && isAdmin) { if (fs.vote == AmendmentVote::obsolete) v[jss::vetoed] = "Obsolete"; @@ -921,7 +923,7 @@ AmendmentTableImpl::injectJson( } v[jss::enabled] = fs.enabled; - if (!fs.enabled && lastVote_) + if (!fs.enabled && lastVote_ && isAdmin) { auto const votesTotal = lastVote_->trustedValidations(); auto const votesNeeded = lastVote_->threshold(); @@ -936,7 +938,7 @@ AmendmentTableImpl::injectJson( } Json::Value -AmendmentTableImpl::getJson() const +AmendmentTableImpl::getJson(bool isAdmin) const { Json::Value ret(Json::objectValue); { @@ -947,6 +949,7 @@ AmendmentTableImpl::getJson() const ret[to_string(e.first)] = Json::objectValue, e.first, e.second, + isAdmin, lock); } } @@ -954,16 +957,19 @@ AmendmentTableImpl::getJson() const } Json::Value -AmendmentTableImpl::getJson(uint256 const& amendmentID) const +AmendmentTableImpl::getJson(uint256 const& amendmentID, bool isAdmin) const { Json::Value ret = Json::objectValue; - Json::Value& jAmendment = (ret[to_string(amendmentID)] = Json::objectValue); { std::lock_guard lock(mutex_); AmendmentState const* a = get(amendmentID, lock); if (a) - injectJson(jAmendment, amendmentID, *a, lock); + { + Json::Value& jAmendment = + (ret[to_string(amendmentID)] = Json::objectValue); + injectJson(jAmendment, amendmentID, *a, isAdmin, lock); + } } return ret; diff --git a/src/ripple/app/misc/impl/Manifest.cpp b/src/ripple/app/misc/impl/Manifest.cpp index 60b52133053..2916d6d2f32 100644 --- a/src/ripple/app/misc/impl/Manifest.cpp +++ b/src/ripple/app/misc/impl/Manifest.cpp @@ -22,8 +22,6 @@ #include #include #include -#include -#include #include #include #include @@ -32,7 +30,6 @@ #include #include -#include #include namespace ripple { @@ -45,8 +42,11 @@ to_string(Manifest const& m) if (m.revoked()) return "Revocation Manifest " + mk; + if (!m.signingKey) + Throw("No SigningKey in manifest " + mk); + return "Manifest " + mk + " (" + std::to_string(m.sequence) + ": " + - toBase58(TokenType::NodePublic, m.signingKey) + ")"; + toBase58(TokenType::NodePublic, *m.signingKey) + ")"; } std::optional @@ -96,25 +96,27 @@ deserializeManifest(Slice s, beast::Journal journal) if (!publicKeyType(makeSlice(pk))) return std::nullopt; - Manifest m; - m.serialized.assign(reinterpret_cast(s.data()), s.size()); - m.masterKey = PublicKey(makeSlice(pk)); - m.sequence = st.getFieldU32(sfSequence); + PublicKey const masterKey = PublicKey(makeSlice(pk)); + std::uint32_t const seq = st.getFieldU32(sfSequence); + + std::string domain; + + std::optional signingKey; if (st.isFieldPresent(sfDomain)) { auto const d = st.getFieldVL(sfDomain); - m.domain.assign(reinterpret_cast(d.data()), d.size()); + domain.assign(reinterpret_cast(d.data()), d.size()); - if (!isProperlyFormedTomlDomain(m.domain)) + if (!isProperlyFormedTomlDomain(domain)) return std::nullopt; } bool const hasEphemeralKey = st.isFieldPresent(sfSigningPubKey); bool const hasEphemeralSig = st.isFieldPresent(sfSignature); - if (m.revoked()) + if (Manifest::revoked(seq)) { // Revocation manifests should not specify a new signing key // or a signing key signature. @@ -139,14 +141,18 @@ deserializeManifest(Slice s, beast::Journal journal) if (!publicKeyType(makeSlice(spk))) return std::nullopt; - m.signingKey = PublicKey(makeSlice(spk)); + signingKey.emplace(makeSlice(spk)); // The signing and master keys can't be the same - if (m.signingKey == m.masterKey) + if (*signingKey == masterKey) return std::nullopt; } - return m; + std::string const serialized( + reinterpret_cast(s.data()), s.size()); + + // If the manifest is revoked, then the signingKey will be unseated + return Manifest(serialized, masterKey, signingKey, seq, domain); } catch (std::exception const& ex) { @@ -192,9 +198,14 @@ Manifest::verify() const SerialIter sit(serialized.data(), serialized.size()); st.set(sit); + // The manifest must either have a signing key or be revoked. This check + // prevents us from accessing an unseated signingKey in the next check. + if (!revoked() && !signingKey) + return false; + // Signing key and signature are not required for // master key revocations - if (!revoked() && !ripple::verify(st, HashPrefix::manifest, signingKey)) + if (!revoked() && !ripple::verify(st, HashPrefix::manifest, *signingKey)) return false; return ripple::verify( @@ -217,6 +228,14 @@ Manifest::revoked() const The maximum possible sequence number means that the master key has been revoked. */ + return revoked(sequence); +} + +bool +Manifest::revoked(std::uint32_t sequence) +{ + // The maximum possible sequence number means that the master key has + // been revoked. return sequence == std::numeric_limits::max(); } @@ -287,7 +306,7 @@ loadValidatorToken(std::vector const& blob, beast::Journal journal) } } -PublicKey +std::optional ManifestCache::getSigningKey(PublicKey const& pk) const { std::shared_lock lock{mutex_}; @@ -423,9 +442,18 @@ ManifestCache::applyManifest(Manifest m) if (!revoked) { + if (!m.signingKey) + { + JLOG(j_.warn()) << to_string(m) + << ": is not revoked and the manifest has no " + "signing key. Hence, the manifest is " + "invalid"; + return ManifestDisposition::invalid; + } + // Sanity check: the ephemeral key of this manifest should not be // used as the master or ephemeral key of another manifest: - if (auto const x = signingToMasterKeys_.find(m.signingKey); + if (auto const x = signingToMasterKeys_.find(*m.signingKey); x != signingToMasterKeys_.end()) { JLOG(j_.warn()) @@ -436,7 +464,7 @@ ManifestCache::applyManifest(Manifest m) return ManifestDisposition::badEphemeralKey; } - if (auto const x = map_.find(m.signingKey); x != map_.end()) + if (auto const x = map_.find(*m.signingKey); x != map_.end()) { JLOG(j_.warn()) << to_string(m) << ": Ephemeral key used as master key for " @@ -479,7 +507,7 @@ ManifestCache::applyManifest(Manifest m) logMftAct(stream, "AcceptedNew", m.masterKey, m.sequence); if (!revoked) - signingToMasterKeys_[m.signingKey] = m.masterKey; + signingToMasterKeys_.emplace(*m.signingKey, m.masterKey); auto masterKey = m.masterKey; map_.emplace(std::move(masterKey), std::move(m)); @@ -496,10 +524,10 @@ ManifestCache::applyManifest(Manifest m) m.sequence, iter->second.sequence); - signingToMasterKeys_.erase(iter->second.signingKey); + signingToMasterKeys_.erase(*iter->second.signingKey); if (!revoked) - signingToMasterKeys_[m.signingKey] = m.masterKey; + signingToMasterKeys_.emplace(*m.signingKey, m.masterKey); iter->second = std::move(m); diff --git a/src/ripple/app/misc/impl/ValidatorKeys.cpp b/src/ripple/app/misc/impl/ValidatorKeys.cpp index f9e55ae45a6..8da1992a2ef 100644 --- a/src/ripple/app/misc/impl/ValidatorKeys.cpp +++ b/src/ripple/app/misc/impl/ValidatorKeys.cpp @@ -56,9 +56,7 @@ ValidatorKeys::ValidatorKeys(Config const& config, beast::Journal j) } else { - secretKey = token->validationSecret; - publicKey = pk; - masterPublicKey = m->masterKey; + keys.emplace(m->masterKey, pk, token->validationSecret); nodeID = calcNodeID(m->masterKey); sequence = m->sequence; manifest = std::move(token->manifest); @@ -83,10 +81,10 @@ ValidatorKeys::ValidatorKeys(Config const& config, beast::Journal j) } else { - secretKey = generateSecretKey(KeyType::secp256k1, *seed); - publicKey = derivePublicKey(KeyType::secp256k1, secretKey); - masterPublicKey = publicKey; - nodeID = calcNodeID(publicKey); + SecretKey const sk = generateSecretKey(KeyType::secp256k1, *seed); + PublicKey const pk = derivePublicKey(KeyType::secp256k1, sk); + keys.emplace(pk, pk, sk); + nodeID = calcNodeID(pk); sequence = 0; } } diff --git a/src/ripple/app/misc/impl/ValidatorList.cpp b/src/ripple/app/misc/impl/ValidatorList.cpp index 832628dce3e..ff5fbd90eac 100644 --- a/src/ripple/app/misc/impl/ValidatorList.cpp +++ b/src/ripple/app/misc/impl/ValidatorList.cpp @@ -24,9 +24,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -135,7 +135,7 @@ ValidatorList::ValidatorList( bool ValidatorList::load( - PublicKey const& localSigningKey, + std::optional const& localSigningKey, std::vector const& configKeys, std::vector const& publisherKeys) { @@ -193,11 +193,12 @@ ValidatorList::load( JLOG(j_.debug()) << "Loaded " << count << " keys"; - localPubKey_ = validatorManifests_.getMasterKey(localSigningKey); + if (localSigningKey) + localPubKey_ = validatorManifests_.getMasterKey(*localSigningKey); // Treat local validator key as though it was listed in the config - if (localPubKey_.size()) - keyListings_.insert({localPubKey_, 1}); + if (localPubKey_) + keyListings_.insert({*localPubKey_, 1}); JLOG(j_.debug()) << "Loading configured validator keys"; @@ -232,16 +233,16 @@ ValidatorList::load( JLOG(j_.warn()) << "Duplicate node identity: " << match[1]; continue; } - auto [it, inserted] = publisherLists_.emplace(); - // Config listed keys never expire - auto& current = it->second.current; - if (inserted) - current.validUntil = TimeKeeper::time_point::max(); - current.list.emplace_back(*id); - it->second.status = PublisherStatus::available; + localPublisherList.list.emplace_back(*id); ++count; } + // Config listed keys never expire + // set the expiration time for the newly created publisher list + // exactly once + if (count > 0) + localPublisherList.validUntil = TimeKeeper::time_point::max(); + JLOG(j_.debug()) << "Loaded " << count << " entries"; return true; @@ -884,9 +885,11 @@ ValidatorList::applyListsAndBroadcast( if (disposition == ListDisposition::accepted) { bool good = true; - for (auto const& [pubKey, listCollection] : publisherLists_) + + // localPublisherList never expires, so localPublisherList is excluded + // from the below check. + for (auto const& [_, listCollection] : publisherLists_) { - (void)pubKey; if (listCollection.status != PublisherStatus::available) { good = false; @@ -900,12 +903,15 @@ ValidatorList::applyListsAndBroadcast( } bool broadcast = disposition <= ListDisposition::known_sequence; - if (broadcast) + // this function is only called for PublicKeys which are not specified + // in the config file (Note: Keys specified in the local config file are + // stored in ValidatorList::localPublisherList data member). + if (broadcast && result.status <= PublisherStatus::expired && + result.publisherKey && + publisherLists_[*result.publisherKey].maxSequence) { auto const& pubCollection = publisherLists_[*result.publisherKey]; - assert( - result.status <= PublisherStatus::expired && result.publisherKey && - pubCollection.maxSequence); + broadcastBlobs( *result.publisherKey, pubCollection, @@ -1071,9 +1077,24 @@ ValidatorList::applyList( using namespace std::string_literals; Json::Value list; - PublicKey pubKey; auto const& manifest = localManifest ? *localManifest : globalManifest; - auto const result = verify(lock, list, pubKey, manifest, blob, signature); + auto [result, pubKeyOpt] = verify(lock, list, manifest, blob, signature); + + if (!pubKeyOpt) + { + JLOG(j_.info()) << "ValidatorList::applyList unable to retrieve the " + "master public key from the verify function\n"; + return PublisherListStats{result}; + } + + if (!publicKeyType(*pubKeyOpt)) + { + JLOG(j_.info()) << "ValidatorList::applyList Invalid Public Key type" + " retrieved from the verify function\n "; + return PublisherListStats{result}; + } + + PublicKey pubKey = *pubKeyOpt; if (result > ListDisposition::pending) { if (publisherLists_.count(pubKey)) @@ -1256,11 +1277,12 @@ ValidatorList::loadLists() return sites; } -ListDisposition +// The returned PublicKey value is read from the manifest. Manifests do not +// contain the default-constructed public keys +std::pair> ValidatorList::verify( ValidatorList::lock_guard const& lock, Json::Value& list, - PublicKey& pubKey, std::string const& manifest, std::string const& blob, std::string const& signature) @@ -1268,35 +1290,33 @@ ValidatorList::verify( auto m = deserializeManifest(base64_decode(manifest)); if (!m || !publisherLists_.count(m->masterKey)) - return ListDisposition::untrusted; + return {ListDisposition::untrusted, {}}; - pubKey = m->masterKey; + PublicKey masterPubKey = m->masterKey; auto const revoked = m->revoked(); auto const result = publisherManifests_.applyManifest(std::move(*m)); if (revoked && result == ManifestDisposition::accepted) { - removePublisherList(lock, pubKey, PublisherStatus::revoked); + removePublisherList(lock, masterPubKey, PublisherStatus::revoked); // If the manifest is revoked, no future list is valid either - publisherLists_[pubKey].remaining.clear(); + publisherLists_[masterPubKey].remaining.clear(); } - if (revoked || result == ManifestDisposition::invalid) - return ListDisposition::untrusted; + auto const signingKey = publisherManifests_.getSigningKey(masterPubKey); + + if (revoked || !signingKey || result == ManifestDisposition::invalid) + return {ListDisposition::untrusted, masterPubKey}; auto const sig = strUnHex(signature); auto const data = base64_decode(blob); - if (!sig || - !ripple::verify( - publisherManifests_.getSigningKey(pubKey), - makeSlice(data), - makeSlice(*sig))) - return ListDisposition::invalid; + if (!sig || !ripple::verify(*signingKey, makeSlice(data), makeSlice(*sig))) + return {ListDisposition::invalid, masterPubKey}; Json::Reader r; if (!r.parse(data, list)) - return ListDisposition::invalid; + return {ListDisposition::invalid, masterPubKey}; if (list.isMember(jss::sequence) && list[jss::sequence].isInt() && list.isMember(jss::expiration) && list[jss::expiration].isInt() && @@ -1309,15 +1329,15 @@ ValidatorList::verify( auto const validUntil = TimeKeeper::time_point{ TimeKeeper::duration{list[jss::expiration].asUInt()}}; auto const now = timeKeeper_.now(); - auto const& listCollection = publisherLists_[pubKey]; + auto const& listCollection = publisherLists_[masterPubKey]; if (validUntil <= validFrom) - return ListDisposition::invalid; + return {ListDisposition::invalid, masterPubKey}; else if (sequence < listCollection.current.sequence) - return ListDisposition::stale; + return {ListDisposition::stale, masterPubKey}; else if (sequence == listCollection.current.sequence) - return ListDisposition::same_sequence; + return {ListDisposition::same_sequence, masterPubKey}; else if (validUntil <= now) - return ListDisposition::expired; + return {ListDisposition::expired, masterPubKey}; else if (validFrom > now) // Not yet valid. Return pending if one of the following is true // * There's no maxSequence, indicating this is the first blob seen @@ -1335,15 +1355,15 @@ ValidatorList::verify( validFrom < listCollection.remaining .at(*listCollection.maxSequence) .validFrom) - ? ListDisposition::pending - : ListDisposition::known_sequence; + ? std::make_pair(ListDisposition::pending, masterPubKey) + : std::make_pair(ListDisposition::known_sequence, masterPubKey); } else { - return ListDisposition::invalid; + return {ListDisposition::invalid, masterPubKey}; } - return ListDisposition::accepted; + return {ListDisposition::accepted, masterPubKey}; } bool @@ -1409,7 +1429,7 @@ ValidatorList::trustedPublisher(PublicKey const& identity) const publisherLists_.at(identity).status < PublisherStatus::revoked; } -PublicKey +std::optional ValidatorList::localPublicKey() const { std::shared_lock read_lock{mutex_}; @@ -1453,7 +1473,7 @@ ValidatorList::removePublisherList( std::size_t ValidatorList::count(ValidatorList::shared_lock const&) const { - return publisherLists_.size(); + return publisherLists_.size() + (localPublisherList.list.size() > 0); } std::size_t @@ -1467,13 +1487,14 @@ std::optional ValidatorList::expires(ValidatorList::shared_lock const&) const { std::optional res{}; - for (auto const& [pubKey, collection] : publisherLists_) + for (auto const& [_, collection] : publisherLists_) { - (void)pubKey; // Unfetched auto const& current = collection.current; if (current.validUntil == TimeKeeper::time_point{}) + { return std::nullopt; + } // Find the latest validUntil in a chain where the next validFrom // overlaps with the previous validUntil. applyLists has already cleaned @@ -1494,6 +1515,20 @@ ValidatorList::expires(ValidatorList::shared_lock const&) const res = chainedExpiration; } } + + if (localPublisherList.list.size() > 0) + { + PublisherList collection = localPublisherList; + // Unfetched + auto const& current = collection; + auto chainedExpiration = current.validUntil; + + // Earliest + if (!res || chainedExpiration < *res) + { + res = chainedExpiration; + } + } return res; } @@ -1542,23 +1577,18 @@ ValidatorList::getJson() const } } - // Local static keys - PublicKey local; + // Validator keys listed in the local config file Json::Value& jLocalStaticKeys = (res[jss::local_static_keys] = Json::arrayValue); - if (auto it = publisherLists_.find(local); it != publisherLists_.end()) - { - for (auto const& key : it->second.current.list) - jLocalStaticKeys.append(toBase58(TokenType::NodePublic, key)); - } + + for (auto const& key : localPublisherList.list) + jLocalStaticKeys.append(toBase58(TokenType::NodePublic, key)); // Publisher lists Json::Value& jPublisherLists = (res[jss::publisher_lists] = Json::arrayValue); for (auto const& [publicKey, pubCollection] : publisherLists_) { - if (local == publicKey) - continue; Json::Value& curr = jPublisherLists.append(Json::objectValue); curr[jss::pubkey_publisher] = strHex(publicKey); curr[jss::available] = @@ -1618,10 +1648,10 @@ ValidatorList::getJson() const validatorManifests_.for_each_manifest([&jSigningKeys, this](Manifest const& manifest) { auto it = keyListings_.find(manifest.masterKey); - if (it != keyListings_.end()) + if (it != keyListings_.end() && manifest.signingKey) { jSigningKeys[toBase58(TokenType::NodePublic, manifest.masterKey)] = - toBase58(TokenType::NodePublic, manifest.signingKey); + toBase58(TokenType::NodePublic, *manifest.signingKey); } }); @@ -1662,7 +1692,7 @@ ValidatorList::for_each_available( for (auto const& [key, plCollection] : publisherLists_) { - if (plCollection.status != PublisherStatus::available || key.empty()) + if (plCollection.status != PublisherStatus::available) continue; assert(plCollection.maxSequence); func( @@ -1762,10 +1792,8 @@ ValidatorList::calculateQuorum( // Note that the negative UNL protocol introduced the // AbsoluteMinimumQuorum which is 60% of the original UNL size. The // effective quorum should not be lower than it. - static ConsensusParms const parms; return static_cast(std::max( - std::ceil(effectiveUnlSize * parms.minCONSENSUS_FACTOR), - std::ceil(unlSize * parms.negUNL_MIN_CONSENSUS_FACTOR))); + std::ceil(effectiveUnlSize * 0.8f), std::ceil(unlSize * 0.6f))); } TrustChanges @@ -1784,6 +1812,9 @@ ValidatorList::updateTrusted( // Rotate pending and remove expired published lists bool good = true; + // localPublisherList is not processed here. This is because the + // Validators specified in the local config file do not expire nor do + // they have a "remaining" section of PublisherList. for (auto& [pubKey, collection] : publisherLists_) { { @@ -1839,6 +1870,8 @@ ValidatorList::updateTrusted( } } // Remove if expired + // ValidatorLists specified in the local config file never expire. + // Hence, the below steps are not relevant for localPublisherList if (collection.status == PublisherStatus::available && collection.current.validUntil <= closeTime) { @@ -1880,8 +1913,15 @@ ValidatorList::updateTrusted( { trustedSigningKeys_.clear(); + // trustedMasterKeys_ contain non-revoked manifests only. Hence the + // manifests must contain a valid signingKey for (auto const& k : trustedMasterKeys_) - trustedSigningKeys_.insert(validatorManifests_.getSigningKey(k)); + { + std::optional const signingKey = + validatorManifests_.getSigningKey(k); + assert(signingKey); + trustedSigningKeys_.insert(*signingKey); + } } JLOG(j_.debug()) @@ -1923,7 +1963,8 @@ ValidatorList::updateTrusted( << unlSize << ")"; } - if (publisherLists_.size() && unlSize == 0) + if ((publisherLists_.size() || localPublisherList.list.size()) && + unlSize == 0) { // No validators. Lock down. ops.setUNLBlocked(); diff --git a/src/ripple/app/paths/AMMLiquidity.h b/src/ripple/app/paths/AMMLiquidity.h index 7757bd4684d..f1be112b2d6 100644 --- a/src/ripple/app/paths/AMMLiquidity.h +++ b/src/ripple/app/paths/AMMLiquidity.h @@ -137,10 +137,17 @@ class AMMLiquidity TAmounts generateFibSeqOffer(TAmounts const& balances) const; - /** Generate max offer + /** Generate max offer. + * If `fixAMMOverflowOffer` is active, the offer is generated as: + * takerGets = 99% * balances.out takerPays = swapOut(takerGets). + * Return nullopt if takerGets is 0 or takerGets == balances.out. + * + * If `fixAMMOverflowOffer` is not active, the offer is generated as: + * takerPays = max input amount; + * takerGets = swapIn(takerPays). */ - AMMOffer - maxOffer(TAmounts const& balances) const; + std::optional> + maxOffer(TAmounts const& balances, Rules const& rules) const; }; } // namespace ripple diff --git a/src/ripple/app/paths/AMMOffer.h b/src/ripple/app/paths/AMMOffer.h index 10e6017dd96..426ba96a772 100644 --- a/src/ripple/app/paths/AMMOffer.h +++ b/src/ripple/app/paths/AMMOffer.h @@ -50,9 +50,8 @@ class AMMOffer // the swap out of the entire side of the pool, in which case // the swap in amount is infinite. TAmounts const amounts_; - // If seated then current pool balances. Used in one-path limiting steps - // to swap in/out. - std::optional> const balances_; + // Current pool balances. + TAmounts const balances_; // The Spot Price quality if balances != amounts // else the amounts quality Quality const quality_; @@ -63,7 +62,7 @@ class AMMOffer AMMOffer( AMMLiquidity const& ammLiquidity, TAmounts const& amounts, - std::optional> const& balances, + TAmounts const& balances, Quality const& quality); Quality @@ -142,6 +141,12 @@ class AMMOffer // AMM doesn't pay transfer fee on Payment tx return {ofrInRate, QUALITY_ONE}; } + + /** Check the new pool product is greater or equal to the old pool + * product or if decreases then within some threshold. + */ + bool + checkInvariant(TAmounts const& consumed, beast::Journal j) const; }; } // namespace ripple diff --git a/src/ripple/app/paths/PathRequest.cpp b/src/ripple/app/paths/PathRequest.cpp index f2aa363f934..948c6698ad1 100644 --- a/src/ripple/app/paths/PathRequest.cpp +++ b/src/ripple/app/paths/PathRequest.cpp @@ -28,8 +28,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/src/ripple/app/paths/PathRequests.cpp b/src/ripple/app/paths/PathRequests.cpp index 951f55dc800..700cf137209 100644 --- a/src/ripple/app/paths/PathRequests.cpp +++ b/src/ripple/app/paths/PathRequests.cpp @@ -22,8 +22,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -200,6 +200,9 @@ PathRequests::updateAll(std::shared_ptr const& inLedger) break; } + // Hold on to the line cache until after the lock is released, so it can + // be destroyed outside of the lock + std::shared_ptr lastCache; { // Get the latest requests, cache, and ledger for next pass std::lock_guard sl(mLock); @@ -207,6 +210,7 @@ PathRequests::updateAll(std::shared_ptr const& inLedger) if (requests_.empty()) break; requests = requests_; + lastCache = cache; cache = getLineCache(cache->getLedger(), false); } } while (!app_.getJobQueue().isStopping()); diff --git a/src/ripple/app/paths/impl/AMMLiquidity.cpp b/src/ripple/app/paths/impl/AMMLiquidity.cpp index 3f22ebacec5..bcc086e23da 100644 --- a/src/ripple/app/paths/impl/AMMLiquidity.cpp +++ b/src/ripple/app/paths/impl/AMMLiquidity.cpp @@ -93,6 +93,7 @@ AMMLiquidity::generateFibSeqOffer( return cur; } +namespace { template constexpr T maxAmount() @@ -105,16 +106,41 @@ maxAmount() return STAmount(STAmount::cMaxValue / 2, STAmount::cMaxOffset); } +template +T +maxOut(T const& out, Issue const& iss) +{ + Number const res = out * Number{99, -2}; + return toAmount(iss, res, Number::rounding_mode::downward); +} +} // namespace + template -AMMOffer -AMMLiquidity::maxOffer(TAmounts const& balances) const +std::optional> +AMMLiquidity::maxOffer( + TAmounts const& balances, + Rules const& rules) const { - return AMMOffer( - *this, - {maxAmount(), - swapAssetIn(balances, maxAmount(), tradingFee_)}, - balances, - Quality{balances}); + if (!rules.enabled(fixAMMOverflowOffer)) + { + return AMMOffer( + *this, + {maxAmount(), + swapAssetIn(balances, maxAmount(), tradingFee_)}, + balances, + Quality{balances}); + } + else + { + auto const out = maxOut(balances.out, issueOut()); + if (out <= TOut{0} || out >= balances.out) + return std::nullopt; + return AMMOffer( + *this, + {swapAssetOut(balances, out, tradingFee_), out}, + balances, + Quality{balances}); + } } template @@ -167,15 +193,16 @@ AMMLiquidity::getOffer( if (clobQuality && Quality{amounts} < clobQuality) return std::nullopt; return AMMOffer( - *this, amounts, std::nullopt, Quality{amounts}); + *this, amounts, balances, Quality{amounts}); } else if (!clobQuality) { // If there is no CLOB to compare against, return the largest // amount, which doesn't overflow. The size is going to be // changed in BookStep per either deliver amount limit, or - // sendmax, or available output or input funds. - return maxOffer(balances); + // sendmax, or available output or input funds. Might return + // nullopt if the pool is small. + return maxOffer(balances, view.rules()); } else if ( auto const amounts = @@ -188,7 +215,10 @@ AMMLiquidity::getOffer( catch (std::overflow_error const& e) { JLOG(j_.error()) << "AMMLiquidity::getOffer overflow " << e.what(); - return maxOffer(balances); + if (!view.rules().enabled(fixAMMOverflowOffer)) + return maxOffer(balances, view.rules()); + else + return std::nullopt; } catch (std::exception const& e) { diff --git a/src/ripple/app/paths/impl/AMMOffer.cpp b/src/ripple/app/paths/impl/AMMOffer.cpp index 10b75b78565..759851b7afe 100644 --- a/src/ripple/app/paths/impl/AMMOffer.cpp +++ b/src/ripple/app/paths/impl/AMMOffer.cpp @@ -27,7 +27,7 @@ template AMMOffer::AMMOffer( AMMLiquidity const& ammLiquidity, TAmounts const& amounts, - std::optional> const& balances, + TAmounts const& balances, Quality const& quality) : ammLiquidity_(ammLiquidity) , amounts_(amounts) @@ -110,7 +110,7 @@ AMMOffer::limitOut( // Change the offer size according to the conservation function. The offer // quality is increased in this case, but it doesn't matter since there is // only one path. - return {swapAssetOut(*balances_, limit, ammLiquidity_.tradingFee()), limit}; + return {swapAssetOut(balances_, limit, ammLiquidity_.tradingFee()), limit}; } template @@ -122,7 +122,7 @@ AMMOffer::limitIn( // See the comments above in limitOut(). if (ammLiquidity_.multiPath()) return quality().ceil_in(offrAmt, limit); - return {limit, swapAssetIn(*balances_, limit, ammLiquidity_.tradingFee())}; + return {limit, swapAssetIn(balances_, limit, ammLiquidity_.tradingFee())}; } template @@ -132,7 +132,45 @@ AMMOffer::getQualityFunc() const if (ammLiquidity_.multiPath()) return QualityFunction{quality(), QualityFunction::CLOBLikeTag{}}; return QualityFunction{ - *balances_, ammLiquidity_.tradingFee(), QualityFunction::AMMTag{}}; + balances_, ammLiquidity_.tradingFee(), QualityFunction::AMMTag{}}; +} + +template +bool +AMMOffer::checkInvariant( + TAmounts const& consumed, + beast::Journal j) const +{ + if (consumed.in > amounts_.in || consumed.out > amounts_.out) + { + JLOG(j.error()) << "AMMOffer::checkInvariant failed: consumed " + << to_string(consumed.in) << " " + << to_string(consumed.out) << " amounts " + << to_string(amounts_.in) << " " + << to_string(amounts_.out); + + return false; + } + + Number const product = balances_.in * balances_.out; + auto const newBalances = TAmounts{ + balances_.in + consumed.in, balances_.out - consumed.out}; + Number const newProduct = newBalances.in * newBalances.out; + + if (newProduct >= product || + withinRelativeDistance(product, newProduct, Number{1, -7})) + return true; + + JLOG(j.error()) << "AMMOffer::checkInvariant failed: balances " + << to_string(balances_.in) << " " + << to_string(balances_.out) << " new balances " + << to_string(newBalances.in) << " " + << to_string(newBalances.out) << " product/newProduct " + << product << " " << newProduct << " diff " + << (product != Number{0} + ? to_string((product - newProduct) / product) + : "undefined"); + return false; } template class AMMOffer; diff --git a/src/ripple/app/paths/impl/BookStep.cpp b/src/ripple/app/paths/impl/BookStep.cpp index dd6abe577f5..358dac4c796 100644 --- a/src/ripple/app/paths/impl/BookStep.cpp +++ b/src/ripple/app/paths/impl/BookStep.cpp @@ -793,6 +793,17 @@ BookStep::consumeOffer( TAmounts const& stepAmt, TOut const& ownerGives) const { + if (!offer.checkInvariant(ofrAmt, j_)) + { + // purposely written as separate if statements so we get logging even + // when the amendment isn't active. + if (sb.rules().enabled(fixAMMOverflowOffer)) + { + Throw( + tecINVARIANT_FAILED, "AMM pool product invariant failed."); + } + } + // The offer owner gets the ofrAmt. The difference between ofrAmt and // stepAmt is a transfer fee that goes to book_.in.account { diff --git a/src/ripple/app/rdb/backend/detail/impl/Node.cpp b/src/ripple/app/rdb/backend/detail/impl/Node.cpp index 8948360a3ad..0905d6121ae 100644 --- a/src/ripple/app/rdb/backend/detail/impl/Node.cpp +++ b/src/ripple/app/rdb/backend/detail/impl/Node.cpp @@ -1125,7 +1125,7 @@ accountTxPage( { sql = boost::str( boost::format( - prefix + (R"(AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u' + prefix + (R"(AccountTransactions.LedgerSeq BETWEEN %u AND %u ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s LIMIT %u;)")) % @@ -1148,12 +1148,14 @@ accountTxPage( FROM AccountTransactions, Transactions WHERE (AccountTransactions.TransID = Transactions.TransID AND AccountTransactions.Account = '%s' AND - AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u') - OR + AccountTransactions.LedgerSeq BETWEEN %u AND %u) + UNION + SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,Status,RawTxn,TxnMeta + FROM AccountTransactions, Transactions WHERE (AccountTransactions.TransID = Transactions.TransID AND AccountTransactions.Account = '%s' AND - AccountTransactions.LedgerSeq = '%u' AND - AccountTransactions.TxnSeq %s '%u') + AccountTransactions.LedgerSeq = %u AND + AccountTransactions.TxnSeq %s %u) ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s LIMIT %u; diff --git a/src/ripple/app/tx/impl/AMMBid.cpp b/src/ripple/app/tx/impl/AMMBid.cpp index 822e72203a6..e49c378ceeb 100644 --- a/src/ripple/app/tx/impl/AMMBid.cpp +++ b/src/ripple/app/tx/impl/AMMBid.cpp @@ -173,8 +173,18 @@ applyBid( return {tecINTERNAL, false}; STAmount const lptAMMBalance = (*ammSle)[sfLPTokenBalance]; auto const lpTokens = ammLPHolds(sb, *ammSle, account_, ctx_.journal); - if (!ammSle->isFieldPresent(sfAuctionSlot)) - ammSle->makeFieldPresent(sfAuctionSlot); + auto const& rules = ctx_.view().rules(); + if (!rules.enabled(fixInnerObjTemplate)) + { + if (!ammSle->isFieldPresent(sfAuctionSlot)) + ammSle->makeFieldPresent(sfAuctionSlot); + } + else + { + assert(ammSle->isFieldPresent(sfAuctionSlot)); + if (!ammSle->isFieldPresent(sfAuctionSlot)) + return {tecINTERNAL, false}; + } auto& auctionSlot = ammSle->peekFieldObject(sfAuctionSlot); auto const current = duration_cast( diff --git a/src/ripple/app/tx/impl/AMMCreate.h b/src/ripple/app/tx/impl/AMMCreate.h index 81917c4114a..f521f5870b4 100644 --- a/src/ripple/app/tx/impl/AMMCreate.h +++ b/src/ripple/app/tx/impl/AMMCreate.h @@ -27,7 +27,7 @@ namespace ripple { /** AMMCreate implements Automatic Market Maker(AMM) creation Transactor. * It creates a new AMM instance with two tokens. Any trader, or Liquidity * Provider (LP), can create the AMM instance and receive in return shares - * of the AMM pool in the form on LPTokens. The number of tokens that LP gets + * of the AMM pool in the form of LPTokens. The number of tokens that LP gets * are determined by LPTokens = sqrt(A * B), where A and B is the current * composition of the AMM pool. LP can add (AMMDeposit) or withdraw * (AMMWithdraw) tokens from AMM and diff --git a/src/ripple/app/tx/impl/AMMVote.cpp b/src/ripple/app/tx/impl/AMMVote.cpp index ff0598aaa40..d908a93c383 100644 --- a/src/ripple/app/tx/impl/AMMVote.cpp +++ b/src/ripple/app/tx/impl/AMMVote.cpp @@ -104,6 +104,7 @@ applyVote( Number den{0}; // Account already has vote entry bool foundAccount = false; + auto const& rules = ctx_.view().rules(); // Iterate over the current vote entries and update each entry // per current total tokens balance and each LP tokens balance. // Find the entry with the least tokens and whether the account @@ -119,7 +120,7 @@ applyVote( continue; } auto feeVal = entry[sfTradingFee]; - STObject newEntry{sfVoteEntry}; + STObject newEntry = STObject::makeInnerObject(sfVoteEntry, rules); // The account already has the vote entry. if (account == account_) { @@ -158,7 +159,7 @@ applyVote( { auto update = [&](std::optional const& minPos = std::nullopt) { - STObject newEntry{sfVoteEntry}; + STObject newEntry = STObject::makeInnerObject(sfVoteEntry, rules); if (feeNew != 0) newEntry.setFieldU16(sfTradingFee, feeNew); newEntry.setFieldU32( @@ -199,6 +200,10 @@ applyVote( } } + assert( + !ctx_.view().rules().enabled(fixInnerObjTemplate) || + ammSle->isFieldPresent(sfAuctionSlot)); + // Update the vote entries and the trading/discounted fee. ammSle->setFieldArray(sfVoteSlots, updatedVoteSlots); if (auto const fee = static_cast(num / den)) diff --git a/src/ripple/app/tx/impl/DID.cpp b/src/ripple/app/tx/impl/DID.cpp index c92162c8306..1cad8992ad9 100644 --- a/src/ripple/app/tx/impl/DID.cpp +++ b/src/ripple/app/tx/impl/DID.cpp @@ -161,6 +161,13 @@ DIDSet::doApply() set(sfURI); set(sfDIDDocument); set(sfData); + if (ctx_.view().rules().enabled(fixEmptyDID) && + !sleDID->isFieldPresent(sfURI) && + !sleDID->isFieldPresent(sfDIDDocument) && + !sleDID->isFieldPresent(sfData)) + { + return tecEMPTY_DID; + } return addSLE(ctx_, sleDID, account_); } diff --git a/src/ripple/app/tx/impl/DeleteAccount.cpp b/src/ripple/app/tx/impl/DeleteAccount.cpp index 49b645e31d9..efa38a4b74e 100644 --- a/src/ripple/app/tx/impl/DeleteAccount.cpp +++ b/src/ripple/app/tx/impl/DeleteAccount.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -146,6 +147,18 @@ removeDIDFromLedger( return DIDDelete::deleteSLE(view, sleDel, account, j); } +TER +removeOracleFromLedger( + Application&, + ApplyView& view, + AccountID const& account, + uint256 const&, + std::shared_ptr const& sleDel, + beast::Journal j) +{ + return DeleteOracle::deleteOracle(view, sleDel, account, j); +} + // Return nullptr if the LedgerEntryType represents an obligation that can't // be deleted. Otherwise return the pointer to the function that can delete // the non-obligation @@ -166,6 +179,8 @@ nonObligationDeleter(LedgerEntryType t) return removeNFTokenOfferFromLedger; case ltDID: return removeDIDFromLedger; + case ltORACLE: + return removeOracleFromLedger; default: return nullptr; } diff --git a/src/ripple/app/tx/impl/DeleteOracle.cpp b/src/ripple/app/tx/impl/DeleteOracle.cpp new file mode 100644 index 00000000000..dfaecc384d4 --- /dev/null +++ b/src/ripple/app/tx/impl/DeleteOracle.cpp @@ -0,0 +1,110 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +NotTEC +DeleteOracle::preflight(PreflightContext const& ctx) +{ + if (!ctx.rules.enabled(featurePriceOracle)) + return temDISABLED; + + if (auto const ret = preflight1(ctx); !isTesSuccess(ret)) + return ret; + + if (ctx.tx.getFlags() & tfUniversalMask) + { + JLOG(ctx.j.debug()) << "Oracle Delete: invalid flags."; + return temINVALID_FLAG; + } + + return preflight2(ctx); +} + +TER +DeleteOracle::preclaim(PreclaimContext const& ctx) +{ + if (!ctx.view.exists(keylet::account(ctx.tx.getAccountID(sfAccount)))) + return terNO_ACCOUNT; + + if (auto const sle = ctx.view.read(keylet::oracle( + ctx.tx.getAccountID(sfAccount), ctx.tx[sfOracleDocumentID])); + !sle) + { + JLOG(ctx.j.debug()) << "Oracle Delete: Oracle does not exist."; + return tecNO_ENTRY; + } + else if (ctx.tx.getAccountID(sfAccount) != sle->getAccountID(sfOwner)) + { + // this can't happen because of the above check + JLOG(ctx.j.debug()) << "Oracle Delete: invalid account."; + return tecINTERNAL; + } + return tesSUCCESS; +} + +TER +DeleteOracle::deleteOracle( + ApplyView& view, + std::shared_ptr const& sle, + AccountID const& account, + beast::Journal j) +{ + if (!sle) + return tesSUCCESS; + + if (!view.dirRemove( + keylet::ownerDir(account), (*sle)[sfOwnerNode], sle->key(), true)) + { + JLOG(j.fatal()) << "Unable to delete Oracle from owner."; + return tefBAD_LEDGER; + } + + auto const sleOwner = view.peek(keylet::account(account)); + if (!sleOwner) + return tecINTERNAL; + + auto const count = + sle->getFieldArray(sfPriceDataSeries).size() > 5 ? -2 : -1; + + adjustOwnerCount(view, sleOwner, count, j); + + view.erase(sle); + + return tesSUCCESS; +} + +TER +DeleteOracle::doApply() +{ + if (auto sle = ctx_.view().peek( + keylet::oracle(account_, ctx_.tx[sfOracleDocumentID]))) + return deleteOracle(ctx_.view(), sle, account_, j_); + + return tecINTERNAL; +} + +} // namespace ripple diff --git a/src/ripple/app/tx/impl/DeleteOracle.h b/src/ripple/app/tx/impl/DeleteOracle.h new file mode 100644 index 00000000000..e578adaaaf0 --- /dev/null +++ b/src/ripple/app/tx/impl/DeleteOracle.h @@ -0,0 +1,64 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_TX_DELETEORACLE_H_INCLUDED +#define RIPPLE_TX_DELETEORACLE_H_INCLUDED + +#include + +namespace ripple { + +/** + Price Oracle is a system that acts as a bridge between + a blockchain network and the external world, providing off-chain price data + to decentralized applications (dApps) on the blockchain. This implementation + conforms to the requirements specified in the XLS-47d. + + The DeleteOracle transactor implements the deletion of Oracle objects. +*/ + +class DeleteOracle : public Transactor +{ +public: + static constexpr ConsequencesFactoryType ConsequencesFactory{Normal}; + + explicit DeleteOracle(ApplyContext& ctx) : Transactor(ctx) + { + } + + static NotTEC + preflight(PreflightContext const& ctx); + + static TER + preclaim(PreclaimContext const& ctx); + + TER + doApply() override; + + static TER + deleteOracle( + ApplyView& view, + std::shared_ptr const& sle, + AccountID const& account, + beast::Journal j); +}; + +} // namespace ripple + +#endif // RIPPLE_TX_DELETEORACLE_H_INCLUDED diff --git a/src/ripple/app/tx/impl/InvariantCheck.cpp b/src/ripple/app/tx/impl/InvariantCheck.cpp index c717777f88f..c0ef5bbf0c5 100644 --- a/src/ripple/app/tx/impl/InvariantCheck.cpp +++ b/src/ripple/app/tx/impl/InvariantCheck.cpp @@ -392,6 +392,7 @@ LedgerEntryTypesMatch::visitEntry( case ltXCHAIN_OWNED_CLAIM_ID: case ltXCHAIN_OWNED_CREATE_ACCOUNT_CLAIM_ID: case ltDID: + case ltORACLE: break; default: invalidTypeAdded_ = true; diff --git a/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp b/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp index 61aa7e0629a..02471c1d482 100644 --- a/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp +++ b/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp @@ -301,6 +301,60 @@ NFTokenAcceptOffer::pay( return tesSUCCESS; } +TER +NFTokenAcceptOffer::transferNFToken( + AccountID const& buyer, + AccountID const& seller, + uint256 const& nftokenID) +{ + auto tokenAndPage = nft::findTokenAndPage(view(), seller, nftokenID); + + if (!tokenAndPage) + return tecINTERNAL; + + if (auto const ret = nft::removeToken( + view(), seller, nftokenID, std::move(tokenAndPage->page)); + !isTesSuccess(ret)) + return ret; + + auto const sleBuyer = view().read(keylet::account(buyer)); + if (!sleBuyer) + return tecINTERNAL; + + std::uint32_t const buyerOwnerCountBefore = + sleBuyer->getFieldU32(sfOwnerCount); + + auto const insertRet = + nft::insertToken(view(), buyer, std::move(tokenAndPage->token)); + + // if fixNFTokenReserve is enabled, check if the buyer has sufficient + // reserve to own a new object, if their OwnerCount changed. + // + // There was an issue where the buyer accepts a sell offer, the ledger + // didn't check if the buyer has enough reserve, meaning that buyer can get + // NFTs free of reserve. + if (view().rules().enabled(fixNFTokenReserve)) + { + // To check if there is sufficient reserve, we cannot use mPriorBalance + // because NFT is sold for a price. So we must use the balance after + // the deduction of the potential offer price. A small caveat here is + // that the balance has already deducted the transaction fee, meaning + // that the reserve requirement is a few drops higher. + auto const buyerBalance = sleBuyer->getFieldAmount(sfBalance); + + auto const buyerOwnerCountAfter = sleBuyer->getFieldU32(sfOwnerCount); + if (buyerOwnerCountAfter > buyerOwnerCountBefore) + { + if (auto const reserve = + view().fees().accountReserve(buyerOwnerCountAfter); + buyerBalance < reserve) + return tecINSUFFICIENT_RESERVE; + } + } + + return insertRet; +} + TER NFTokenAcceptOffer::acceptOffer(std::shared_ptr const& offer) { @@ -333,17 +387,7 @@ NFTokenAcceptOffer::acceptOffer(std::shared_ptr const& offer) } // Now transfer the NFT: - auto tokenAndPage = nft::findTokenAndPage(view(), seller, nftokenID); - - if (!tokenAndPage) - return tecINTERNAL; - - if (auto const ret = nft::removeToken( - view(), seller, nftokenID, std::move(tokenAndPage->page)); - !isTesSuccess(ret)) - return ret; - - return nft::insertToken(view(), buyer, std::move(tokenAndPage->token)); + return transferNFToken(buyer, seller, nftokenID); } TER @@ -431,17 +475,8 @@ NFTokenAcceptOffer::doApply() return r; } - auto tokenAndPage = nft::findTokenAndPage(view(), seller, nftokenID); - - if (!tokenAndPage) - return tecINTERNAL; - - if (auto const ret = nft::removeToken( - view(), seller, nftokenID, std::move(tokenAndPage->page)); - !isTesSuccess(ret)) - return ret; - - return nft::insertToken(view(), buyer, std::move(tokenAndPage->token)); + // Now transfer the NFT: + return transferNFToken(buyer, seller, nftokenID); } if (bo) diff --git a/src/ripple/app/tx/impl/NFTokenAcceptOffer.h b/src/ripple/app/tx/impl/NFTokenAcceptOffer.h index 2d1b14ba284..e1b26cbecea 100644 --- a/src/ripple/app/tx/impl/NFTokenAcceptOffer.h +++ b/src/ripple/app/tx/impl/NFTokenAcceptOffer.h @@ -38,6 +38,12 @@ class NFTokenAcceptOffer : public Transactor std::shared_ptr const& buy, std::shared_ptr const& sell); + TER + transferNFToken( + AccountID const& buyer, + AccountID const& seller, + uint256 const& nfTokenID); + public: static constexpr ConsequencesFactoryType ConsequencesFactory{Normal}; diff --git a/src/ripple/app/tx/impl/Offer.h b/src/ripple/app/tx/impl/Offer.h index 027030ec8d0..bdae4d2b155 100644 --- a/src/ripple/app/tx/impl/Offer.h +++ b/src/ripple/app/tx/impl/Offer.h @@ -163,6 +163,15 @@ class TOffer : private TOfferBase // CLOB offer pays the transfer fee return {ofrInRate, ofrOutRate}; } + + /** Check any required invariant. Limit order book offer + * always returns true. + */ + bool + checkInvariant(TAmounts const&, beast::Journal j) const + { + return true; + } }; using Offer = TOffer<>; diff --git a/src/ripple/app/tx/impl/SetOracle.cpp b/src/ripple/app/tx/impl/SetOracle.cpp new file mode 100644 index 00000000000..37dc6fcd212 --- /dev/null +++ b/src/ripple/app/tx/impl/SetOracle.cpp @@ -0,0 +1,312 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +static inline std::pair +tokenPairKey(STObject const& pair) +{ + return std::make_pair( + pair.getFieldCurrency(sfBaseAsset).currency(), + pair.getFieldCurrency(sfQuoteAsset).currency()); +} + +NotTEC +SetOracle::preflight(PreflightContext const& ctx) +{ + if (!ctx.rules.enabled(featurePriceOracle)) + return temDISABLED; + + if (auto const ret = preflight1(ctx); !isTesSuccess(ret)) + return ret; + + if (ctx.tx.getFlags() & tfUniversalMask) + return temINVALID_FLAG; + + auto const& dataSeries = ctx.tx.getFieldArray(sfPriceDataSeries); + if (dataSeries.empty()) + return temARRAY_EMPTY; + if (dataSeries.size() > maxOracleDataSeries) + return temARRAY_TOO_LARGE; + + auto isInvalidLength = [&](auto const& sField, std::size_t length) { + return ctx.tx.isFieldPresent(sField) && + (ctx.tx[sField].length() == 0 || ctx.tx[sField].length() > length); + }; + + if (isInvalidLength(sfProvider, maxOracleProvider) || + isInvalidLength(sfURI, maxOracleURI) || + isInvalidLength(sfAssetClass, maxOracleSymbolClass)) + return temMALFORMED; + + return preflight2(ctx); +} + +TER +SetOracle::preclaim(PreclaimContext const& ctx) +{ + auto const sleSetter = + ctx.view.read(keylet::account(ctx.tx.getAccountID(sfAccount))); + if (!sleSetter) + return terNO_ACCOUNT; + + // lastUpdateTime must be within maxLastUpdateTimeDelta seconds + // of the last closed ledger + using namespace std::chrono; + std::size_t const closeTime = + duration_cast(ctx.view.info().closeTime.time_since_epoch()) + .count(); + std::size_t const lastUpdateTime = ctx.tx[sfLastUpdateTime]; + if (lastUpdateTime < epoch_offset.count()) + return tecINVALID_UPDATE_TIME; + std::size_t const lastUpdateTimeEpoch = + lastUpdateTime - epoch_offset.count(); + if (closeTime < maxLastUpdateTimeDelta) + Throw( + "Oracle: close time is less than maxLastUpdateTimeDelta"); + if (lastUpdateTimeEpoch < (closeTime - maxLastUpdateTimeDelta) || + lastUpdateTimeEpoch > (closeTime + maxLastUpdateTimeDelta)) + return tecINVALID_UPDATE_TIME; + + auto const sle = ctx.view.read(keylet::oracle( + ctx.tx.getAccountID(sfAccount), ctx.tx[sfOracleDocumentID])); + + // token pairs to add/update + hash_set> pairs; + // token pairs to delete. if a token pair doesn't include + // the price then this pair should be deleted from the object. + hash_set> pairsDel; + for (auto const& entry : ctx.tx.getFieldArray(sfPriceDataSeries)) + { + if (entry[sfBaseAsset] == entry[sfQuoteAsset]) + return temMALFORMED; + auto const key = tokenPairKey(entry); + if (pairs.contains(key) || pairsDel.contains(key)) + return temMALFORMED; + if (entry[~sfScale] > maxPriceScale) + return temMALFORMED; + if (entry.isFieldPresent(sfAssetPrice)) + pairs.emplace(key); + else if (sle) + pairsDel.emplace(key); + else + return temMALFORMED; + } + + // Lambda is used to check if the value of a field, passed + // in the transaction, is equal to the value of that field + // in the on-ledger object. + auto isConsistent = [&ctx, &sle](auto const& field) { + auto const v = ctx.tx[~field]; + return !v || *v == (*sle)[field]; + }; + + std::uint32_t adjustReserve = 0; + if (sle) + { + // update + // Account is the Owner since we can get sle + + // lastUpdateTime must be more recent than the previous one + if (ctx.tx[sfLastUpdateTime] <= (*sle)[sfLastUpdateTime]) + return tecINVALID_UPDATE_TIME; + + if (!isConsistent(sfProvider) || !isConsistent(sfAssetClass)) + return temMALFORMED; + + for (auto const& entry : sle->getFieldArray(sfPriceDataSeries)) + { + auto const key = tokenPairKey(entry); + if (!pairs.contains(key)) + { + if (pairsDel.contains(key)) + pairsDel.erase(key); + else + pairs.emplace(key); + } + } + if (!pairsDel.empty()) + return tecTOKEN_PAIR_NOT_FOUND; + + auto const oldCount = + sle->getFieldArray(sfPriceDataSeries).size() > 5 ? 2 : 1; + auto const newCount = pairs.size() > 5 ? 2 : 1; + adjustReserve = newCount - oldCount; + } + else + { + // create + + if (!ctx.tx.isFieldPresent(sfProvider) || + !ctx.tx.isFieldPresent(sfAssetClass)) + return temMALFORMED; + adjustReserve = pairs.size() > 5 ? 2 : 1; + } + + if (pairs.empty()) + return tecARRAY_EMPTY; + if (pairs.size() > maxOracleDataSeries) + return tecARRAY_TOO_LARGE; + + auto const reserve = ctx.view.fees().accountReserve( + sleSetter->getFieldU32(sfOwnerCount) + adjustReserve); + auto const& balance = sleSetter->getFieldAmount(sfBalance); + + if (balance < reserve) + return tecINSUFFICIENT_RESERVE; + + return tesSUCCESS; +} + +static bool +adjustOwnerCount(ApplyContext& ctx, int count) +{ + if (auto const sleAccount = + ctx.view().peek(keylet::account(ctx.tx[sfAccount]))) + { + adjustOwnerCount(ctx.view(), sleAccount, count, ctx.journal); + return true; + } + + return false; +} + +static void +setPriceDataInnerObjTemplate(STObject& obj) +{ + if (SOTemplate const* elements = + InnerObjectFormats::getInstance().findSOTemplateBySField( + sfPriceData)) + obj.set(*elements); +} + +TER +SetOracle::doApply() +{ + auto const oracleID = keylet::oracle(account_, ctx_.tx[sfOracleDocumentID]); + + if (auto sle = ctx_.view().peek(oracleID)) + { + // update + // the token pair that doesn't have their price updated will not + // include neither price nor scale in the updated PriceDataSeries + + hash_map, STObject> pairs; + // collect current token pairs + for (auto const& entry : sle->getFieldArray(sfPriceDataSeries)) + { + STObject priceData{sfPriceData}; + setPriceDataInnerObjTemplate(priceData); + priceData.setFieldCurrency( + sfBaseAsset, entry.getFieldCurrency(sfBaseAsset)); + priceData.setFieldCurrency( + sfQuoteAsset, entry.getFieldCurrency(sfQuoteAsset)); + pairs.emplace(tokenPairKey(entry), std::move(priceData)); + } + auto const oldCount = pairs.size() > 5 ? 2 : 1; + // update/add/delete pairs + for (auto const& entry : ctx_.tx.getFieldArray(sfPriceDataSeries)) + { + auto const key = tokenPairKey(entry); + if (!entry.isFieldPresent(sfAssetPrice)) + { + // delete token pair + pairs.erase(key); + } + else if (auto iter = pairs.find(key); iter != pairs.end()) + { + // update the price + iter->second.setFieldU64( + sfAssetPrice, entry.getFieldU64(sfAssetPrice)); + if (entry.isFieldPresent(sfScale)) + iter->second.setFieldU8(sfScale, entry.getFieldU8(sfScale)); + } + else + { + // add a token pair with the price + STObject priceData{sfPriceData}; + setPriceDataInnerObjTemplate(priceData); + priceData.setFieldCurrency( + sfBaseAsset, entry.getFieldCurrency(sfBaseAsset)); + priceData.setFieldCurrency( + sfQuoteAsset, entry.getFieldCurrency(sfQuoteAsset)); + priceData.setFieldU64( + sfAssetPrice, entry.getFieldU64(sfAssetPrice)); + if (entry.isFieldPresent(sfScale)) + priceData.setFieldU8(sfScale, entry.getFieldU8(sfScale)); + pairs.emplace(key, std::move(priceData)); + } + } + STArray updatedSeries; + for (auto const& iter : pairs) + updatedSeries.push_back(std::move(iter.second)); + sle->setFieldArray(sfPriceDataSeries, updatedSeries); + if (ctx_.tx.isFieldPresent(sfURI)) + sle->setFieldVL(sfURI, ctx_.tx[sfURI]); + sle->setFieldU32(sfLastUpdateTime, ctx_.tx[sfLastUpdateTime]); + + auto const newCount = pairs.size() > 5 ? 2 : 1; + auto const adjust = newCount - oldCount; + if (adjust != 0 && !adjustOwnerCount(ctx_, adjust)) + return tefINTERNAL; + + ctx_.view().update(sle); + } + else + { + // create + + sle = std::make_shared(oracleID); + sle->setAccountID(sfOwner, ctx_.tx.getAccountID(sfAccount)); + sle->setFieldVL(sfProvider, ctx_.tx[sfProvider]); + if (ctx_.tx.isFieldPresent(sfURI)) + sle->setFieldVL(sfURI, ctx_.tx[sfURI]); + auto const& series = ctx_.tx.getFieldArray(sfPriceDataSeries); + sle->setFieldArray(sfPriceDataSeries, series); + sle->setFieldVL(sfAssetClass, ctx_.tx[sfAssetClass]); + sle->setFieldU32(sfLastUpdateTime, ctx_.tx[sfLastUpdateTime]); + + auto page = ctx_.view().dirInsert( + keylet::ownerDir(account_), sle->key(), describeOwnerDir(account_)); + if (!page) + return tecDIR_FULL; + + (*sle)[sfOwnerNode] = *page; + + auto const count = series.size() > 5 ? 2 : 1; + if (!adjustOwnerCount(ctx_, count)) + return tefINTERNAL; + + ctx_.view().insert(sle); + } + + return tesSUCCESS; +} + +} // namespace ripple diff --git a/src/ripple/basics/SubmitSync.h b/src/ripple/app/tx/impl/SetOracle.h similarity index 55% rename from src/ripple/basics/SubmitSync.h rename to src/ripple/app/tx/impl/SetOracle.h index 12311c676e8..0ab8e603aa5 100644 --- a/src/ripple/basics/SubmitSync.h +++ b/src/ripple/app/tx/impl/SetOracle.h @@ -17,25 +17,41 @@ */ //============================================================================== -#ifndef RIPPLE_BASICS_SUBMITSYNC_H_INCLUDED -#define RIPPLE_BASICS_SUBMITSYNC_H_INCLUDED +#ifndef RIPPLE_TX_SETORACLE_H_INCLUDED +#define RIPPLE_TX_SETORACLE_H_INCLUDED + +#include namespace ripple { -namespace RPC { /** - * Possible values for defining synchronous behavior of the transaction - * submission API. - * 1) sync (default): Process transactions in a batch immediately, - * and return only once the transaction has been processed. - * 2) async: Put transaction into the batch for the next processing - * interval and return immediately. - * 3) wait: Put transaction into the batch for the next processing - * interval and return only after it is processed. - */ -enum class SubmitSync { sync, async, wait }; - -} // namespace RPC + Price Oracle is a system that acts as a bridge between + a blockchain network and the external world, providing off-chain price data + to decentralized applications (dApps) on the blockchain. This implementation + conforms to the requirements specified in the XLS-47d. + + The SetOracle transactor implements creating or updating Oracle objects. +*/ + +class SetOracle : public Transactor +{ +public: + static constexpr ConsequencesFactoryType ConsequencesFactory{Normal}; + + explicit SetOracle(ApplyContext& ctx) : Transactor(ctx) + { + } + + static NotTEC + preflight(PreflightContext const& ctx); + + static TER + preclaim(PreclaimContext const& ctx); + + TER + doApply() override; +}; + } // namespace ripple -#endif \ No newline at end of file +#endif // RIPPLE_TX_SETORACLE_H_INCLUDED diff --git a/src/ripple/app/tx/impl/XChainBridge.cpp b/src/ripple/app/tx/impl/XChainBridge.cpp index 59450113d2b..be315236f2c 100644 --- a/src/ripple/app/tx/impl/XChainBridge.cpp +++ b/src/ripple/app/tx/impl/XChainBridge.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -41,7 +42,6 @@ #include #include #include - #include #include @@ -672,6 +672,12 @@ finalizeClaimHelper( // if the transfer failed, distribute the pool for "OnTransferFail" // cases (the attesters did their job) STAmount const share = [&] { + auto const round_mode = + innerSb.rules().enabled(fixXChainRewardRounding) + ? Number::rounding_mode::downward + : Number::getround(); + saveNumberRoundMode _{Number::setround(round_mode)}; + STAmount const den{rewardAccounts.size()}; return divide(rewardPool, den, rewardPool.issue()); }(); diff --git a/src/ripple/app/tx/impl/apply.cpp b/src/ripple/app/tx/impl/apply.cpp index 4881f2a49b7..c0704c5c3ae 100644 --- a/src/ripple/app/tx/impl/apply.cpp +++ b/src/ripple/app/tx/impl/apply.cpp @@ -134,7 +134,7 @@ applyTransaction( if (retryAssured) flags = flags | tapRETRY; - JLOG(j.trace()) << "TXN " << txn.getTransactionID() + JLOG(j.debug()) << "TXN " << txn.getTransactionID() << (retryAssured ? "/retry" : "/final"); try @@ -142,7 +142,7 @@ applyTransaction( auto const result = apply(app, view, txn, flags, j); if (result.second) { - JLOG(j.trace()) + JLOG(j.debug()) << "Transaction applied: " << transHuman(result.first); return ApplyResult::Success; } @@ -151,17 +151,17 @@ applyTransaction( isTelLocal(result.first)) { // failure - JLOG(j.trace()) + JLOG(j.debug()) << "Transaction failure: " << transHuman(result.first); return ApplyResult::Fail; } - JLOG(j.trace()) << "Transaction retry: " << transHuman(result.first); + JLOG(j.debug()) << "Transaction retry: " << transHuman(result.first); return ApplyResult::Retry; } catch (std::exception const& ex) { - JLOG(j.trace()) << "Throws: " << ex.what(); + JLOG(j.warn()) << "Throws: " << ex.what(); return ApplyResult::Fail; } } diff --git a/src/ripple/app/tx/impl/applySteps.cpp b/src/ripple/app/tx/impl/applySteps.cpp index 10e2b0c4524..1a1fc343e3c 100644 --- a/src/ripple/app/tx/impl/applySteps.cpp +++ b/src/ripple/app/tx/impl/applySteps.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -159,6 +161,10 @@ with_txn_type(TxType txnType, F&& f) return f.template operator()(); case ttDID_DELETE: return f.template operator()(); + case ttORACLE_SET: + return f.template operator()(); + case ttORACLE_DELETE: + return f.template operator()(); default: throw UnknownTxnType(txnType); } diff --git a/src/ripple/basics/impl/contract.cpp b/src/ripple/basics/impl/contract.cpp index bf7df682587..3a50db38010 100644 --- a/src/ripple/basics/impl/contract.cpp +++ b/src/ripple/basics/impl/contract.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include namespace ripple { diff --git a/src/ripple/beast/container/detail/aged_unordered_container.h b/src/ripple/beast/container/detail/aged_unordered_container.h index fbd2315794b..fcdccd2a637 100644 --- a/src/ripple/beast/container/detail/aged_unordered_container.h +++ b/src/ripple/beast/container/detail/aged_unordered_container.h @@ -1184,12 +1184,9 @@ class aged_unordered_container beast::detail::aged_container_iterator first, beast::detail::aged_container_iterator last); - /* - * This is broken as of at least gcc 11.3.0 template auto erase(K const& k) -> size_type; - */ void swap(aged_unordered_container& other) noexcept; @@ -3065,7 +3062,6 @@ aged_unordered_container< first.iterator()); } -/* template < bool IsMulti, bool IsMap, @@ -3105,7 +3101,6 @@ aged_unordered_container< } return n; } -*/ template < bool IsMulti, diff --git a/src/ripple/beast/hash/impl/xxhash.cpp b/src/ripple/beast/hash/impl/xxhash.cpp deleted file mode 100644 index 4a6c85db815..00000000000 --- a/src/ripple/beast/hash/impl/xxhash.cpp +++ /dev/null @@ -1,1004 +0,0 @@ -/* -xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- public discussion board : https://groups.google.com/forum/#!forum/lz4c -*/ - -#include - -#include - -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as -// x86. For others CPU, the compiler will be more cautious, and insert extra -// code to ensure aligned access is respected. If you know your target CPU -// supports unaligned memory access, you want to force this option manually to -// improve performance. You can also enable this parameter if you know your -// input data will always be aligned (boundaries of 4, for U32). -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || \ - defined(__x86_64__) || defined(_M_X64) -#define XXH_USE_UNALIGNED_ACCESS 1 -#endif - -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger -// a memory access error, since it is a bad pointer. When this option is -// enabled, xxHash output for null input pointers will be the same as a -// null-length input. This option has a very small performance cost (only -// measurable on small inputs). By default, this option is disabled. To enable -// it, uncomment below define : #define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on -// little-endian convention. Results are therefore identical for little-endian -// and big-endian CPU. This comes at a performance cost for big-endian CPU, -// since some swapping is required to emulate little-endian format. Should -// endian-independance be of no importance for your application, you may set the -// #define below to 1. It will improve speed for Big-endian CPU. This option has -// no impact on Little_Endian CPU. -#define XXH_FORCE_NATIVE_FORMAT 0 - -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -#pragma warning( \ - disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#ifdef _MSC_VER // Visual Studio -#define FORCE_INLINE static __forceinline -#else -#ifdef __GNUC__ -#define FORCE_INLINE static inline __attribute__((always_inline)) -#else -#define FORCE_INLINE static inline -#endif -#endif - -//************************************** -// Includes & Memory related functions -//************************************** -// #include "xxhash.h" -// Modify the local functions below should you wish to use some other memory -// routines for malloc(), free() -#include -static void* -XXH_malloc(size_t s) -{ - return malloc(s); -} -static void -XXH_free(void* p) -{ - free(p); -} -// for memcpy() -#include -static void* -XXH_memcpy(void* dest, const void* src, size_t size) -{ - return memcpy(dest, src, size); -} - -//************************************** -// Basic Types -//************************************** -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -#include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -#define _PACKED __attribute__((packed)) -#else -#define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -#ifdef __IBMC__ -#pragma pack(1) -#else -#pragma pack(push, 1) -#endif -#endif - -namespace beast { -namespace detail { - -typedef struct _U32_S -{ - U32 v; -} _PACKED U32_S; -typedef struct _U64_S -{ - U64 v; -} _PACKED U64_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -#pragma pack(pop) -#endif - -#define A32(x) (((U32_S*)(x))->v) -#define A64(x) (((U64_S*)(x))->v) - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// Note : although _rotl exists for minGW (GCC under windows), performance seems -// poor -#if defined(_MSC_VER) -#define XXH_rotl32(x, r) _rotl(x, r) -#define XXH_rotl64(x, r) _rotl64(x, r) -#else -#define XXH_rotl32(x, r) ((x << r) | (x >> (32 - r))) -#define XXH_rotl64(x, r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) // Visual Studio -#define XXH_swap32 _byteswap_ulong -#define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -#define XXH_swap32 __builtin_bswap32 -#define XXH_swap64 __builtin_bswap64 -#else -static inline U32 -XXH_swap32(U32 x) -{ - return ((x << 24) & 0xff000000) | ((x << 8) & 0x00ff0000) | - ((x >> 8) & 0x0000ff00) | ((x >> 24) & 0x000000ff); -} -static inline U64 -XXH_swap64(U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - -#define PRIME64_1 11400714785074694791ULL -#define PRIME64_2 14029467366897019727ULL -#define PRIME64_3 1609587929392839161ULL -#define PRIME64_4 9650029242287828579ULL -#define PRIME64_5 2870177450012600261ULL - -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian = 0, XXH_littleEndian = 1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN - // externally, for example using a compiler - // switch -static const int one = 1; -#define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) -#endif - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) \ - { \ - enum { XXH_static_assert = 1 / (!!(c)) }; \ - } // use only *after* variable declarations - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 -XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align == XXH_unaligned) - return endian == XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian == XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); -} - -FORCE_INLINE U32 -XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -FORCE_INLINE U64 -XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align == XXH_unaligned) - { - // Use memcpy to avoid unaligned UB - U64 tmp_aligned; - std::memcpy(&tmp_aligned, ptr, sizeof(U64)); - return endian == XXH_littleEndian ? tmp_aligned - : XXH_swap64(tmp_aligned); - } - else - return endian == XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); -} - -FORCE_INLINE U64 -XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -//**************************** -// Simple Hash Functions -//**************************** -FORCE_INLINE U32 -XXH32_endian_align( - const void* input, - size_t len, - U32 seed, - XXH_endianess endian, - XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p == NULL) - { - len = 0; - bEnd = p = (const BYTE*)(size_t)16; - } -#endif - - if (len >= 16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do - { - v1 += XXH_get32bits(p) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p += 4; - v2 += XXH_get32bits(p) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p += 4; - v3 += XXH_get32bits(p) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p += 4; - v4 += XXH_get32bits(p) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p += 4; - } while (p <= limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + - XXH_rotl32(v4, 18); - } - else - { - h32 = seed + PRIME32_5; - } - - h32 += (U32)len; - - while (p + 4 <= bEnd) - { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p += 4; - } - - while (p < bEnd) - { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - -unsigned int -XXH32(const void* input, size_t len, unsigned seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, input, len); - return XXH32_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == - 0) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align( - input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align( - input, len, seed, XXH_bigEndian, XXH_aligned); - } -#endif - - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align( - input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align( - input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -FORCE_INLINE U64 -XXH64_endian_align( - const void* input, - size_t len, - U64 seed, - XXH_endianess endian, - XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p == NULL) - { - len = 0; - bEnd = p = (const BYTE*)(size_t)32; - } -#endif - - if (len >= 32) - { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do - { - v1 += XXH_get64bits(p) * PRIME64_2; - p += 8; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - v2 += XXH_get64bits(p) * PRIME64_2; - p += 8; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - v3 += XXH_get64bits(p) * PRIME64_2; - p += 8; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - v4 += XXH_get64bits(p) * PRIME64_2; - p += 8; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - } while (p <= limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + - XXH_rotl64(v4, 18); - - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64 * PRIME64_1 + PRIME64_4; - } - else - { - h64 = seed + PRIME64_5; - } - - h64 += (U64)len; - - while (p + 8 <= bEnd) - { - U64 k1 = XXH_get64bits(p); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = XXH_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; - p += 8; - } - - if (p + 4 <= bEnd) - { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p += 4; - } - - while (p < bEnd) - { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - -unsigned long long -XXH64(const void* input, size_t len, unsigned long long seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, input, len); - return XXH64_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 7) == - 0) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align( - input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align( - input, len, seed, XXH_bigEndian, XXH_aligned); - } -#endif - - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align( - input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align( - input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -/**************************************************** - * Advanced Hash Functions - ****************************************************/ - -/*** Allocation ***/ -typedef struct -{ - U64 total_len; - U32 seed; - U32 v1; - U32 v2; - U32 v3; - U32 v4; - U32 mem32[4]; /* defined as U32 for alignment */ - U32 memsize; -} XXH_istate32_t; - -typedef struct -{ - U64 total_len; - U64 seed; - U64 v1; - U64 v2; - U64 v3; - U64 v4; - U64 mem64[4]; /* defined as U64 for alignment */ - U32 memsize; -} XXH_istate64_t; - -XXH32_state_t* -XXH32_createState(void) -{ - static_assert( - sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t), - ""); // A compilation error here means XXH32_state_t is not large - // enough - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_errorcode -XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -}; - -XXH64_state_t* -XXH64_createState(void) -{ - static_assert( - sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t), - ""); // A compilation error here means XXH64_state_t is not large - // enough - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_errorcode -XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -}; - -/*** Hash feed ***/ - -XXH_errorcode -XXH32_reset(XXH32_state_t* state_in, U32 seed) -{ - XXH_istate32_t* state = (XXH_istate32_t*)state_in; - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - -XXH_errorcode -XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) -{ - XXH_istate64_t* state = (XXH_istate64_t*)state_in; - state->seed = seed; - state->v1 = seed + PRIME64_1 + PRIME64_2; - state->v2 = seed + PRIME64_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME64_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - -FORCE_INLINE XXH_errorcode -XXH32_update_endian( - XXH32_state_t* state_in, - const void* input, - size_t len, - XXH_endianess endian) -{ - XXH_istate32_t* state = (XXH_istate32_t*)state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input == NULL) - return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) // fill in tmp buffer - { - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy( - (BYTE*)(state->mem32) + state->memsize, input, 16 - state->memsize); - { - const U32* p32 = state->mem32; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v1 = XXH_rotl32(state->v1, 13); - state->v1 *= PRIME32_1; - p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v2 = XXH_rotl32(state->v2, 13); - state->v2 *= PRIME32_1; - p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v3 = XXH_rotl32(state->v3, 13); - state->v3 *= PRIME32_1; - p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v4 = XXH_rotl32(state->v4, 13); - state->v4 *= PRIME32_1; - p32++; - } - p += 16 - state->memsize; - state->memsize = 0; - } - - if (p <= bEnd - 16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do - { - v1 += XXH_readLE32(p, endian) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p += 4; - v2 += XXH_readLE32(p, endian) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p += 4; - v3 += XXH_readLE32(p, endian) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p += 4; - v4 += XXH_readLE32(p, endian) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p += 4; - } while (p <= limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->mem32, p, bEnd - p); - state->memsize = (int)(bEnd - p); - } - - return XXH_OK; -} - -XXH_errorcode -XXH32_update(XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - -FORCE_INLINE U32 -XXH32_digest_endian(const XXH32_state_t* state_in, XXH_endianess endian) -{ - XXH_istate32_t* state = (XXH_istate32_t*)state_in; - const BYTE* p = (const BYTE*)state->mem32; - BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->total_len >= 16) - { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + - XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32)state->total_len; - - while (p + 4 <= bEnd) - { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p += 4; - } - - while (p < bEnd) - { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - -U32 -XXH32_digest(const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - -FORCE_INLINE XXH_errorcode -XXH64_update_endian( - XXH64_state_t* state_in, - const void* input, - size_t len, - XXH_endianess endian) -{ - XXH_istate64_t* state = (XXH_istate64_t*)state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input == NULL) - return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) // fill in tmp buffer - { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy( - ((BYTE*)state->mem64) + state->memsize, input, 32 - state->memsize); - { - const U64* p64 = state->mem64; - state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v1 = XXH_rotl64(state->v1, 31); - state->v1 *= PRIME64_1; - p64++; - state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v2 = XXH_rotl64(state->v2, 31); - state->v2 *= PRIME64_1; - p64++; - state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v3 = XXH_rotl64(state->v3, 31); - state->v3 *= PRIME64_1; - p64++; - state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v4 = XXH_rotl64(state->v4, 31); - state->v4 *= PRIME64_1; - p64++; - } - p += 32 - state->memsize; - state->memsize = 0; - } - - if (p + 32 <= bEnd) - { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do - { - v1 += XXH_readLE64(p, endian) * PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - p += 8; - v2 += XXH_readLE64(p, endian) * PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - p += 8; - v3 += XXH_readLE64(p, endian) * PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - p += 8; - v4 += XXH_readLE64(p, endian) * PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - p += 8; - } while (p <= limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->mem64, p, bEnd - p); - state->memsize = (int)(bEnd - p); - } - - return XXH_OK; -} - -XXH_errorcode -XXH64_update(XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - -FORCE_INLINE U64 -XXH64_digest_endian(const XXH64_state_t* state_in, XXH_endianess endian) -{ - XXH_istate64_t* state = (XXH_istate64_t*)state_in; - const BYTE* p = (const BYTE*)state->mem64; - BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) - { - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + - XXH_rotl64(v4, 18); - - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64 * PRIME64_1 + PRIME64_4; - } - else - { - h64 = state->seed + PRIME64_5; - } - - h64 += (U64)state->total_len; - - while (p + 8 <= bEnd) - { - U64 k1 = XXH_readLE64(p, endian); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = XXH_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; - p += 8; - } - - if (p + 4 <= bEnd) - { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p += 4; - } - - while (p < bEnd) - { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - -unsigned long long -XXH64_digest(const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - -} // namespace detail -} // namespace beast diff --git a/src/ripple/beast/hash/impl/xxhash.h b/src/ripple/beast/hash/impl/xxhash.h deleted file mode 100644 index 063838a829a..00000000000 --- a/src/ripple/beast/hash/impl/xxhash.h +++ /dev/null @@ -1,170 +0,0 @@ -/* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo -@3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. -*/ - -#ifndef BEAST_HASH_XXHASH_H_INCLUDED -#define BEAST_HASH_XXHASH_H_INCLUDED - -/***************************** - Includes -*****************************/ -#include /* size_t */ - -namespace beast { -namespace detail { - -/***************************** - Type -*****************************/ -typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; - -/***************************** - Simple Hash Functions -*****************************/ - -unsigned int -XXH32(const void* input, size_t length, unsigned seed); -unsigned long long -XXH64(const void* input, size_t length, unsigned long long seed); - -/* -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory -address "input". The memory between input & input+length must be valid -(allocated and read-accessible). "seed" can be used to alter the result -predictably. This function successfully passes all SMHasher tests. Speed on Core -2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s XXH64() : Calculate -the 64-bits hash of sequence of length "len" stored at memory address "input". -*/ - -/***************************** - Advanced Hash Functions -*****************************/ -typedef struct -{ - long long ll[6]; -} XXH32_state_t; -typedef struct -{ - long long ll[11]; -} XXH64_state_t; - -/* -These structures allow static allocation of XXH states. -States must then be initialized using XXHnn_reset() before first use. - -If you prefer dynamic allocation, please refer to functions below. -*/ - -XXH32_state_t* -XXH32_createState(void); -XXH_errorcode -XXH32_freeState(XXH32_state_t* statePtr); - -XXH64_state_t* -XXH64_createState(void); -XXH_errorcode -XXH64_freeState(XXH64_state_t* statePtr); - -/* -These functions create and release memory for XXH state. -States must then be initialized using XXHnn_reset() before first use. -*/ - -XXH_errorcode -XXH32_reset(XXH32_state_t* statePtr, unsigned seed); -XXH_errorcode -XXH32_update(XXH32_state_t* statePtr, const void* input, size_t length); -unsigned int -XXH32_digest(const XXH32_state_t* statePtr); - -XXH_errorcode -XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed); -XXH_errorcode -XXH64_update(XXH64_state_t* statePtr, const void* input, size_t length); -unsigned long long -XXH64_digest(const XXH64_state_t* statePtr); - -/* -These functions calculate the xxHash of an input provided in multiple smaller -packets, as opposed to an input provided as a single block. - -XXH state space must first be allocated, using either static or dynamic method -provided above. - -Start a new hash by initializing state with a seed, using XXHnn_reset(). - -Then, feed the hash state by calling XXHnn_update() as many times as necessary. -Obviously, input must be valid, meaning allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value -meaning there is an error. - -Finally, you can produce a hash anytime, by using XXHnn_digest(). -This function returns the final nn-bits hash. -You can nonetheless continue feeding the hash state with more input, -and therefore get some new hashes, by calling again XXHnn_digest(). - -When you are done, don't forget to free XXH state space, using typically -XXHnn_freeState(). -*/ - -} // namespace detail -} // namespace beast - -#endif diff --git a/src/ripple/beast/hash/xxhasher.h b/src/ripple/beast/hash/xxhasher.h index 512c451a051..1a6fc502321 100644 --- a/src/ripple/beast/hash/xxhasher.h +++ b/src/ripple/beast/hash/xxhasher.h @@ -20,9 +20,11 @@ #ifndef BEAST_HASH_XXHASHER_H_INCLUDED #define BEAST_HASH_XXHASHER_H_INCLUDED -#include #include +#include + #include +#include #include namespace beast { @@ -33,16 +35,35 @@ class xxhasher // requires 64-bit std::size_t static_assert(sizeof(std::size_t) == 8, ""); - detail::XXH64_state_t state_; + XXH3_state_t* state_; + + static XXH3_state_t* + allocState() + { + auto ret = XXH3_createState(); + if (ret == nullptr) + throw std::bad_alloc(); + return ret; + } public: using result_type = std::size_t; static constexpr auto const endian = boost::endian::order::native; - xxhasher() noexcept + xxhasher(xxhasher const&) = delete; + xxhasher& + operator=(xxhasher const&) = delete; + + xxhasher() + { + state_ = allocState(); + XXH3_64bits_reset(state_); + } + + ~xxhasher() noexcept { - detail::XXH64_reset(&state_, 1); + XXH3_freeState(state_); } template < @@ -50,7 +71,8 @@ class xxhasher std::enable_if_t::value>* = nullptr> explicit xxhasher(Seed seed) { - detail::XXH64_reset(&state_, seed); + state_ = allocState(); + XXH3_64bits_reset_withSeed(state_, seed); } template < @@ -58,18 +80,19 @@ class xxhasher std::enable_if_t::value>* = nullptr> xxhasher(Seed seed, Seed) { - detail::XXH64_reset(&state_, seed); + state_ = allocState(); + XXH3_64bits_reset_withSeed(state_, seed); } void operator()(void const* key, std::size_t len) noexcept { - detail::XXH64_update(&state_, key, len); + XXH3_64bits_update(state_, key, len); } explicit operator std::size_t() noexcept { - return detail::XXH64_digest(&state_); + return XXH3_64bits_digest(state_); } }; diff --git a/src/ripple/beast/rfc2616.h b/src/ripple/beast/rfc2616.h index 5aff5526a89..7f96e924eb5 100644 --- a/src/ripple/beast/rfc2616.h +++ b/src/ripple/beast/rfc2616.h @@ -20,11 +20,16 @@ #ifndef BEAST_RFC2616_HPP #define BEAST_RFC2616_HPP +// TODO: This include is a workaround for beast compilation bug. +// Remove when fix https://github.com/boostorg/beast/pull/2682/ is available. +#include + #include #include #include #include #include + #include #include #include diff --git a/src/ripple/consensus/Consensus.cpp b/src/ripple/consensus/Consensus.cpp index c40bd1294e7..cc1f84270e7 100644 --- a/src/ripple/consensus/Consensus.cpp +++ b/src/ripple/consensus/Consensus.cpp @@ -32,18 +32,17 @@ shouldCloseLedger( std::chrono::milliseconds timeSincePrevClose, // Time since last ledger's close time std::chrono::milliseconds openTime, // Time waiting to close this ledger - std::optional validationDelay, std::chrono::milliseconds idleInterval, ConsensusParms const& parms, beast::Journal j) { using namespace std::chrono_literals; - if ((prevRoundTime < -1s) || (prevRoundTime > 10min) || (timeSincePrevClose > 10min)) { // These are unexpected cases, we just close the ledger - JLOG(j.warn()) << "Trans=" << (anyTransactions ? "yes" : "no") + JLOG(j.warn()) << "shouldCloseLedger Trans=" + << (anyTransactions ? "yes" : "no") << " Prop: " << prevProposers << "/" << proposersClosed << " Secs: " << timeSincePrevClose.count() << " (last: " << prevRoundTime.count() << ")"; @@ -57,12 +56,6 @@ shouldCloseLedger( return true; } - // The openTime is the time spent so far waiting to close the ledger. - // Any time spent retrying ledger validation in the previous round is - // also counted. - if (validationDelay) - openTime += *validationDelay; - if (!anyTransactions) { // Only close at the end of the idle interval @@ -94,11 +87,24 @@ checkConsensusReached( std::size_t agreeing, std::size_t total, bool count_self, - std::size_t minConsensusPct) + std::size_t minConsensusPct, + bool reachedMax) { - // If we are alone, we have a consensus + // If we are alone for too long, we have consensus. + // Delaying consensus like this avoids a circumstance where a peer + // gets ahead of proposers insofar as it has not received any proposals. + // This could happen if there's a slowdown in receiving proposals. Reaching + // consensus prematurely in this way means that the peer will likely desync. + // The check for reachedMax should allow plenty of time for proposals to + // arrive, and there should be no downside. If a peer is truly not + // receiving any proposals, then there should be no hurry. There's + // really nowhere to go. if (total == 0) - return true; + { + if (reachedMax) + return true; + return false; + } if (count_self) { @@ -127,7 +133,16 @@ checkConsensus( << prevProposers << " agree=" << currentAgree << " validated=" << currentFinished << " time=" << currentAgreeTime.count() << "/" - << previousAgreeTime.count(); + << previousAgreeTime.count() << " proposing? " << proposing + << " minimum duration to reach consensus: " + << parms.ledgerMIN_CONSENSUS.count() << "ms" + << " max consensus time " + << parms.ledgerMAX_CONSENSUS.count() << "s" + << " minimum consensus percentage: " + << parms.minCONSENSUS_PCT; + + if (currentAgreeTime <= parms.ledgerMIN_CONSENSUS) + return ConsensusState::No; if (currentProposers < (prevProposers * 3 / 4)) { @@ -143,7 +158,11 @@ checkConsensus( // Have we, together with the nodes on our UNL list, reached the threshold // to declare consensus? if (checkConsensusReached( - currentAgree, currentProposers, proposing, parms.minCONSENSUS_PCT)) + currentAgree, + currentProposers, + proposing, + parms.minCONSENSUS_PCT, + currentAgreeTime > parms.ledgerMAX_CONSENSUS)) { JLOG(j.debug()) << "normal consensus"; return ConsensusState::Yes; @@ -152,14 +171,18 @@ checkConsensus( // Have sufficient nodes on our UNL list moved on and reached the threshold // to declare consensus? if (checkConsensusReached( - currentFinished, currentProposers, false, parms.minCONSENSUS_PCT)) + currentFinished, + currentProposers, + false, + parms.minCONSENSUS_PCT, + currentAgreeTime > parms.ledgerMAX_CONSENSUS)) { JLOG(j.warn()) << "We see no consensus, but 80% of nodes have moved on"; return ConsensusState::MovedOn; } // no consensus yet - JLOG(j.trace()) << "checkConsensus no consensus"; + JLOG(j.trace()) << "no consensus"; return ConsensusState::No; } diff --git a/src/ripple/consensus/Consensus.h b/src/ripple/consensus/Consensus.h index 98742e41fc0..248bbdc4a1b 100644 --- a/src/ripple/consensus/Consensus.h +++ b/src/ripple/consensus/Consensus.h @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -30,12 +29,10 @@ #include #include #include -#include #include #include #include -#include #include #include @@ -55,7 +52,6 @@ namespace ripple { @param timeSincePrevClose time since the previous ledger's (possibly rounded) close time @param openTime duration this ledger has been open - @param validationDelay duration retrying ledger validation @param idleInterval the network's desired idle interval @param parms Consensus constant parameters @param j journal for logging @@ -69,7 +65,6 @@ shouldCloseLedger( std::chrono::milliseconds prevRoundTime, std::chrono::milliseconds timeSincePrevClose, std::chrono::milliseconds openTime, - std::optional validationDelay, std::chrono::milliseconds idleInterval, ConsensusParms const& parms, beast::Journal j); @@ -122,20 +117,9 @@ checkConsensus( reached consensus with its peers on which transactions to include. It transitions to the `Accept` phase. In this phase, the node works on applying the transactions to the prior ledger to generate a new closed - ledger. - - Try to avoid advancing to a new ledger that hasn't been validated. - One scenario that causes this is if we came to consensus on a - transaction set as other peers were updating their proposals, but - we haven't received the updated proposals. This could cause the rest - of the network to settle on a different transaction set. - As a validator, it is necessary to first build a new ledger and - send a validation for it. Otherwise it's impossible to know for sure - whether or not the ledger would be validated--we can't otherwise - know the ledger hash. If this ledger does become validated, then - proceed with book-keeping and make a call to `startRound` to start - the cycle again. If it doesn't become validated, pause, check - if there is a better transaction set, and try again. + ledger. Once the new ledger is completed, the node shares the validated + ledger with the network, does some book-keeping, then makes a call to + `startRound` to start the cycle again. This class uses a generic interface to allow adapting Consensus for specific applications. The Adaptor template implements a set of helper functions that @@ -263,31 +247,20 @@ checkConsensus( // Called when ledger closes Result onClose(Ledger const &, Ledger const & prev, Mode mode); - // Called after a transaction set is agreed upon to create the new - // ledger and attempt to validate it. - std::pair - buildAndValidate( - Result const& result, - Ledger_t const& prevLedger, - NetClock::duration const& closeResolution, - ConsensusMode const& mode, - Json::Value&& consensusJson); - - // Called when the built ledger is accepted by consensus - void onAccept(Result const& result, - ConsensusCloseTimes const& rawCloseTimes, - ConsensusMode const& mode, - Json::Value&& consensusJson, - std::pair&& txsBuilt); + // Called when ledger is accepted by consensus + void onAccept(Result const & result, + RCLCxLedger const & prevLedger, + NetClock::duration closeResolution, + CloseTimes const & rawCloseTimes, + Mode const & mode); // Called when ledger was forcibly accepted by consensus via the simulate // function. - void onForceAccept(Result const& result, - RCLCxLedger const& prevLedger, - NetClock::duration const& closeResolution, - ConsensusCloseTimes const& rawCloseTimes, - ConsensusMode const& mode, - Json::Value&& consensusJson); + void onForceAccept(Result const & result, + RCLCxLedger const & prevLedger, + NetClock::duration closeResolution, + CloseTimes const & rawCloseTimes, + Mode const & mode); // Propose the position to peers. void propose(ConsensusProposal<...> const & pos); @@ -321,8 +294,7 @@ class Consensus using Proposal_t = ConsensusProposal< NodeID_t, typename Ledger_t::ID, - typename TxSet_t::ID, - typename Ledger_t::Seq>; + typename TxSet_t::ID>; using Result = ConsensusResult; @@ -362,7 +334,7 @@ class Consensus @param adaptor The instance of the adaptor class @param j The journal to log debug output */ - Consensus(clock_type& clock, Adaptor& adaptor, beast::Journal j); + Consensus(clock_type const& clock, Adaptor& adaptor, beast::Journal j); /** Kick-off the next round of consensus. @@ -544,15 +516,8 @@ class Consensus closeLedger(); // Adjust our positions to try to agree with other validators. - /** Adjust our positions to try to agree with other validators. - * - * Share them with the network unless we've already accepted a - * consensus position. - * - * @param share Whether to share with the network. - */ void - updateOurPositions(bool const share); + updateOurPositions(); bool haveConsensus(); @@ -575,6 +540,7 @@ class Consensus NetClock::time_point asCloseTime(NetClock::time_point raw) const; +private: Adaptor& adaptor_; ConsensusPhase phase_{ConsensusPhase::accepted}; @@ -582,7 +548,7 @@ class Consensus bool firstRound_ = true; bool haveCloseTimeConsensus_ = false; - clock_type& clock_; + clock_type const& clock_; // How long the consensus convergence has taken, expressed as // a percentage of the time that we expected it to take. @@ -612,16 +578,8 @@ class Consensus // Last validated ledger seen by consensus Ledger_t previousLedger_; - // Transaction Sets, indexed by hash of transaction tree. - using AcquiredType = beast::aged_unordered_map< - typename TxSet_t::ID, - const TxSet_t, - clock_type::clock_type, - beast::uhash<>>; - AcquiredType acquired_; - - // Tx sets that can be purged only once there is a new consensus round. - std::stack acquiredPurge_; + // Transaction Sets, indexed by hash of transaction tree + hash_map acquired_; std::optional result_; ConsensusCloseTimes rawCloseTimes_; @@ -633,18 +591,8 @@ class Consensus hash_map currPeerPositions_; // Recently received peer positions, available when transitioning between - // ledgers or rounds. Collected by ledger sequence. This allows us to - // know which positions are likely relevant to the ledger on which we are - // currently working. Also allows us to catch up faster if we fall behind - // the rest of the network since we won't need to re-aquire proposals - // and related transaction sets. - std::map> - recentPeerPositions_; - - // These are for peers not using code that adds a ledger sequence - // to the proposal message. TODO This should be removed eventually when - // the network fully upgrades. - hash_map> recentPeerPositionsLegacy_; + // ledgers or rounds + hash_map> recentPeerPositions_; // The number of proposers who participated in the last consensus round std::size_t prevProposers_ = 0; @@ -658,10 +606,10 @@ class Consensus template Consensus::Consensus( - clock_type& clock, + clock_type const& clock, Adaptor& adaptor, beast::Journal journal) - : adaptor_(adaptor), clock_(clock), acquired_(clock), j_{journal} + : adaptor_(adaptor), clock_(clock), j_{journal} { JLOG(j_.debug()) << "Creating consensus object"; } @@ -687,21 +635,8 @@ Consensus::startRound( prevCloseTime_ = rawCloseTimes_.self; } - // Clear positions that we know will not ever be necessary again. - auto it = recentPeerPositions_.begin(); - while (it != recentPeerPositions_.end() && it->first <= prevLedger.seq()) - it = recentPeerPositions_.erase(it); - // Get rid of untrusted positions for the current working ledger. - auto currentPositions = - recentPeerPositions_.find(prevLedger.seq() + typename Ledger_t::Seq{1}); - if (currentPositions != recentPeerPositions_.end()) - { - for (NodeID_t const& n : nowUntrusted) - currentPositions->second.erase(n); - } - for (NodeID_t const& n : nowUntrusted) - recentPeerPositionsLegacy_.erase(n); + recentPeerPositions_.erase(n); ConsensusMode startMode = proposing ? ConsensusMode::proposing : ConsensusMode::observing; @@ -743,29 +678,8 @@ Consensus::startRoundInternal( convergePercent_ = 0; haveCloseTimeConsensus_ = false; openTime_.reset(clock_.now()); - - // beast::aged_unordered_map::erase by key is broken and - // is not used anywhere in the existing codebase. - while (!acquiredPurge_.empty()) - { - auto found = acquired_.find(acquiredPurge_.top()); - if (found != acquired_.end()) - acquired_.erase(found); - acquiredPurge_.pop(); - } - for (auto it = currPeerPositions_.begin(); it != currPeerPositions_.end();) - { - if (auto found = acquired_.find(it->second.proposal().position()); - found != acquired_.end()) - { - acquired_.erase(found); - } - it = currPeerPositions_.erase(it); - } - - // Hold up to 30 minutes worth of acquired tx sets. This to help - // catch up quickly from extended de-sync periods. - beast::expire(acquired_, std::chrono::minutes(30)); + currPeerPositions_.clear(); + acquired_.clear(); rawCloseTimes_.peers.clear(); rawCloseTimes_.self = {}; deadNodes_.clear(); @@ -793,45 +707,14 @@ Consensus::peerProposal( auto const& peerID = newPeerPos.proposal().nodeID(); // Always need to store recent positions - if (newPeerPos.proposal().ledgerSeq().has_value()) - { - // Ignore proposals from prior ledgers. - typename Ledger_t::Seq const& propLedgerSeq = - *newPeerPos.proposal().ledgerSeq(); - if (propLedgerSeq <= previousLedger_.seq()) - return false; - - auto& bySeq = recentPeerPositions_[propLedgerSeq]; - { - auto peerProp = bySeq.find(peerID); - if (peerProp == bySeq.end()) - { - bySeq.emplace(peerID, newPeerPos); - } - else - { - // Only store if it's the latest proposal from this peer for the - // consensus round in the proposal. - if (newPeerPos.proposal().proposeSeq() <= - peerProp->second.proposal().proposeSeq()) - { - return false; - } - peerProp->second = newPeerPos; - } - } - } - else { - // legacy proposal with no ledger sequence - auto& props = recentPeerPositionsLegacy_[peerID]; + auto& props = recentPeerPositions_[peerID]; if (props.size() >= 10) props.pop_front(); props.push_back(newPeerPos); } - return peerProposalInternal(now, newPeerPos); } @@ -841,6 +724,10 @@ Consensus::peerProposalInternal( NetClock::time_point const& now, PeerPosition_t const& newPeerPos) { + // Nothing to do for now if we are currently working on a ledger + if (phase_ == ConsensusPhase::accepted) + return false; + now_ = now; auto const& newPeerProp = newPeerPos.proposal(); @@ -849,20 +736,6 @@ Consensus::peerProposalInternal( { JLOG(j_.debug()) << "Got proposal for " << newPeerProp.prevLedger() << " but we are on " << prevLedgerID_; - - if (!acquired_.count(newPeerProp.position())) - { - // acquireTxSet will return the set if it is available, or - // spawn a request for it and return nullopt/nullptr. It will call - // gotTxSet once it arrives. If we're behind, this should save - // time when we catch up. - if (auto set = adaptor_.acquireTxSet(newPeerProp.position())) - gotTxSet(now_, *set); - else - JLOG(j_.debug()) << "Do not have tx set for peer"; - } - - // There's nothing else to do with this proposal currently. return false; } @@ -896,45 +769,16 @@ Consensus::peerProposalInternal( it.second.unVote(peerID); } if (peerPosIt != currPeerPositions_.end()) - { - // Remove from acquired_ or else it will consume space for - // awhile. beast::aged_unordered_map::erase by key is broken and - // is not used anywhere in the existing codebase. - if (auto found = - acquired_.find(peerPosIt->second.proposal().position()); - found != acquired_.end()) - { - acquiredPurge_.push( - peerPosIt->second.proposal().position()); - } currPeerPositions_.erase(peerID); - } deadNodes_.insert(peerID); return true; } if (peerPosIt != currPeerPositions_.end()) - { - // Remove from acquired_ or else it will consume space for awhile. - // beast::aged_unordered_container::erase by key is broken and - // is not used anywhere in the existing codebase. - if (auto found = acquired_.find(newPeerPos.proposal().position()); - found != acquired_.end()) - { - acquiredPurge_.push(newPeerPos.proposal().position()); - } - // The proposal's arrival time determines how long the network - // has been proposing, so new proposals from the same peer - // should reflect the original's arrival time. - newPeerPos.proposal().arrivalTime() = - peerPosIt->second.proposal().arrivalTime(); peerPosIt->second = newPeerPos; - } else - { currPeerPositions_.emplace(peerID, newPeerPos); - } } if (newPeerProp.isInitial()) @@ -983,9 +827,13 @@ Consensus::timerEntry(NetClock::time_point const& now) checkLedger(); if (phase_ == ConsensusPhase::open) + { phaseOpen(); + } else if (phase_ == ConsensusPhase::establish) + { phaseEstablish(); + } } template @@ -994,6 +842,10 @@ Consensus::gotTxSet( NetClock::time_point const& now, TxSet_t const& txSet) { + // Nothing to do if we've finished work on a ledger + if (phase_ == ConsensusPhase::accepted) + return; + now_ = now; auto id = txSet.id(); @@ -1173,18 +1025,7 @@ Consensus::handleWrongLedger(typename Ledger_t::ID const& lgrId) result_->compares.clear(); } - for (auto it = currPeerPositions_.begin(); - it != currPeerPositions_.end();) - { - // beast::aged_unordered_map::erase by key is broken and - // is not used anywhere in the existing codebase. - if (auto found = acquired_.find(it->second.proposal().position()); - found != acquired_.end()) - { - acquiredPurge_.push(it->second.proposal().position()); - } - it = currPeerPositions_.erase(it); - } + currPeerPositions_.clear(); rawCloseTimes_.peers.clear(); deadNodes_.clear(); @@ -1235,30 +1076,7 @@ template void Consensus::playbackProposals() { - // Only use proposals for the ledger sequence we're currently working on. - auto const currentPositions = recentPeerPositions_.find( - previousLedger_.seq() + typename Ledger_t::Seq{1}); - if (currentPositions != recentPeerPositions_.end()) - { - for (auto const& [peerID, pos] : currentPositions->second) - { - if (pos.proposal().prevLedger() == prevLedgerID_ && - peerProposalInternal(now_, pos)) - { - adaptor_.share(pos); - } - } - } - - // It's safe to do this--if a proposal is based on the wrong ledger, - // then peerProposalInternal() will not replace it in currPeerPositions_. - // TODO Eventually, remove code to check for non-existent ledger sequence - // in peer proposal messages and make that parameter required in - // the protobuf definition. Do this only after the network is running on - // rippled versions with that parameter set in peer proposals. This - // can be done once an amendment for another feature forces that kind - // of upgrade, but this particular feature does not require an amendment. - for (auto const& it : recentPeerPositionsLegacy_) + for (auto const& it : recentPeerPositions_) { for (auto const& pos : it.second) { @@ -1316,13 +1134,11 @@ Consensus::phaseOpen() prevRoundTime_, sinceClose, openTime_.read(), - adaptor_.getValidationDelay(), idleInterval, adaptor_.parms(), j_)) { closeLedger(); - adaptor_.setValidationDelay(); } } @@ -1341,7 +1157,7 @@ Consensus::shouldPause() const std::size_t const offline = trustedKeys.size(); std::stringstream vars; - vars << " (working seq: " << previousLedger_.seq() << ", " + vars << " consensuslog (working seq: " << previousLedger_.seq() << ", " << "validated seq: " << adaptor_.getValidLedgerIndex() << ", " << "am validator: " << adaptor_.validator() << ", " << "have validated: " << adaptor_.haveValidated() << ", " @@ -1456,52 +1272,11 @@ Consensus::phaseEstablish() convergePercent_ = result_->roundTime.read() * 100 / std::max(prevRoundTime_, parms.avMIN_CONSENSUS_TIME); - { - // Give everyone a chance to take an initial position unless enough - // have already submitted theirs a long enough time ago - // --because that means we're already - // behind. Optimize pause duration if pausing. Pause until exactly - // the number of ms after roundTime.read(), or the time since - // receiving the earliest qualifying peer proposal. To protect - // from faulty peers on the UNL, discard the earliest proposals - // beyond the quorum threshold. For example, with a UNL of 20, - // 80% quorum is 16. Assume the remaining 4 are Byzantine actors. - // We therefore ignore the first 4 proposals received - // for this calculation. We then take the earliest of either the - // 5th proposal or our own proposal to determine whether enough - // time has passed to possibly close. If not, then use that to - // precisely determine how long to pause until checking again. - std::size_t const q = adaptor_.quorum(); - std::size_t const discard = - static_cast(q / parms.minCONSENSUS_FACTOR) - q; - - std::chrono::milliseconds beginning; - if (currPeerPositions_.size() > discard) - { - std::multiset arrivals; - for (auto& pos : currPeerPositions_) - { - pos.second.proposal().arrivalTime().tick(clock_.now()); - arrivals.insert(pos.second.proposal().arrivalTime().read()); - } - auto it = arrivals.rbegin(); - std::advance(it, discard); - beginning = *it; - } - else - { - beginning = result_->roundTime.read(); - } - - // Give everyone a chance to take an initial position - if (beginning < parms.ledgerMIN_CONSENSUS) - { - adaptor_.setTimerDelay(parms.ledgerMIN_CONSENSUS - beginning); - return; - } - } + // Give everyone a chance to take an initial position + if (result_->roundTime.read() < parms.ledgerMIN_CONSENSUS) + return; - updateOurPositions(true); + updateOurPositions(); // Nothing to do if too many laggards or we don't have consensus. if (shouldPause() || !haveConsensus()) @@ -1520,96 +1295,13 @@ Consensus::phaseEstablish() prevRoundTime_ = result_->roundTime.read(); phase_ = ConsensusPhase::accepted; JLOG(j_.debug()) << "transitioned to ConsensusPhase::accepted"; - - std::optional> - txsBuilt; - // Track time spent retrying new ledger validation. - std::optional> - startDelay; - // Amount of time to pause checking for ledger to become validated. - static auto const validationWait = std::chrono::milliseconds(100); - - // Make a copy of the result_ because it may be reset during the accept - // phase if ledgers are switched and a new round is started. - assert(result_.has_value()); - std::optional result{result_}; - // Building the new ledger is time-consuming and safe to not lock, but - // the rest of the logic below needs to be locked, until - // finishing (onAccept). - std::unique_lock lock(adaptor_.peekMutex()); - do - { - if (!result_.has_value() || - result_->position.prevLedger() != result->position.prevLedger()) - { - JLOG(j_.debug()) << "A new consensus round has started based on " - "a different ledger."; - return; - } - if (txsBuilt) - { - if (!startDelay) - startDelay = std::chrono::steady_clock::now(); - - // Only send a single validation per round. - adaptor_.clearValidating(); - // Check if a better proposal has been shared by the network. - auto prevProposal = result_->position; - updateOurPositions(false); - - if (prevProposal == result_->position) - { - JLOG(j_.debug()) - << "old and new positions " - "match: " - << prevProposal.position() << " delay so far " - << std::chrono::duration_cast( - std::chrono::steady_clock::now() - *startDelay) - .count() - << "ms. pausing"; - adaptor_.getLedgerMaster().waitForValidated(validationWait); - continue; - } - JLOG(j_.debug()) << "retrying buildAndValidate with " - "new position: " - << result_->position.position(); - // Update the result used for the remainder of this Consensus round. - assert(result_.has_value()); - result.emplace(*result_); - } - lock.unlock(); - - // This is time-consuming and safe to not have under mutex. - assert(result.has_value()); - txsBuilt = adaptor_.buildAndValidate( - *result, - previousLedger_, - closeResolution_, - mode_.get(), - getJson(true)); - lock.lock(); - } while (adaptor_.retryAccept(txsBuilt->second, startDelay)); - - if (startDelay) - { - auto const delay = - std::chrono::duration_cast( - std::chrono::steady_clock::now() - *startDelay); - JLOG(j_.debug()) << "validationDelay will be " << delay.count() << "ms"; - adaptor_.setValidationDelay(delay); - } - - lock.unlock(); - - assert(result.has_value()); adaptor_.onAccept( - *result, + *result_, + previousLedger_, + closeResolution_, rawCloseTimes_, mode_.get(), - getJson(true), - std::move(*txsBuilt)); + getJson(true)); } template @@ -1623,8 +1315,7 @@ Consensus::closeLedger() JLOG(j_.debug()) << "transitioned to ConsensusPhase::establish"; rawCloseTimes_.self = now_; - result_.emplace( - adaptor_.onClose(previousLedger_, now_, mode_.get(), clock_)); + result_.emplace(adaptor_.onClose(previousLedger_, now_, mode_.get())); result_->roundTime.reset(clock_.now()); // Share the newly created transaction set if we haven't already // received it from a peer @@ -1640,11 +1331,10 @@ Consensus::closeLedger() auto const& pos = pit.second.proposal().position(); auto const it = acquired_.find(pos); if (it != acquired_.end()) + { createDisputes(it->second); + } } - // There's no reason to pause, especially if we have fallen behind and - // can possible agree to a consensus proposal already. - timerEntry(now_); } /** How many of the participants must agree to reach a given threshold? @@ -1669,7 +1359,7 @@ participantsNeeded(int participants, int percent) template void -Consensus::updateOurPositions(bool const share) +Consensus::updateOurPositions() { // We must have a position if we are updating it assert(result_); @@ -1693,14 +1383,6 @@ Consensus::updateOurPositions(bool const share) JLOG(j_.warn()) << "Removing stale proposal from " << peerID; for (auto& dt : result_->disputes) dt.second.unVote(peerID); - // Remove from acquired_ or else it will consume space for - // awhile. beast::aged_unordered_map::erase by key is broken and - // is not used anywhere in the existing codebase. - if (auto found = acquired_.find(peerProp.position()); - found != acquired_.end()) - { - acquiredPurge_.push(peerProp.position()); - } it = currPeerPositions_.erase(it); } else @@ -1759,27 +1441,15 @@ Consensus::updateOurPositions(bool const share) else { int neededWeight; - // It's possible to be at a close time impasse (described below), so - // keep track of whether this round has taken a long time. - bool stuck = false; if (convergePercent_ < parms.avMID_CONSENSUS_TIME) - { neededWeight = parms.avINIT_CONSENSUS_PCT; - } else if (convergePercent_ < parms.avLATE_CONSENSUS_TIME) - { neededWeight = parms.avMID_CONSENSUS_PCT; - } else if (convergePercent_ < parms.avSTUCK_CONSENSUS_TIME) - { neededWeight = parms.avLATE_CONSENSUS_PCT; - } else - { neededWeight = parms.avSTUCK_CONSENSUS_PCT; - stuck = true; - } int participants = currPeerPositions_.size(); if (mode_.get() == ConsensusMode::proposing) @@ -1799,156 +1469,34 @@ Consensus::updateOurPositions(bool const share) << " nw:" << neededWeight << " thrV:" << threshVote << " thrC:" << threshConsensus; - // Choose a close time and decide whether there is consensus for it. - // - // Close time is chosen based on the threshVote threshold - // calculated above. If a close time has votes equal to or greater than - // that threshold, then that is the best close time. If multiple - // close times have an equal number of votes, then choose the greatest - // of them. Ensure that our close time then matches that which meets - // the criteria. But if no close time meet the criteria, make no - // changes. - // - // This is implemented slightly differently for validators vs - // non-validators. For non-validators, it is sufficient to merely - // count the close times from all peer positions to determine - // the best. Validators, however, risk putting the network into an - // impasse unless they are able to change their own position without - // first having counted it towards the close time totals. - // - // Here's how the impasse could occur: - // Imagine 5 validators. 3 have close time t1, and 2 have t2. - // As consensus time increases, the threshVote threshold also increases. - // Once threshVote exceeds 60%, no members of either set of validators - // will change their close times. - // - // Avoiding the impasse means that validators should identify - // whether they currently have the best close time. First, choose - // the close time with the most votes. However, if multiple close times - // have the same number of votes, pick the latest of them. - // If the validator does not currently have the best close time, - // switch to it and increase the local vote tally for that better - // close time. This will result in consensus in the next iteration - // assuming that the peer messages propagate successfully. - // In this case the validators in set t1 will remain the same but - // those in t2 switch to t1. - // - // Another wrinkle, however, is that too many position changes - // from validators also has a destabilizing affect. Consensus can - // take longer if peers have to keep re-calculating positions, - // and mistakes can be made if peers settle on the wrong position. - // Therefore, avoiding an impasse should also minimize the likelihood - // of gratuitous change of position. - // - // The solution for validators is to first track whether it's - // possible that the network is at an impasse based on how much - // time this current consensus round has taken. This is reflected - // in the "stuck" boolean. When stuck, validators perform the - // above-described position change based solely on whether or not - // they agree with the best position, and ignore the threshVote - // criteria used for the earlier part of the phase. - // - // Determining whether there is close time consensus is very simple - // in comparison: if votes for the best close time meet or exceed - // threshConsensus, then we have close time consensus. Otherwise, not. - - // First, find the best close time with which to agree: first criteria - // is the close time with the most votes. If a tie, the latest - // close time of those tied for the maximum number of votes. - std::multimap votesByCloseTime; - std::stringstream ss; - ss << "Close time calculation for ledger sequence " - << static_cast(previousLedger_.seq()) + 1 - << " Close times and vote count are as follows: "; - bool first = true; - for (auto const& [closeTime, voteCount] : closeTimeVotes) - { - if (first) - first = false; - else - ss << ','; - votesByCloseTime.insert({voteCount, closeTime}); - ss << closeTime.time_since_epoch().count() << ':' << voteCount; - } - // These always gets populated because currPeerPositions_ is not - // empty to end up here, so at least 1 close time has at least 1 vote. - assert(!currPeerPositions_.empty()); - std::optional maxVote; - std::set maxCloseTimes; - // Highest vote getter is last. Track each close time that is tied - // with the highest. - for (auto rit = votesByCloseTime.crbegin(); - rit != votesByCloseTime.crend(); - ++rit) - { - int const voteCount = rit->first; - if (!maxVote.has_value()) - maxVote = voteCount; - else if (voteCount < *maxVote) - break; - maxCloseTimes.insert(rit->second); - } - // The best close time is the latest close time of those that have - // the maximum number of votes. - NetClock::time_point const bestCloseTime = *maxCloseTimes.crbegin(); - ss << ". The best close time has the most votes. If there is a tie, " - "choose the latest. This is " - << bestCloseTime.time_since_epoch().count() << "with " << *maxVote - << " votes. "; - - // If we are a validator potentially at an impasse and our own close - // time is not the best, change our close time to match it and - // tally another vote for it. - if (adaptor_.validating() && stuck && - consensusCloseTime != bestCloseTime) - { - consensusCloseTime = bestCloseTime; - ++*maxVote; - ss << " We are a validator. Consensus has taken " - << result_->roundTime.read().count() - << "ms. Previous round " - "took " - << prevRoundTime_.count() - << "ms. Now changing our " - "close time to " - << bestCloseTime.time_since_epoch().count() - << " that " - "now has " - << *maxVote << " votes."; - } - // If the close time with the most votes also meets or exceeds the - // threshold to change our position, then change our position. - // Then check if we have met or exceeded the threshold for close - // time consensus. - // - // The 2nd check has been nested within the first historically. - // It's possible that this can be optimized by doing the - // 2nd check independently--this may make consensus happen faster in - // some cases. Then again, the trade-offs have not been modelled. - if (*maxVote >= threshVote) + for (auto const& [t, v] : closeTimeVotes) { - consensusCloseTime = bestCloseTime; - ss << "Close time " << bestCloseTime.time_since_epoch().count() - << " has " << *maxVote << " votes, which is >= the threshold (" - << threshVote - << " to make that our position if it isn't already."; - if (*maxVote >= threshConsensus) + JLOG(j_.debug()) + << "CCTime: seq " + << static_cast(previousLedger_.seq()) + 1 << ": " + << t.time_since_epoch().count() << " has " << v << ", " + << threshVote << " required"; + + if (v >= threshVote) { - haveCloseTimeConsensus_ = true; - ss << " The maximum votes also >= the threshold (" - << threshConsensus << ") for consensus."; + // A close time has enough votes for us to try to agree + consensusCloseTime = t; + threshVote = v; + + if (threshVote >= threshConsensus) + haveCloseTimeConsensus_ = true; } } if (!haveCloseTimeConsensus_) { - ss << " No CT consensus:" - << " Proposers:" << currPeerPositions_.size() - << " Mode:" << to_string(mode_.get()) - << " Thresh:" << threshConsensus - << " Pos:" << consensusCloseTime.time_since_epoch().count(); + JLOG(j_.debug()) + << "No CT consensus:" + << " Proposers:" << currPeerPositions_.size() + << " Mode:" << to_string(mode_.get()) + << " Thresh:" << threshConsensus + << " Pos:" << consensusCloseTime.time_since_epoch().count(); } - JLOG(j_.debug()) << ss.str(); } if (!ourNewSet && @@ -1972,10 +1520,8 @@ Consensus::updateOurPositions(bool const share) result_->position.changePosition(newID, consensusCloseTime, now_); // Share our new transaction set and update disputes - // if we haven't already received it. Unless we have already - // accepted a position, but are recalculating because it didn't - // validate. - if (acquired_.emplace(newID, result_->txns).second && share) + // if we haven't already received it + if (acquired_.emplace(newID, result_->txns).second) { if (!result_->position.isBowOut()) adaptor_.share(result_->txns); @@ -1988,11 +1534,9 @@ Consensus::updateOurPositions(bool const share) } } - // Share our new position if we are still participating this round, - // unless we have already accepted a position but are recalculating - // because it didn't validate. + // Share our new position if we are still participating this round if (!result_->position.isBowOut() && - (mode_.get() == ConsensusMode::proposing) && share) + (mode_.get() == ConsensusMode::proposing)) adaptor_.propose(result_->position); } } @@ -2014,9 +1558,14 @@ Consensus::haveConsensus() { Proposal_t const& peerProp = peerPos.proposal(); if (peerProp.position() == ourPosition) + { ++agree; + } else + { + JLOG(j_.debug()) << nodeId << " has " << peerProp.position(); ++disagree; + } } auto currentFinished = adaptor_.proposersFinished(previousLedger_, prevLedgerID_); @@ -2043,8 +1592,8 @@ Consensus::haveConsensus() // without us. if (result_->state == ConsensusState::MovedOn) { - JLOG(j_.error()) << "Unable to reach consensus MovedOn: " - << Json::Compact{getJson(true)}; + JLOG(j_.error()) << "Unable to reach consensus"; + JLOG(j_.error()) << Json::Compact{getJson(true)}; } return true; @@ -2103,7 +1652,7 @@ Consensus::createDisputes(TxSet_t const& o) if (result_->disputes.find(txID) != result_->disputes.end()) continue; - JLOG(j_.trace()) << "Transaction " << txID << " is disputed"; + JLOG(j_.debug()) << "Transaction " << txID << " is disputed"; typename Result::Dispute_t dtx{ tx, @@ -2123,7 +1672,7 @@ Consensus::createDisputes(TxSet_t const& o) result_->disputes.emplace(txID, std::move(dtx)); } - JLOG(j_.trace()) << dc << " differences found"; + JLOG(j_.debug()) << dc << " differences found"; } template diff --git a/src/ripple/consensus/ConsensusParms.h b/src/ripple/consensus/ConsensusParms.h index 61722e2c439..a0b6c6be8d4 100644 --- a/src/ripple/consensus/ConsensusParms.h +++ b/src/ripple/consensus/ConsensusParms.h @@ -70,16 +70,8 @@ struct ConsensusParms // Consensus durations are relative to the internal Consensus clock and use // millisecond resolution. - //! The percentage threshold and floating point factor above which we can - //! declare consensus. + //! The percentage threshold above which we can declare consensus. std::size_t minCONSENSUS_PCT = 80; - float minCONSENSUS_FACTOR = static_cast(minCONSENSUS_PCT / 100.0f); - - //! The percentage threshold and floating point factor above which we can - //! declare consensus based on nodes having fallen off of the UNL. - std::size_t negUNL_MIN_CONSENSUS_PCT = 60; - float negUNL_MIN_CONSENSUS_FACTOR = - static_cast(negUNL_MIN_CONSENSUS_PCT / 100.0f); //! The duration a ledger may remain idle before closing std::chrono::milliseconds ledgerIDLE_INTERVAL = std::chrono::seconds{15}; @@ -94,7 +86,7 @@ struct ConsensusParms * validators don't appear to be offline that are merely waiting for * laggards. */ - std::chrono::milliseconds ledgerMAX_CONSENSUS = std::chrono::seconds{10}; + std::chrono::milliseconds ledgerMAX_CONSENSUS = std::chrono::seconds{15}; //! Minimum number of seconds to wait to ensure others have computed the LCL std::chrono::milliseconds ledgerMIN_CLOSE = std::chrono::seconds{2}; diff --git a/src/ripple/consensus/ConsensusProposal.h b/src/ripple/consensus/ConsensusProposal.h index 95acb3014a3..c5103cfe0d5 100644 --- a/src/ripple/consensus/ConsensusProposal.h +++ b/src/ripple/consensus/ConsensusProposal.h @@ -21,13 +21,9 @@ #include #include -#include -#include #include #include -#include #include -#include #include #include @@ -55,15 +51,12 @@ namespace ripple { @tparam Position_t Type used to represent the position taken on transactions under consideration during this round of consensus */ -template +template class ConsensusProposal { public: using NodeID = NodeID_t; - //! Clock type for measuring time within the consensus code - using clock_type = beast::abstract_clock; - //< Sequence value when a peer initially joins consensus static std::uint32_t const seqJoin = 0; @@ -78,8 +71,6 @@ class ConsensusProposal @param closeTime Position of when this ledger closed. @param now Time when the proposal was taken. @param nodeID ID of node/peer taking this position. - @param ledgerSeq Ledger sequence of proposal. - @param clock Clock that works with real and test time. */ ConsensusProposal( LedgerID_t const& prevLedger, @@ -87,20 +78,14 @@ class ConsensusProposal Position_t const& position, NetClock::time_point closeTime, NetClock::time_point now, - NodeID_t const& nodeID, - std::optional const& ledgerSeq, - clock_type const& clock) + NodeID_t const& nodeID) : previousLedger_(prevLedger) , position_(position) , closeTime_(closeTime) , time_(now) , proposeSeq_(seq) , nodeID_(nodeID) - , ledgerSeq_(ledgerSeq) { - // Track the arrive time to know how long our peers have been - // sending proposals. - arrivalTime_.reset(clock.now()); } //! Identifying which peer took this position. @@ -247,18 +232,6 @@ class ConsensusProposal return signingHash_.value(); } - std::optional const& - ledgerSeq() const - { - return ledgerSeq_; - } - - ConsensusTimer& - arrivalTime() const - { - return arrivalTime_; - } - private: //! Unique identifier of prior ledger this proposal is based on LedgerID_t previousLedger_; @@ -278,19 +251,15 @@ class ConsensusProposal //! The identifier of the node taking this position NodeID_t nodeID_; - std::optional ledgerSeq_; - //! The signing hash for this proposal mutable std::optional signingHash_; - - mutable ConsensusTimer arrivalTime_; }; -template +template bool operator==( - ConsensusProposal const& a, - ConsensusProposal const& b) + ConsensusProposal const& a, + ConsensusProposal const& b) { return a.nodeID() == b.nodeID() && a.proposeSeq() == b.proposeSeq() && a.prevLedger() == b.prevLedger() && a.position() == b.position() && diff --git a/src/ripple/consensus/ConsensusTypes.h b/src/ripple/consensus/ConsensusTypes.h index 42c0b9561a5..05d03c8a9c6 100644 --- a/src/ripple/consensus/ConsensusTypes.h +++ b/src/ripple/consensus/ConsensusTypes.h @@ -21,6 +21,7 @@ #define RIPPLE_CONSENSUS_CONSENSUS_TYPES_H_INCLUDED #include +#include #include #include #include @@ -188,8 +189,6 @@ enum class ConsensusState { Yes //!< We have consensus along with the network }; -template -class ConsensusProposal; /** Encapsulates the result of consensus. Stores all relevant data for the outcome of consensus on a single @@ -209,8 +208,7 @@ struct ConsensusResult using Proposal_t = ConsensusProposal< NodeID_t, typename Ledger_t::ID, - typename TxSet_t::ID, - typename Ledger_t::Seq>; + typename TxSet_t::ID>; using Dispute_t = DisputedTx; ConsensusResult(TxSet_t&& s, Proposal_t&& p) diff --git a/src/ripple/consensus/DisputedTx.h b/src/ripple/consensus/DisputedTx.h index 92d9917145d..ae127197eec 100644 --- a/src/ripple/consensus/DisputedTx.h +++ b/src/ripple/consensus/DisputedTx.h @@ -152,19 +152,19 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) { if (votesYes) { - JLOG(j_.trace()) << "Peer " << peer << " votes YES on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " votes YES on " << tx_.id(); ++yays_; } else { - JLOG(j_.trace()) << "Peer " << peer << " votes NO on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " votes NO on " << tx_.id(); ++nays_; } } // changes vote to yes else if (votesYes && !it->second) { - JLOG(j_.trace()) << "Peer " << peer << " now votes YES on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " now votes YES on " << tx_.id(); --nays_; ++yays_; it->second = true; @@ -172,7 +172,7 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) // changes vote to no else if (!votesYes && it->second) { - JLOG(j_.trace()) << "Peer " << peer << " now votes NO on " << tx_.id(); + JLOG(j_.debug()) << "Peer " << peer << " now votes NO on " << tx_.id(); ++nays_; --yays_; it->second = false; @@ -238,17 +238,17 @@ DisputedTx::updateVote( if (newPosition == ourVote_) { - JLOG(j_.trace()) << "No change (" << (ourVote_ ? "YES" : "NO") - << ") : weight " << weight << ", percent " - << percentTime; - JLOG(j_.trace()) << Json::Compact{getJson()}; + JLOG(j_.info()) << "No change (" << (ourVote_ ? "YES" : "NO") + << ") : weight " << weight << ", percent " + << percentTime; + JLOG(j_.debug()) << Json::Compact{getJson()}; return false; } ourVote_ = newPosition; - JLOG(j_.trace()) << "We now vote " << (ourVote_ ? "YES" : "NO") << " on " + JLOG(j_.debug()) << "We now vote " << (ourVote_ ? "YES" : "NO") << " on " << tx_.id(); - JLOG(j_.trace()) << Json::Compact{getJson()}; + JLOG(j_.debug()) << Json::Compact{getJson()}; return true; } diff --git a/src/ripple/consensus/Validations.h b/src/ripple/consensus/Validations.h index a9dbd5585e2..1bef76d961c 100644 --- a/src/ripple/consensus/Validations.h +++ b/src/ripple/consensus/Validations.h @@ -1142,6 +1142,34 @@ class Validations return laggards; } + + std::size_t + sizeOfCurrentCache() const + { + std::lock_guard lock{mutex_}; + return current_.size(); + } + + std::size_t + sizeOfSeqEnforcersCache() const + { + std::lock_guard lock{mutex_}; + return seqEnforcers_.size(); + } + + std::size_t + sizeOfByLedgerCache() const + { + std::lock_guard lock{mutex_}; + return byLedger_.size(); + } + + std::size_t + sizeOfBySequenceCache() const + { + std::lock_guard lock{mutex_}; + return bySequence_.size(); + } }; } // namespace ripple diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index a9acd4c6b2b..cf41678a16c 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -215,7 +215,7 @@ class Config : public BasicConfig // Node storage configuration std::uint32_t LEDGER_HISTORY = 256; - std::uint32_t FETCH_DEPTH = 1'000'000'000; + std::uint32_t FETCH_DEPTH = 1000000000; // Tunable that adjusts various parameters, typically associated // with hardware parameters (RAM size and CPU cores). The default @@ -232,11 +232,10 @@ class Config : public BasicConfig // Enable the experimental Ledger Replay functionality bool LEDGER_REPLAY = false; - // Work queue limits. 10000 transactions is 2 full seconds of slowdown at - // 5000/s. - int MAX_TRANSACTIONS = 10'000; - static constexpr int MAX_JOB_QUEUE_TX = 100'000; - static constexpr int MIN_JOB_QUEUE_TX = 1'000; + // Work queue limits + int MAX_TRANSACTIONS = 250; + static constexpr int MAX_JOB_QUEUE_TX = 1000; + static constexpr int MIN_JOB_QUEUE_TX = 100; // Amendment majority time std::chrono::seconds AMENDMENT_MAJORITY_TIME = defaultAmendmentMajorityTime; diff --git a/src/ripple/core/ConfigSections.h b/src/ripple/core/ConfigSections.h index 03c702f9a52..b4e460f1cfc 100644 --- a/src/ripple/core/ConfigSections.h +++ b/src/ripple/core/ConfigSections.h @@ -49,6 +49,7 @@ struct ConfigSection // VFALCO TODO Rename and replace these macros with variables. #define SECTION_AMENDMENTS "amendments" #define SECTION_AMENDMENT_MAJORITY_TIME "amendment_majority_time" +#define SECTION_BETA_RPC_API "beta_rpc_api" #define SECTION_CLUSTER_NODES "cluster_nodes" #define SECTION_COMPRESSION "compression" #define SECTION_DEBUG_LOGFILE "debug_logfile" @@ -57,10 +58,13 @@ struct ConfigSection #define SECTION_FETCH_DEPTH "fetch_depth" #define SECTION_HISTORICAL_SHARD_PATHS "historical_shard_paths" #define SECTION_INSIGHT "insight" +#define SECTION_IO_WORKERS "io_workers" #define SECTION_IPS "ips" #define SECTION_IPS_FIXED "ips_fixed" #define SECTION_LEDGER_HISTORY "ledger_history" +#define SECTION_LEDGER_REPLAY "ledger_replay" #define SECTION_MAX_TRANSACTIONS "max_transactions" +#define SECTION_NETWORK_ID "network_id" #define SECTION_NETWORK_QUORUM "network_quorum" #define SECTION_NODE_SEED "node_seed" #define SECTION_NODE_SIZE "node_size" @@ -73,6 +77,8 @@ struct ConfigSection #define SECTION_PEERS_MAX "peers_max" #define SECTION_PEERS_IN_MAX "peers_in_max" #define SECTION_PEERS_OUT_MAX "peers_out_max" +#define SECTION_PORT_GRPC "port_grpc" +#define SECTION_PREFETCH_WORKERS "prefetch_workers" #define SECTION_REDUCE_RELAY "reduce_relay" #define SECTION_RELATIONAL_DB "relational_db" #define SECTION_RELAY_PROPOSALS "relay_proposals" @@ -84,6 +90,7 @@ struct ConfigSection #define SECTION_SSL_VERIFY_FILE "ssl_verify_file" #define SECTION_SSL_VERIFY_DIR "ssl_verify_dir" #define SECTION_SERVER_DOMAIN "server_domain" +#define SECTION_SWEEP_INTERVAL "sweep_interval" #define SECTION_VALIDATORS_FILE "validators_file" #define SECTION_VALIDATION_SEED "validation_seed" #define SECTION_VALIDATOR_KEYS "validator_keys" @@ -94,12 +101,6 @@ struct ConfigSection #define SECTION_VALIDATOR_TOKEN "validator_token" #define SECTION_VETO_AMENDMENTS "veto_amendments" #define SECTION_WORKERS "workers" -#define SECTION_IO_WORKERS "io_workers" -#define SECTION_PREFETCH_WORKERS "prefetch_workers" -#define SECTION_LEDGER_REPLAY "ledger_replay" -#define SECTION_BETA_RPC_API "beta_rpc_api" -#define SECTION_SWEEP_INTERVAL "sweep_interval" -#define SECTION_NETWORK_ID "network_id" } // namespace ripple diff --git a/src/ripple/json/MultivarJson.h b/src/ripple/json/MultivarJson.h deleted file mode 100644 index 94d0090edc4..00000000000 --- a/src/ripple/json/MultivarJson.h +++ /dev/null @@ -1,112 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2023 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_JSON_MULTIVARJSON_H_INCLUDED -#define RIPPLE_JSON_MULTIVARJSON_H_INCLUDED - -#include - -#include -#include -#include -#include - -namespace ripple { -template -struct MultivarJson -{ - std::array val; - constexpr static std::size_t size = Size; - - Json::Value const& - select(auto&& selector) const - requires std::same_as - { - auto const index = selector(); - assert(index < size); - return val[index]; - } - - void - set(const char* key, - auto const& - v) requires std::constructible_from - { - for (auto& a : this->val) - a[key] = v; - } - - // Intentionally not using class enum here, MultivarJson is scope enough - enum IsMemberResult : int { none = 0, some, all }; - - [[nodiscard]] IsMemberResult - isMember(const char* key) const - { - int count = 0; - for (auto& a : this->val) - if (a.isMember(key)) - count += 1; - - return (count == 0 ? none : (count < size ? some : all)); - } -}; - -// Wrapper for Json for all supported API versions. -using MultiApiJson = MultivarJson<2>; - -/* - -NOTE: - -If a future API version change adds another possible format, change the size of -`MultiApiJson`, and update `apiVersionSelector()` to return the appropriate -selection value for the new `apiVersion` and higher. - -e.g. There are 2 formats now, the first, for version one, the second for -versions > 1. Hypothetically, if API version 4 adds a new format, `MultiApiJson` -would be MultivarJson<3>, and `apiVersionSelector` would return -`static_cast(apiVersion < 2 ? 0u : (apiVersion < 4 ? 1u : 2u))` - -NOTE: - -The more different JSON formats we support, the more CPU cycles we need to -pre-build JSON for different API versions e.g. when publishing streams to -`subscribe` clients. Hence it is desirable to keep MultiApiJson small and -instead fully deprecate and remove support for old API versions. For example, if -we removed support for API version 1 and added a different format for API -version 3, the `apiVersionSelector` would change to -`static_cast(apiVersion > 2)` - -*/ - -// Helper to create appropriate selector for indexing MultiApiJson by apiVersion -constexpr auto -apiVersionSelector(unsigned int apiVersion) noexcept -{ - return [apiVersion]() constexpr - { - // apiVersion <= 1 returns 0 - // apiVersion > 1 returns 1 - return static_cast(apiVersion > 1); - }; -} - -} // namespace ripple - -#endif diff --git a/src/ripple/json/impl/json_reader.cpp b/src/ripple/json/impl/json_reader.cpp index e4124e9bc8f..001049bcdbc 100644 --- a/src/ripple/json/impl/json_reader.cpp +++ b/src/ripple/json/impl/json_reader.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include @@ -922,10 +921,8 @@ Reader::getLocationLineAndColumn(Location location) const { int line, column; getLocationLineAndColumn(location, line, column); - constexpr std::size_t n = 18 + 16 + 16 + 1; - char buffer[n]; - snprintf(buffer, n, "Line %d, Column %d", line, column); - return buffer; + return "Line " + std::to_string(line) + ", Column " + + std::to_string(column); } std::string diff --git a/src/ripple/ledger/CachedView.h b/src/ripple/ledger/CachedView.h index e827d23c8c3..da483b34948 100644 --- a/src/ripple/ledger/CachedView.h +++ b/src/ripple/ledger/CachedView.h @@ -38,10 +38,7 @@ class CachedViewImpl : public DigestAwareReadView DigestAwareReadView const& base_; CachedSLEs& cache_; std::mutex mutable mutex_; - std::unordered_map< - key_type, - std::shared_ptr, - hardened_hash<>> mutable map_; + std::unordered_map> mutable map_; public: CachedViewImpl() = delete; diff --git a/src/ripple/ledger/impl/CachedView.cpp b/src/ripple/ledger/impl/CachedView.cpp index 3ad8a812b01..210031346d3 100644 --- a/src/ripple/ledger/impl/CachedView.cpp +++ b/src/ripple/ledger/impl/CachedView.cpp @@ -33,25 +33,40 @@ CachedViewImpl::exists(Keylet const& k) const std::shared_ptr CachedViewImpl::read(Keylet const& k) const { - { - std::lock_guard lock(mutex_); - auto const iter = map_.find(k.key); - if (iter != map_.end()) + static CountedObjects::Counter hits{"CachedView::hit"}; + static CountedObjects::Counter hitsexpired{"CachedView::hitExpired"}; + static CountedObjects::Counter misses{"CachedView::miss"}; + bool cacheHit = false; + bool baseRead = false; + + auto const digest = [&]() -> std::optional { { - if (!iter->second || !k.check(*iter->second)) - return nullptr; - return iter->second; + std::lock_guard lock(mutex_); + auto const iter = map_.find(k.key); + if (iter != map_.end()) + { + cacheHit = true; + return iter->second; + } } - } - auto const digest = base_.digest(k.key); + return base_.digest(k.key); + }(); if (!digest) return nullptr; - auto sle = cache_.fetch(*digest, [&]() { return base_.read(k); }); + auto sle = cache_.fetch(*digest, [&]() { + baseRead = true; + return base_.read(k); + }); + if (cacheHit && baseRead) + hitsexpired.increment(); + else if (cacheHit) + hits.increment(); + else + misses.increment(); std::lock_guard lock(mutex_); - auto const er = map_.emplace(k.key, sle); - auto const& iter = er.first; + auto const er = map_.emplace(k.key, *digest); bool const inserted = er.second; - if (iter->second && !k.check(*iter->second)) + if (sle && !k.check(*sle)) { if (!inserted) { @@ -62,7 +77,7 @@ CachedViewImpl::read(Keylet const& k) const } return nullptr; } - return iter->second; + return sle; } } // namespace detail diff --git a/src/ripple/net/InfoSub.h b/src/ripple/net/InfoSub.h index 092ba0c0035..33441dde632 100644 --- a/src/ripple/net/InfoSub.h +++ b/src/ripple/net/InfoSub.h @@ -33,7 +33,7 @@ namespace ripple { // Operations that clients may wish to perform against the network // Master operational handler, server sequencer, network tracker -class InfoSubRequest +class InfoSubRequest : public CountedObject { public: using pointer = std::shared_ptr; diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index 4e3e6c09fc4..8537f76b703 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -30,9 +30,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/nodestore/Types.h b/src/ripple/nodestore/Types.h index 08bcac2d25c..6d8583ed9d1 100644 --- a/src/ripple/nodestore/Types.h +++ b/src/ripple/nodestore/Types.h @@ -36,9 +36,8 @@ enum { // This sets a limit on the maximum number of writes // in a batch. Actual usage can be twice this since // we have a new batch growing as we write the old. - // The main goal is to avoid delays while persisting the ledger. // - batchWriteLimitSize = 262144 + batchWriteLimitSize = 65536 }; /** Return codes from Backend operations. */ diff --git a/src/ripple/nodestore/backend/NuDBFactory.cpp b/src/ripple/nodestore/backend/NuDBFactory.cpp index 74afe2ed7f1..30b848e82af 100644 --- a/src/ripple/nodestore/backend/NuDBFactory.cpp +++ b/src/ripple/nodestore/backend/NuDBFactory.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -36,7 +35,7 @@ namespace ripple { namespace NodeStore { -class NuDBBackend : public Backend, public BatchWriter::Callback +class NuDBBackend : public Backend { public: static constexpr std::uint64_t currentType = 1; @@ -47,7 +46,6 @@ class NuDBBackend : public Backend, public BatchWriter::Callback beast::Journal const j_; size_t const keyBytes_; - BatchWriter batch_; std::size_t const burstSize_; std::string const name_; nudb::store db_; @@ -62,7 +60,6 @@ class NuDBBackend : public Backend, public BatchWriter::Callback beast::Journal journal) : j_(journal) , keyBytes_(keyBytes) - , batch_(*this, scheduler) , burstSize_(burstSize) , name_(get(keyValues, "path")) , deletePath_(false) @@ -82,7 +79,6 @@ class NuDBBackend : public Backend, public BatchWriter::Callback beast::Journal journal) : j_(journal) , keyBytes_(keyBytes) - , batch_(*this, scheduler) , burstSize_(burstSize) , name_(get(keyValues, "path")) , db_(context) @@ -266,7 +262,13 @@ class NuDBBackend : public Backend, public BatchWriter::Callback void store(std::shared_ptr const& no) override { - batch_.store(no); + BatchWriteReport report; + report.writeCount = 1; + auto const start = std::chrono::steady_clock::now(); + do_insert(no); + report.elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + scheduler_.onBatchWrite(report); } void @@ -327,7 +329,7 @@ class NuDBBackend : public Backend, public BatchWriter::Callback int getWriteLoad() override { - return batch_.getWriteLoad(); + return 0; } void @@ -355,12 +357,6 @@ class NuDBBackend : public Backend, public BatchWriter::Callback Throw(ec); } - void - writeBatch(Batch const& batch) override - { - storeBatch(batch); - } - int fdRequired() const override { diff --git a/src/ripple/overlay/Message.h b/src/ripple/overlay/Message.h index 6cb6900c639..0d6479366e8 100644 --- a/src/ripple/overlay/Message.h +++ b/src/ripple/overlay/Message.h @@ -100,14 +100,6 @@ class Message : public std::enable_shared_from_this return validatorKey_; } - /** Get the message type from the payload header. - * First four bytes are the compression/algorithm flag and the payload size. - * Next two bytes are the message type - * @return Message type - */ - int - getType() const; - private: std::vector buffer_; std::vector bufferCompressed_; @@ -137,6 +129,15 @@ class Message : public std::enable_shared_from_this */ void compress(); + + /** Get the message type from the payload header. + * First four bytes are the compression/algorithm flag and the payload size. + * Next two bytes are the message type + * @param in Payload header pointer + * @return Message type + */ + int + getType(std::uint8_t const* in) const; }; } // namespace ripple diff --git a/src/ripple/overlay/Peer.h b/src/ripple/overlay/Peer.h index dbe5416e590..ba415974151 100644 --- a/src/ripple/overlay/Peer.h +++ b/src/ripple/overlay/Peer.h @@ -39,7 +39,6 @@ enum class ProtocolFeature { ValidatorListPropagation, ValidatorList2Propagation, LedgerReplay, - StartProtocol }; /** Represents a peer connection in the overlay. */ diff --git a/src/ripple/overlay/README.md b/src/ripple/overlay/README.md index bfead075135..6525e5edf86 100644 --- a/src/ripple/overlay/README.md +++ b/src/ripple/overlay/README.md @@ -365,50 +365,6 @@ transferred between A and B and will not be able to intelligently tamper with th message stream between Alice and Bob, although she may be still be able to inject delays or terminate the link. -## Peer Connection Sequence - -The _PeerImp_ object can be constructed as either an outbound or an inbound peer. -The outbound peer is constructed by the _ConnectAttempt_ - the client side of -the connection. The inbound peer is constructed by the _InboundHandoff_ - -the server side of the connection. This differentiation of the peers matters only -in terms of the object construction. Once constructed, both inbound and outbound -peer play the same role. - -### Outbound Peer - -An outbound connection is initiated once a second by -the _OverlayImpl::Timer::on_timer()_ method. This method calls -_OverlayImpl::autoConnect()_, which in turn calls _OverlayImpl::connect()_ for -every outbound endpoint generated by _PeerFinder::autoconnect()_. _connect()_ -method constructs _ConnectAttempt_ object. _ConnectAttempt_ attempts to connect -to the provided endpoint and on a successful connection executes the client side -of the handshake protocol described above. If the handshake is successful then -the outbound _PeerImp_ object is constructed and passed to the overlay manager -_OverlayImpl_, which adds the object to the list of peers and children. The latter -maintains a list of objects which might be executing an asynchronous operation -and therefore have to be stopped on shutdown. The outbound _PeerImp_ sends -_TMStartProtocol_ message on start to instruct the connected inbound peer that -the outbound peer is ready to receive the protocol messages. - -### Inbound Peer - -Construction of the inbound peer is more involved. A multi protocol-server, -_ServerImpl_ located in _src/ripple/server_ module, maintains multiple configured -listening ports. Each listening port allows for multiple protocols including HTTP, -HTTP/S, WebSocket, Secure WebSocket, and the Peer protocol. For simplicity this -sequence describes only the Peer protocol. _ServerImpl_ constructs -_Door_ object for each configured protocol. Each instance of the _Door_ object -accepts connections on the configured port. On a successful connection the _Door_ -constructs _SSLHTTPPeer_ object since the Peer protocol always uses SSL -connection. _SSLHTTPPeer_ executes the SSL handshake. If the handshake is successful -then a server handler, _ServerHandlerImpl_ located in _src/ripple/src/impl_, hands off -the connection to the _OverlayImpl::onHandoff()_ method. _onHandoff()_ method -validates the client's HTTP handshake request described above. If the request is -valid then the _InboundHandoff_ object is constructed. _InboundHandoff_ sends -HTTP response to the connected client, constructs the inbound _PeerImp_ object, -and passes it to the overlay manager _OverlayImpl_, which adds the object to -the list of peers and children. Once the inbound _PeerImp_ receives -_TMStartProtocol_ message, it starts sending the protocol messages. # Ripple Clustering # diff --git a/src/ripple/overlay/impl/InboundHandoff.cpp b/src/ripple/overlay/impl/InboundHandoff.cpp deleted file mode 100644 index 1f45e1d37a7..00000000000 --- a/src/ripple/overlay/impl/InboundHandoff.cpp +++ /dev/null @@ -1,185 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include - -#include - -namespace ripple { - -InboundHandoff::InboundHandoff( - Application& app, - id_t id, - std::shared_ptr const& slot, - http_request_type&& request, - PublicKey const& publicKey, - ProtocolVersion protocol, - Resource::Consumer consumer, - std::unique_ptr&& stream_ptr, - OverlayImpl& overlay) - : OverlayImpl::Child(overlay) - , app_(app) - , id_(id) - , sink_( - app_.journal("Peer"), - [id]() { - std::stringstream ss; - ss << "[" << std::setfill('0') << std::setw(3) << id << "] "; - return ss.str(); - }()) - , journal_(sink_) - , stream_ptr_(std::move(stream_ptr)) - , strand_(stream_ptr_->next_layer().socket().get_executor()) - , remote_address_(slot->remote_endpoint()) - , protocol_(protocol) - , publicKey_(publicKey) - , usage_(consumer) - , slot_(slot) - , request_(std::move(request)) -{ -} - -void -InboundHandoff::run() -{ - if (!strand_.running_in_this_thread()) - return post( - strand_, std::bind(&InboundHandoff::run, shared_from_this())); - sendResponse(); -} - -void -InboundHandoff::stop() -{ - if (!strand_.running_in_this_thread()) - return post( - strand_, std::bind(&InboundHandoff::stop, shared_from_this())); - if (stream_ptr_->next_layer().socket().is_open()) - { - JLOG(journal_.debug()) << "Stop"; - } - close(); -} - -void -InboundHandoff::sendResponse() -{ - auto const sharedValue = makeSharedValue(*stream_ptr_, journal_); - // This shouldn't fail since we already computed - // the shared value successfully in OverlayImpl - if (!sharedValue) - return fail("makeSharedValue: Unexpected failure"); - - JLOG(journal_.info()) << "Protocol: " << to_string(protocol_); - JLOG(journal_.info()) << "Public Key: " - << toBase58(TokenType::NodePublic, publicKey_); - - auto write_buffer = std::make_shared(); - - boost::beast::ostream(*write_buffer) << makeResponse( - !overlay_.peerFinder().config().peerPrivate, - request_, - overlay_.setup().public_ip, - remote_address_.address(), - *sharedValue, - overlay_.setup().networkID, - protocol_, - app_); - - // Write the whole buffer and only start protocol when that's done. - boost::asio::async_write( - *stream_ptr_, - write_buffer->data(), - boost::asio::transfer_all(), - bind_executor( - strand_, - [this, write_buffer, self = shared_from_this()]( - error_code ec, std::size_t bytes_transferred) { - if (!stream_ptr_->next_layer().socket().is_open()) - return; - if (ec == boost::asio::error::operation_aborted) - return; - if (ec) - return fail("onWriteResponse", ec); - if (write_buffer->size() == bytes_transferred) - return createPeer(); - return fail("Failed to write header"); - })); -} - -void -InboundHandoff::fail(std::string const& name, error_code const& ec) -{ - if (socket().is_open()) - { - JLOG(journal_.warn()) - << name << " from " << toBase58(TokenType::NodePublic, publicKey_) - << " at " << remote_address_.to_string() << ": " << ec.message(); - } - close(); -} - -void -InboundHandoff::fail(std::string const& reason) -{ - if (journal_.active(beast::severities::kWarning) && socket().is_open()) - { - auto const n = app_.cluster().member(publicKey_); - JLOG(journal_.warn()) - << (n ? remote_address_.to_string() : *n) << " failed: " << reason; - } - close(); -} - -void -InboundHandoff::close() -{ - if (socket().is_open()) - { - socket().close(); - JLOG(journal_.debug()) << "Closed"; - } -} - -void -InboundHandoff::createPeer() -{ - auto peer = std::make_shared( - app_, - id_, - slot_, - std::move(request_), - publicKey_, - protocol_, - usage_, - std::move(stream_ptr_), - overlay_); - - overlay_.add_active(peer); -} - -InboundHandoff::socket_type& -InboundHandoff::socket() const -{ - return stream_ptr_->next_layer().socket(); -} - -} // namespace ripple \ No newline at end of file diff --git a/src/ripple/overlay/impl/InboundHandoff.h b/src/ripple/overlay/impl/InboundHandoff.h deleted file mode 100644 index 3f3154c3a8f..00000000000 --- a/src/ripple/overlay/impl/InboundHandoff.h +++ /dev/null @@ -1,102 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_OVERLAY_INBOUNDHANDOFF_H_INCLUDED -#define RIPPLE_OVERLAY_INBOUNDHANDOFF_H_INCLUDED - -#include - -namespace ripple { - -/** Sends HTTP response. Instantiates the inbound peer - * once the response is sent. Maintains all data members - * required for the inbound peer instantiation. - */ -class InboundHandoff : public OverlayImpl::Child, - public std::enable_shared_from_this -{ -private: - using error_code = boost::system::error_code; - using socket_type = boost::asio::ip::tcp::socket; - using middle_type = boost::beast::tcp_stream; - using stream_type = boost::beast::ssl_stream; - using id_t = Peer::id_t; - Application& app_; - id_t const id_; - beast::WrappedSink sink_; - beast::Journal const journal_; - std::unique_ptr stream_ptr_; - boost::asio::strand strand_; - beast::IP::Endpoint const remote_address_; - ProtocolVersion protocol_; - PublicKey const publicKey_; - Resource::Consumer usage_; - std::shared_ptr const slot_; - http_request_type request_; - -public: - virtual ~InboundHandoff() override = default; - - InboundHandoff( - Application& app, - id_t id, - std::shared_ptr const& slot, - http_request_type&& request, - PublicKey const& publicKey, - ProtocolVersion protocol, - Resource::Consumer consumer, - std::unique_ptr&& stream_ptr, - OverlayImpl& overlay); - - // This class isn't meant to be copied - InboundHandoff(InboundHandoff const&) = delete; - InboundHandoff& - operator=(InboundHandoff const&) = delete; - - /** Start the handshake */ - void - run(); - /** Stop the child */ - void - stop() override; - -private: - /** Send upgrade response to the client */ - void - sendResponse(); - /** Instantiate and run the overlay peer */ - void - createPeer(); - /** Log and close */ - void - fail(std::string const& name, error_code const& ec); - /** Log and close */ - void - fail(std::string const& reason); - /** Close connection */ - void - close(); - /** Get underlying socket */ - socket_type& - socket() const; -}; - -} // namespace ripple - -#endif // RIPPLE_OVERLAY_INBOUNDHANDOFF_H_INCLUDED diff --git a/src/ripple/overlay/impl/Message.cpp b/src/ripple/overlay/impl/Message.cpp index 1b434225501..b4cb1f192aa 100644 --- a/src/ripple/overlay/impl/Message.cpp +++ b/src/ripple/overlay/impl/Message.cpp @@ -70,7 +70,7 @@ Message::compress() using namespace ripple::compression; auto const messageBytes = buffer_.size() - headerBytes; - auto type = getType(); + auto type = getType(buffer_.data()); bool const compressible = [&] { if (messageBytes <= 70) @@ -221,10 +221,9 @@ Message::getBuffer(Compressed tryCompressed) } int -Message::getType() const +Message::getType(std::uint8_t const* in) const { - int type = - (static_cast(*(buffer_.data() + 4)) << 8) + *(buffer_.data() + 5); + int type = (static_cast(*(in + 4)) << 8) + *(in + 5); return type; } diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index c48ab378cb3..1bb9a381edd 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -280,7 +279,7 @@ OverlayImpl::onHandoff( } } - auto const ih = std::make_shared( + auto const peer = std::make_shared( app_, id, slot, @@ -291,10 +290,18 @@ OverlayImpl::onHandoff( std::move(stream_ptr), *this); { + // As we are not on the strand, run() must be called + // while holding the lock, otherwise new I/O can be + // queued after a call to stop(). std::lock_guard lock(mutex_); - list_.emplace(ih.get(), ih); + { + auto const result = m_peers.emplace(peer->slot(), peer); + assert(result.second); + (void)result.second; + } + list_.emplace(peer.get(), peer); - ih->run(); + peer->run(); } handoff.moved = true; return handoff; @@ -466,7 +473,7 @@ OverlayImpl::start() PeerFinder::Config config = PeerFinder::Config::makeConfig( app_.config(), serverHandler_.setup().overlay.port, - !app_.getValidationPublicKey().empty(), + app_.getValidationPublicKey().has_value(), setup_.ipLimit); m_peerFinder->setConfig(config); @@ -484,9 +491,6 @@ OverlayImpl::start() // Pool of servers operated by Ripple Labs Inc. - https://ripple.com bootstrapIps.push_back("r.ripple.com 51235"); - // Pool of servers operated by Alloy Networks - https://www.alloy.ee - bootstrapIps.push_back("zaphod.alloy.ee 51235"); - // Pool of servers operated by ISRDC - https://isrdc.in bootstrapIps.push_back("sahyadri.isrdc.in 51235"); } diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index dc23f3325f8..f93c9f135ad 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -28,12 +28,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -41,14 +39,13 @@ #include #include #include -#include #include +#include #include #include #include -#include #include #include #include @@ -66,30 +63,6 @@ std::chrono::milliseconds constexpr peerHighLatency{300}; std::chrono::seconds constexpr peerTimerInterval{60}; } // namespace -std::string -closeReasonToString(protocol::TMCloseReason reason) -{ - switch (reason) - { - case protocol::TMCloseReason::crCHARGE_RESOURCES: - return "Charge: Resources"; - case protocol::TMCloseReason::crMALFORMED_HANDSHAKE1: - return "Malformed handshake data (1)"; - case protocol::TMCloseReason::crMALFORMED_HANDSHAKE2: - return "Malformed handshake data (2)"; - case protocol::TMCloseReason::crMALFORMED_HANDSHAKE3: - return "Malformed handshake data (3)"; - case protocol::TMCloseReason::crLARGE_SENDQUEUE: - return "Large send queue"; - case protocol::TMCloseReason::crNOT_USEFUL: - return "Not useful"; - case protocol::TMCloseReason::crPING_TIMEOUT: - return "Ping timeout"; - default: - return "Unknown reason"; - } -} - PeerImp::PeerImp( Application& app, id_t id, @@ -156,11 +129,6 @@ PeerImp::PeerImp( << " tx reduce-relay enabled " << txReduceRelayEnabled_ << " on " << remote_address_ << " " << id_; - if (auto member = app_.cluster().member(publicKey_)) - { - name_ = *member; - JLOG(journal_.info()) << "Cluster name: " << *member; - } } PeerImp::~PeerImp() @@ -211,7 +179,7 @@ PeerImp::run() closed = parseLedgerHash(iter->value()); if (!closed) - fail(protocol::TMCloseReason::crMALFORMED_HANDSHAKE1); + fail("Malformed handshake data (1)"); } if (auto const iter = headers_.find("Previous-Ledger"); @@ -220,11 +188,11 @@ PeerImp::run() previous = parseLedgerHash(iter->value()); if (!previous) - fail(protocol::TMCloseReason::crMALFORMED_HANDSHAKE2); + fail("Malformed handshake data (2)"); } if (previous && !closed) - fail(protocol::TMCloseReason::crMALFORMED_HANDSHAKE3); + fail("Malformed handshake data (3)"); { std::lock_guard sl(recentLock_); @@ -234,7 +202,10 @@ PeerImp::run() previousLedgerHash_ = *previous; } - doProtocolStart(); + if (inbound_) + doAccept(); + else + doProtocolStart(); // Anything else that needs to be done with the connection should be // done in doProtocolStart @@ -376,7 +347,7 @@ PeerImp::charge(Resource::Charge const& fee) { // Sever the connection overlay_.incPeerDisconnectCharges(); - fail(protocol::TMCloseReason::crCHARGE_RESOURCES); + fail("charge: Resources"); } } @@ -534,8 +505,6 @@ PeerImp::supportsFeature(ProtocolFeature f) const return protocol_ >= make_protocol(2, 2); case ProtocolFeature::LedgerReplay: return ledgerReplayEnabled_; - case ProtocolFeature::StartProtocol: - return protocol_ >= make_protocol(2, 3); } return false; } @@ -628,34 +597,22 @@ PeerImp::close() } void -PeerImp::fail(protocol::TMCloseReason reason) +PeerImp::fail(std::string const& reason) { if (!strand_.running_in_this_thread()) return post( strand_, std::bind( - (void (Peer::*)(protocol::TMCloseReason)) & PeerImp::fail, + (void (Peer::*)(std::string const&)) & PeerImp::fail, shared_from_this(), reason)); if (journal_.active(beast::severities::kWarning) && socket_.is_open()) { std::string const n = name(); JLOG(journal_.warn()) << (n.empty() ? remote_address_.to_string() : n) - << " failed: " << closeReasonToString(reason); - } - - // erase all outstanding messages except for the one - // currently being executed - if (send_queue_.size() > 1) - { - decltype(send_queue_) q({send_queue_.front()}); - send_queue_.swap(q); + << " failed: " << reason; } - - closeOnWriteComplete_ = true; - protocol::TMGracefulClose tmGC; - tmGC.set_reason(reason); - send(std::make_shared(tmGC, protocol::mtGRACEFUL_CLOSE)); + close(); } void @@ -747,7 +704,7 @@ PeerImp::onTimer(error_code const& ec) if (large_sendq_++ >= Tuning::sendqIntervals) { - fail(protocol::TMCloseReason::crLARGE_SENDQUEUE); + fail("Large send queue"); return; } @@ -766,7 +723,7 @@ PeerImp::onTimer(error_code const& ec) (duration > app_.config().MAX_UNKNOWN_TIME))) { overlay_.peerFinder().on_failure(slot_); - fail(protocol::TMCloseReason::crLARGE_SENDQUEUE); + fail("Not useful"); return; } } @@ -774,7 +731,7 @@ PeerImp::onTimer(error_code const& ec) // Already waiting for PONG if (lastPingSeq_) { - fail(protocol::TMCloseReason::crPING_TIMEOUT); + fail("Ping Timeout"); return; } @@ -806,6 +763,71 @@ PeerImp::onShutdown(error_code ec) } //------------------------------------------------------------------------------ +void +PeerImp::doAccept() +{ + assert(read_buffer_.size() == 0); + + JLOG(journal_.debug()) << "doAccept: " << remote_address_; + + auto const sharedValue = makeSharedValue(*stream_ptr_, journal_); + + // This shouldn't fail since we already computed + // the shared value successfully in OverlayImpl + if (!sharedValue) + return fail("makeSharedValue: Unexpected failure"); + + JLOG(journal_.info()) << "Protocol: " << to_string(protocol_); + JLOG(journal_.info()) << "Public Key: " + << toBase58(TokenType::NodePublic, publicKey_); + + if (auto member = app_.cluster().member(publicKey_)) + { + { + std::unique_lock lock{nameMutex_}; + name_ = *member; + } + JLOG(journal_.info()) << "Cluster name: " << *member; + } + + overlay_.activate(shared_from_this()); + + // XXX Set timer: connection is in grace period to be useful. + // XXX Set timer: connection idle (idle may vary depending on connection + // type.) + + auto write_buffer = std::make_shared(); + + boost::beast::ostream(*write_buffer) << makeResponse( + !overlay_.peerFinder().config().peerPrivate, + request_, + overlay_.setup().public_ip, + remote_address_.address(), + *sharedValue, + overlay_.setup().networkID, + protocol_, + app_); + + // Write the whole buffer and only start protocol when that's done. + boost::asio::async_write( + stream_, + write_buffer->data(), + boost::asio::transfer_all(), + bind_executor( + strand_, + [this, write_buffer, self = shared_from_this()]( + error_code ec, std::size_t bytes_transferred) { + if (!socket_.is_open()) + return; + if (ec == boost::asio::error::operation_aborted) + return; + if (ec) + return fail("onWriteResponse", ec); + if (write_buffer->size() == bytes_transferred) + return doProtocolStart(); + return fail("Failed to write header"); + })); +} std::string PeerImp::name() const @@ -829,50 +851,40 @@ PeerImp::doProtocolStart() { onReadMessage(error_code(), 0); - bool supportedProtocol = supportsFeature(ProtocolFeature::StartProtocol); - - if (!inbound_) + // Send all the validator lists that have been loaded + if (inbound_ && supportsFeature(ProtocolFeature::ValidatorListPropagation)) { - // Instruct connected inbound peer to start sending - // protocol messages - if (supportedProtocol) - { - JLOG(journal_.debug()) - << "doProtocolStart(): outbound sending mtSTART_PROTOCOL to " - << remote_address_; - protocol::TMStartProtocol tmPS; - tmPS.set_starttime(std::chrono::duration_cast( - clock_type::now().time_since_epoch()) - .count()); - send(std::make_shared(tmPS, protocol::mtSTART_PROTOCOL)); - } - else - { - JLOG(journal_.debug()) << "doProtocolStart(): outbound connected " - "to an older protocol on " - << remote_address_ << " " << protocol_.first - << " " << protocol_.second; - } - - if (auto m = overlay_.getManifestsMessage()) - send(m); + app_.validators().for_each_available( + [&](std::string const& manifest, + std::uint32_t version, + std::map const& blobInfos, + PublicKey const& pubKey, + std::size_t maxSequence, + uint256 const& hash) { + ValidatorList::sendValidatorList( + *this, + 0, + pubKey, + maxSequence, + version, + manifest, + blobInfos, + app_.getHashRouter(), + p_journal_); - // Request shard info from peer - protocol::TMGetPeerShardInfoV2 tmGPS; - tmGPS.set_relays(0); - send(std::make_shared( - tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2)); - } - // Backward compatibility with the older protocols - else if (!supportedProtocol) - { - JLOG(journal_.debug()) - << "doProtocolStart(): inbound handling of an older protocol on " - << remote_address_ << " " << protocol_.first << " " - << protocol_.second; - onStartProtocol(); + // Don't send it next time. + app_.getHashRouter().addSuppressionPeer(hash, id_); + }); } + if (auto m = overlay_.getManifestsMessage()) + send(m); + + // Request shard info from peer + protocol::TMGetPeerShardInfoV2 tmGPS; + tmGPS.set_relays(0); + send(std::make_shared(tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2)); + setTimer(); } @@ -939,11 +951,7 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytes_transferred) if (!socket_.is_open()) return; if (ec == boost::asio::error::operation_aborted) - { - if (closeOnWriteComplete_) - close(); return; - } if (ec) return fail("onWriteMessage", ec); if (auto stream = journal_.trace()) @@ -957,11 +965,6 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytes_transferred) metrics_.sent.add_message(bytes_transferred); assert(!send_queue_.empty()); - if (send_queue_.front()->getType() == protocol::mtGRACEFUL_CLOSE) - { - close(); - return; - } send_queue_.pop(); if (!send_queue_.empty()) { @@ -1571,7 +1574,9 @@ PeerImp::handleTransaction( flags |= SF_TRUSTED; } - if (app_.getValidationPublicKey().empty()) + // for non-validator nodes only -- localPublicKey is set for + // validators only + if (!app_.getValidationPublicKey()) { // For now, be paranoid and have each validator // check each transaction, regardless of source @@ -1989,10 +1994,6 @@ PeerImp::onMessage(std::shared_ptr const& m) JLOG(p_journal_.trace()) << "Proposal: " << (isTrusted ? "trusted" : "untrusted"); - std::optional ledgerSeq; - if (set.has_ledgerseq()) - ledgerSeq = set.ledgerseq(); - auto proposal = RCLCxPeerPos( publicKey, sig, @@ -2003,9 +2004,7 @@ PeerImp::onMessage(std::shared_ptr const& m) proposeHash, closeTime, app_.timeKeeper().closeTime(), - calcNodeID(app_.validatorManifests().getMasterKey(publicKey)), - ledgerSeq, - beast::get_abstract_clock()}); + calcNodeID(app_.validatorManifests().getMasterKey(publicKey))}); std::weak_ptr weak = shared_from_this(); app_.getJobQueue().addJob( @@ -2941,69 +2940,6 @@ PeerImp::onMessage(std::shared_ptr const& m) << "onMessage: TMSquelch " << slice << " " << id() << " " << duration; } -void -PeerImp::onStartProtocol() -{ - JLOG(journal_.debug()) << "onStartProtocol(): " << remote_address_; - // Send all the validator lists that have been loaded - if (supportsFeature(ProtocolFeature::ValidatorListPropagation)) - { - app_.validators().for_each_available( - [&](std::string const& manifest, - std::uint32_t version, - std::map const& blobInfos, - PublicKey const& pubKey, - std::size_t maxSequence, - uint256 const& hash) { - ValidatorList::sendValidatorList( - *this, - 0, - pubKey, - maxSequence, - version, - manifest, - blobInfos, - app_.getHashRouter(), - p_journal_); - - // Don't send it next time. - app_.getHashRouter().addSuppressionPeer(hash, id_); - }); - } - - if (auto m = overlay_.getManifestsMessage()) - send(m); - - // Request shard info from peer - protocol::TMGetPeerShardInfoV2 tmGPS; - tmGPS.set_relays(0); - send(std::make_shared(tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2)); -} - -void -PeerImp::onMessage(std::shared_ptr const& m) -{ - JLOG(journal_.debug()) << "onMessage(TMStartProtocol): " << remote_address_; - onStartProtocol(); -} - -void -PeerImp::onMessage(const std::shared_ptr& m) -{ - using on_message_fn = - void (PeerImp::*)(std::shared_ptr const&); - if (!strand_.running_in_this_thread()) - return post( - strand_, - std::bind( - (on_message_fn)&PeerImp::onMessage, shared_from_this(), m)); - - JLOG(journal_.info()) << "got graceful close from: " << remote_address_ - << " reason: " << closeReasonToString(m->reason()); - - close(); -} - //-------------------------------------------------------------------------- void @@ -3175,11 +3111,7 @@ PeerImp::checkTransaction( bool const trusted(flags & SF_TRUSTED); app_.getOPs().processTransaction( - tx, - trusted, - RPC::SubmitSync::async, - false, - NetworkOPs::FailHard::no); + tx, trusted, false, NetworkOPs::FailHard::no); } catch (std::exception const& ex) { diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index d922e757946..710ab4d74d6 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -180,8 +180,6 @@ class PeerImp : public Peer, bool vpReduceRelayEnabled_ = false; bool ledgerReplayEnabled_ = false; LedgerReplayMsgHandler ledgerReplayMsgHandler_; - // close connection when async write is complete - bool closeOnWriteComplete_ = false; friend class OverlayImpl; @@ -237,7 +235,7 @@ class PeerImp : public Peer, /** Create outgoing, handshaked peer. */ // VFALCO legacyPublicKey should be implied by the Slot - template + template PeerImp( Application& app, std::unique_ptr&& stream_ptr, @@ -415,7 +413,7 @@ class PeerImp : public Peer, isHighLatency() const override; void - fail(protocol::TMCloseReason reason); + fail(std::string const& reason); // Return any known shard info from this peer and its sub peers [[nodiscard]] hash_map const @@ -460,6 +458,9 @@ class PeerImp : public Peer, void onShutdown(error_code ec); + void + doAccept(); + std::string name() const; @@ -583,10 +584,6 @@ class PeerImp : public Peer, onMessage(std::shared_ptr const& m); void onMessage(std::shared_ptr const& m); - void - onMessage(std::shared_ptr const& m); - void - onMessage(std::shared_ptr const& m); private: //-------------------------------------------------------------------------- @@ -645,9 +642,6 @@ class PeerImp : public Peer, void processLedgerRequest(std::shared_ptr const& m); - - void - onStartProtocol(); }; //------------------------------------------------------------------------------ diff --git a/src/ripple/overlay/impl/ProtocolMessage.h b/src/ripple/overlay/impl/ProtocolMessage.h index 6071a621db5..d6fb14bc78c 100644 --- a/src/ripple/overlay/impl/ProtocolMessage.h +++ b/src/ripple/overlay/impl/ProtocolMessage.h @@ -112,10 +112,6 @@ protocolMessageName(int type) return "get_peer_shard_info_v2"; case protocol::mtPEER_SHARD_INFO_V2: return "peer_shard_info_v2"; - case protocol::mtSTART_PROTOCOL: - return "start_protocol"; - case protocol::mtGRACEFUL_CLOSE: - return "graceful_close"; default: break; } @@ -496,14 +492,6 @@ invokeProtocolMessage( success = detail::invoke( *header, buffers, handler); break; - case protocol::mtSTART_PROTOCOL: - success = detail::invoke( - *header, buffers, handler); - break; - case protocol::mtGRACEFUL_CLOSE: - success = detail::invoke( - *header, buffers, handler); - break; default: handler.onMessageUnknown(header->message_type); success = true; diff --git a/src/ripple/overlay/impl/ProtocolVersion.cpp b/src/ripple/overlay/impl/ProtocolVersion.cpp index 8325f6d32fb..fbd48474420 100644 --- a/src/ripple/overlay/impl/ProtocolVersion.cpp +++ b/src/ripple/overlay/impl/ProtocolVersion.cpp @@ -37,8 +37,7 @@ namespace ripple { constexpr ProtocolVersion const supportedProtocolList[] { {2, 1}, - {2, 2}, - {2, 3} + {2, 2} }; // clang-format on diff --git a/src/ripple/peerfinder/impl/Logic.h b/src/ripple/peerfinder/impl/Logic.h index dc325cc480d..7a2b6a7543a 100644 --- a/src/ripple/peerfinder/impl/Logic.h +++ b/src/ripple/peerfinder/impl/Logic.h @@ -273,7 +273,7 @@ class Logic std::lock_guard _(lock_); - // Check for duplicate connection + // Check for connection limit per address if (is_public(remote_endpoint)) { auto const count = @@ -287,6 +287,15 @@ class Logic } } + // Check for duplicate connection + if (slots_.find(remote_endpoint) != slots_.end()) + { + JLOG(m_journal.debug()) + << beast::leftw(18) << "Logic dropping " << remote_endpoint + << " as duplicate incoming"; + return SlotImp::ptr(); + } + // Create the slot SlotImp::ptr const slot(std::make_shared( local_endpoint, diff --git a/src/ripple/proto/ripple.proto b/src/ripple/proto/ripple.proto index 5ea0fcba450..74cbfe8f6cb 100644 --- a/src/ripple/proto/ripple.proto +++ b/src/ripple/proto/ripple.proto @@ -33,8 +33,6 @@ enum MessageType mtPEER_SHARD_INFO_V2 = 62; mtHAVE_TRANSACTIONS = 63; mtTRANSACTIONS = 64; - mtSTART_PROTOCOL = 65; - mtGRACEFUL_CLOSE = 66; } // token, iterations, target, challenge = issue demand for proof of work @@ -233,8 +231,6 @@ message TMProposeSet // Number of hops traveled optional uint32 hops = 12 [deprecated=true]; - - optional uint32 ledgerSeq = 14; // sequence of the ledger we are proposing } enum TxSetStatus @@ -454,24 +450,3 @@ message TMHaveTransactions repeated bytes hashes = 1; } -message TMStartProtocol -{ - required uint64 startTime = 1; -} - -enum TMCloseReason -{ - crMALFORMED_HANDSHAKE1 = 1; - crMALFORMED_HANDSHAKE2 = 2; - crMALFORMED_HANDSHAKE3 = 3; - crCHARGE_RESOURCES = 4; - crLARGE_SENDQUEUE = 5; - crNOT_USEFUL = 6; - crPING_TIMEOUT = 7; -} - -message TMGracefulClose -{ - required TMCloseReason reason = 1; -} - diff --git a/src/ripple/protocol/ApiVersion.h b/src/ripple/protocol/ApiVersion.h new file mode 100644 index 00000000000..1cd03b0651d --- /dev/null +++ b/src/ripple/protocol/ApiVersion.h @@ -0,0 +1,119 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_PROTOCOL_APIVERSION_H_INCLUDED +#define RIPPLE_PROTOCOL_APIVERSION_H_INCLUDED + +#include +#include +#include + +namespace ripple { + +/** + * API version numbers used in later API versions + * + * Requests with a version number in the range + * [apiMinimumSupportedVersion, apiMaximumSupportedVersion] + * are supported. + * + * If [beta_rpc_api] is enabled in config, the version numbers + * in the range [apiMinimumSupportedVersion, apiBetaVersion] + * are supported. + * + * Network Requests without explicit version numbers use + * apiVersionIfUnspecified. apiVersionIfUnspecified is 1, + * because all the RPC requests with a version >= 2 must + * explicitly specify the version in the requests. + * Note that apiVersionIfUnspecified will be lower than + * apiMinimumSupportedVersion when we stop supporting API + * version 1. + * + * Command line Requests use apiCommandLineVersion. + */ + +namespace RPC { + +template +constexpr static std::integral_constant apiVersion = {}; + +constexpr static auto apiInvalidVersion = apiVersion<0>; +constexpr static auto apiMinimumSupportedVersion = apiVersion<1>; +constexpr static auto apiMaximumSupportedVersion = apiVersion<2>; +constexpr static auto apiVersionIfUnspecified = apiVersion<1>; +constexpr static auto apiCommandLineVersion = + apiVersion<1>; // TODO Bump to 2 later +constexpr static auto apiBetaVersion = apiVersion<3>; +constexpr static auto apiMaximumValidVersion = apiBetaVersion; + +static_assert(apiInvalidVersion < apiMinimumSupportedVersion); +static_assert( + apiVersionIfUnspecified >= apiMinimumSupportedVersion && + apiVersionIfUnspecified <= apiMaximumSupportedVersion); +static_assert( + apiCommandLineVersion >= apiMinimumSupportedVersion && + apiCommandLineVersion <= apiMaximumSupportedVersion); +static_assert(apiMaximumSupportedVersion >= apiMinimumSupportedVersion); +static_assert(apiBetaVersion >= apiMaximumSupportedVersion); +static_assert(apiMaximumValidVersion >= apiMaximumSupportedVersion); + +} // namespace RPC + +template + void + forApiVersions(Fn const& fn, Args&&... args) requires // + (maxVer >= minVer) && // + (minVer >= RPC::apiMinimumSupportedVersion) && // + (RPC::apiMaximumValidVersion >= maxVer) && + requires +{ + fn(std::integral_constant{}, + std::forward(args)...); + fn(std::integral_constant{}, + std::forward(args)...); +} +{ + constexpr auto size = maxVer + 1 - minVer; + [&](std::index_sequence) + { + (((void)fn( + std::integral_constant{}, + std::forward(args)...)), + ...); + } + (std::make_index_sequence{}); +} + +template +void +forAllApiVersions(Fn const& fn, Args&&... args) requires requires +{ + forApiVersions< + RPC::apiMinimumSupportedVersion, + RPC::apiMaximumValidVersion>(fn, std::forward(args)...); +} +{ + forApiVersions< + RPC::apiMinimumSupportedVersion, + RPC::apiMaximumValidVersion>(fn, std::forward(args)...); +} + +} // namespace ripple + +#endif diff --git a/src/ripple/protocol/ErrorCodes.h b/src/ripple/protocol/ErrorCodes.h index 8319b69c8c2..ad849f2b4ef 100644 --- a/src/ripple/protocol/ErrorCodes.h +++ b/src/ripple/protocol/ErrorCodes.h @@ -145,7 +145,11 @@ enum error_code_i { // AMM rpcISSUE_MALFORMED = 93, - rpcLAST = rpcISSUE_MALFORMED // rpcLAST should always equal the last code.= + // Oracle + rpcORACLE_MALFORMED = 94, + + rpcLAST = + rpcORACLE_MALFORMED // rpcLAST should always equal the last code.= }; /** Codes returned in the `warnings` array of certain RPC commands. diff --git a/src/ripple/protocol/Feature.h b/src/ripple/protocol/Feature.h index 3bdfcb15c59..d00dd1555f2 100644 --- a/src/ripple/protocol/Feature.h +++ b/src/ripple/protocol/Feature.h @@ -74,7 +74,7 @@ namespace detail { // Feature.cpp. Because it's only used to reserve storage, and determine how // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than // the actual number of amendments. A LogicError on startup will verify this. -static constexpr std::size_t numFeatures = 65; +static constexpr std::size_t numFeatures = 71; /** Amendments that this server supports and the default voting behavior. Whether they are enabled depends on the Rules defined in the validated @@ -352,6 +352,12 @@ extern uint256 const featureXChainBridge; extern uint256 const fixDisallowIncomingV1; extern uint256 const featureDID; extern uint256 const fixFillOrKill; +extern uint256 const fixNFTokenReserve; +extern uint256 const fixInnerObjTemplate; +extern uint256 const fixAMMOverflowOffer; +extern uint256 const featurePriceOracle; +extern uint256 const fixEmptyDID; +extern uint256 const fixXChainRewardRounding; } // namespace ripple diff --git a/src/ripple/protocol/Indexes.h b/src/ripple/protocol/Indexes.h index 9a330b6b4f0..d83599f892f 100644 --- a/src/ripple/protocol/Indexes.h +++ b/src/ripple/protocol/Indexes.h @@ -283,6 +283,9 @@ xChainCreateAccountClaimID(STXChainBridge const& bridge, std::uint64_t seq); Keylet did(AccountID const& account) noexcept; +Keylet +oracle(AccountID const& account, std::uint32_t const& documentID) noexcept; + } // namespace keylet // Everything below is deprecated and should be removed in favor of keylets: diff --git a/src/ripple/protocol/LedgerFormats.h b/src/ripple/protocol/LedgerFormats.h index db64942790a..e0ea7bf6f46 100644 --- a/src/ripple/protocol/LedgerFormats.h +++ b/src/ripple/protocol/LedgerFormats.h @@ -192,6 +192,11 @@ enum LedgerEntryType : std::uint16_t */ ltDID = 0x0049, + /** A ledger object which tracks Oracle + \sa keylet::oracle + */ + ltORACLE = 0x0080, + //--------------------------------------------------------------------------- /** A special type, matching any ledger entry type. diff --git a/src/ripple/protocol/MultiApiJson.h b/src/ripple/protocol/MultiApiJson.h new file mode 100644 index 00000000000..b6d1843ae69 --- /dev/null +++ b/src/ripple/protocol/MultiApiJson.h @@ -0,0 +1,247 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_JSON_MULTIAPIJSON_H_INCLUDED +#define RIPPLE_JSON_MULTIAPIJSON_H_INCLUDED + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +namespace detail { +template +constexpr bool is_integral_constant = false; +template +constexpr bool is_integral_constant&> = true; +template +constexpr bool is_integral_constant const&> = true; + +template +concept some_integral_constant = detail::is_integral_constant; + +// This class is designed to wrap a collection of _almost_ identical Json::Value +// objects, indexed by version (i.e. there is some mapping of version to object +// index). It is used e.g. when we need to publish JSON data to users supporting +// different API versions. We allow manipulation and inspection of all objects +// at once with `isMember` and `set`, and also individual inspection and updates +// of an object selected by the user by version, using `visitor_t` nested type. +template +struct MultiApiJson +{ + static_assert(MinVer <= MaxVer); + + static constexpr auto + valid(unsigned int v) noexcept -> bool + { + return v >= MinVer && v <= MaxVer; + } + + static constexpr auto + index(unsigned int v) noexcept -> std::size_t + { + return (v < MinVer) ? 0 : static_cast(v - MinVer); + } + + constexpr static std::size_t size = MaxVer + 1 - MinVer; + std::array val = {}; + + explicit MultiApiJson(Json::Value const& init = {}) + { + if (init == Json::Value{}) + return; // All elements are already default-initialized + for (auto& v : val) + v = init; + } + + void + set(const char* key, + auto const& + v) requires std::constructible_from + { + for (auto& a : this->val) + a[key] = v; + } + + // Intentionally not using class enum here, MultivarJson is scope enough + enum IsMemberResult : int { none = 0, some, all }; + + [[nodiscard]] IsMemberResult + isMember(const char* key) const + { + int count = 0; + for (auto& a : this->val) + if (a.isMember(key)) + count += 1; + + return (count == 0 ? none : (count < size ? some : all)); + } + + static constexpr struct visitor_t final + { + // integral_constant version, extra arguments + template < + typename Json, + unsigned int Version, + typename... Args, + typename Fn> + requires std::same_as, MultiApiJson> auto + operator()( + Json& json, + std::integral_constant const version, + Fn fn, + Args&&... args) const + -> std::invoke_result_t< + Fn, + decltype(json.val[0]), + std::integral_constant, + Args&&...> + { + static_assert( + valid(Version) && index(Version) >= 0 && index(Version) < size); + return std::invoke( + fn, + json.val[index(Version)], + version, + std::forward(args)...); + } + + // integral_constant version, Json only + template + requires std::same_as, MultiApiJson> auto + operator()( + Json& json, + std::integral_constant const, + Fn fn) const -> std::invoke_result_t + { + static_assert( + valid(Version) && index(Version) >= 0 && index(Version) < size); + return std::invoke(fn, json.val[index(Version)]); + } + + // unsigned int version, extra arguments + template < + typename Json, + typename Version, + typename... Args, + typename Fn> + requires(!some_integral_constant) && + std::convertible_to&& std::same_as< + std::remove_cvref_t, + MultiApiJson> auto + operator()(Json& json, Version version, Fn fn, Args&&... args) const + -> std:: + invoke_result_t + { + assert( + valid(version) && index(version) >= 0 && index(version) < size); + return std::invoke( + fn, + json.val[index(version)], + version, + std::forward(args)...); + } + + // unsigned int version, Json only + template + requires(!some_integral_constant) && + std::convertible_to&& std:: + same_as, MultiApiJson> auto + operator()(Json& json, Version version, Fn fn) const + -> std::invoke_result_t + { + assert( + valid(version) && index(version) >= 0 && index(version) < size); + return std::invoke(fn, json.val[index(version)]); + } + } visitor = {}; + + auto + visit() + { + return [self = this](auto... args) requires requires + { + visitor( + std::declval(), + std::declval()...); + } + { + return visitor(*self, std::forward(args)...); + }; + } + + auto + visit() const + { + return [self = this](auto... args) requires requires + { + visitor( + std::declval(), + std::declval()...); + } + { + return visitor(*self, std::forward(args)...); + }; + } + + template + auto + visit(Args... args) + -> std::invoke_result_t requires( + sizeof...(args) > 0) && + requires + { + visitor(*this, std::forward(args)...); + } + { + return visitor(*this, std::forward(args)...); + } + + template + auto + visit(Args... args) const -> std:: + invoke_result_t requires( + sizeof...(args) > 0) && + requires + { + visitor(*this, std::forward(args)...); + } + { + return visitor(*this, std::forward(args)...); + } +}; + +} // namespace detail + +// Wrapper for Json for all supported API versions. +using MultiApiJson = detail:: + MultiApiJson; + +} // namespace ripple + +#endif diff --git a/src/ripple/protocol/Protocol.h b/src/ripple/protocol/Protocol.h index 49642efc4cf..bd723627494 100644 --- a/src/ripple/protocol/Protocol.h +++ b/src/ripple/protocol/Protocol.h @@ -109,6 +109,31 @@ using TxID = uint256; */ std::uint16_t constexpr maxDeletableAMMTrustLines = 512; +/** The maximum length of a URI inside an Oracle */ +std::size_t constexpr maxOracleURI = 256; + +/** The maximum length of a Provider inside an Oracle */ +std::size_t constexpr maxOracleProvider = 256; + +/** The maximum size of a data series array inside an Oracle */ +std::size_t constexpr maxOracleDataSeries = 10; + +/** The maximum length of a SymbolClass inside an Oracle */ +std::size_t constexpr maxOracleSymbolClass = 16; + +/** The maximum allowed time difference between lastUpdateTime and the time + of the last closed ledger +*/ +std::size_t constexpr maxLastUpdateTimeDelta = 300; + +/** The maximum price scaling factor + */ +std::size_t constexpr maxPriceScale = 20; + +/** The maximum percentage of outliers to trim + */ +std::size_t constexpr maxTrim = 25; + } // namespace ripple #endif diff --git a/src/ripple/protocol/PublicKey.h b/src/ripple/protocol/PublicKey.h index 58394cd82d4..9cf1a456953 100644 --- a/src/ripple/protocol/PublicKey.h +++ b/src/ripple/protocol/PublicKey.h @@ -61,13 +61,17 @@ namespace ripple { class PublicKey { protected: - std::size_t size_ = 0; - std::uint8_t buf_[33]; // should be large enough + // All the constructed public keys are valid, non-empty and contain 33 + // bytes of data. + static constexpr std::size_t size_ = 33; + std::uint8_t buf_[size_]; // should be large enough public: using const_iterator = std::uint8_t const*; - PublicKey() = default; +public: + PublicKey() = delete; + PublicKey(PublicKey const& other); PublicKey& operator=(PublicKey const& other); @@ -115,12 +119,6 @@ class PublicKey return buf_ + size_; } - bool - empty() const noexcept - { - return size_ == 0; - } - Slice slice() const noexcept { @@ -141,8 +139,7 @@ operator<<(std::ostream& os, PublicKey const& pk); inline bool operator==(PublicKey const& lhs, PublicKey const& rhs) { - return lhs.size() == rhs.size() && - std::memcmp(lhs.data(), rhs.data(), rhs.size()) == 0; + return std::memcmp(lhs.data(), rhs.data(), rhs.size()) == 0; } inline bool diff --git a/src/ripple/net/RPCErr.h b/src/ripple/protocol/RPCErr.h similarity index 100% rename from src/ripple/net/RPCErr.h rename to src/ripple/protocol/RPCErr.h diff --git a/src/ripple/protocol/SField.h b/src/ripple/protocol/SField.h index 5d7acb12383..727d531ff40 100644 --- a/src/ripple/protocol/SField.h +++ b/src/ripple/protocol/SField.h @@ -51,6 +51,7 @@ template class STInteger; class STXChainBridge; class STVector256; +class STCurrency; #pragma push_macro("XMACRO") #undef XMACRO @@ -85,6 +86,7 @@ class STVector256; STYPE(STI_UINT512, 23) \ STYPE(STI_ISSUE, 24) \ STYPE(STI_XCHAIN_BRIDGE, 25) \ + STYPE(STI_CURRENCY, 26) \ \ /* high-level types */ \ /* cannot be serialized inside other types */ \ @@ -346,6 +348,7 @@ using SF_UINT512 = TypedField>; using SF_ACCOUNT = TypedField; using SF_AMOUNT = TypedField; using SF_ISSUE = TypedField; +using SF_CURRENCY = TypedField; using SF_VL = TypedField; using SF_VECTOR256 = TypedField; using SF_XCHAIN_BRIDGE = TypedField; @@ -364,6 +367,7 @@ extern SF_UINT8 const sfCloseResolution; extern SF_UINT8 const sfMethod; extern SF_UINT8 const sfTransactionResult; extern SF_UINT8 const sfWasLockingChainSend; +extern SF_UINT8 const sfScale; // 8-bit integers (uncommon) extern SF_UINT8 const sfTickSize; @@ -400,6 +404,7 @@ extern SF_UINT32 const sfTransferRate; extern SF_UINT32 const sfWalletSize; extern SF_UINT32 const sfOwnerCount; extern SF_UINT32 const sfDestinationTag; +extern SF_UINT32 const sfLastUpdateTime; // 32-bit integers (uncommon) extern SF_UINT32 const sfHighQualityIn; @@ -435,6 +440,7 @@ extern SF_UINT32 const sfHookStateCount; extern SF_UINT32 const sfEmitGeneration; extern SF_UINT32 const sfVoteWeight; extern SF_UINT32 const sfFirstNFTokenSequence; +extern SF_UINT32 const sfOracleDocumentID; // 64-bit integers (common) extern SF_UINT64 const sfIndexNext; @@ -459,6 +465,7 @@ extern SF_UINT64 const sfReferenceCount; extern SF_UINT64 const sfXChainClaimID; extern SF_UINT64 const sfXChainAccountCreateCount; extern SF_UINT64 const sfXChainAccountClaimCount; +extern SF_UINT64 const sfAssetPrice; // 128-bit extern SF_UINT128 const sfEmailHash; @@ -554,6 +561,8 @@ extern SF_VL const sfMemoData; extern SF_VL const sfMemoFormat; extern SF_VL const sfDIDDocument; extern SF_VL const sfData; +extern SF_VL const sfAssetClass; +extern SF_VL const sfProvider; // variable length (uncommon) extern SF_VL const sfFulfillment; @@ -590,6 +599,10 @@ extern SF_ACCOUNT const sfIssuingChainDoor; // path set extern SField const sfPaths; +// currency +extern SF_CURRENCY const sfBaseAsset; +extern SF_CURRENCY const sfQuoteAsset; + // issue extern SF_ISSUE const sfAsset; extern SF_ISSUE const sfAsset2; @@ -623,6 +636,7 @@ extern SField const sfHook; extern SField const sfVoteEntry; extern SField const sfAuctionSlot; extern SField const sfAuthAccount; +extern SField const sfPriceData; extern SField const sfSigner; extern SField const sfMajority; @@ -651,6 +665,7 @@ extern SField const sfNFTokens; extern SField const sfHooks; extern SField const sfVoteSlots; extern SField const sfAuthAccounts; +extern SField const sfPriceDataSeries; // array of objects (uncommon) extern SField const sfMajorities; diff --git a/src/ripple/protocol/SOTemplate.h b/src/ripple/protocol/SOTemplate.h index 56dcf4492f7..609c2d2c80b 100644 --- a/src/ripple/protocol/SOTemplate.h +++ b/src/ripple/protocol/SOTemplate.h @@ -35,6 +35,8 @@ enum SOEStyle { soeREQUIRED = 0, // required soeOPTIONAL = 1, // optional, may be present with default value soeDEFAULT = 2, // optional, if present, must not have default value + // inner object with the default fields has to be + // constructed with STObject::makeInnerObject() }; //------------------------------------------------------------------------------ diff --git a/src/ripple/protocol/STAccount.h b/src/ripple/protocol/STAccount.h index c622a7c5eef..c44327fe566 100644 --- a/src/ripple/protocol/STAccount.h +++ b/src/ripple/protocol/STAccount.h @@ -20,13 +20,15 @@ #ifndef RIPPLE_PROTOCOL_STACCOUNT_H_INCLUDED #define RIPPLE_PROTOCOL_STACCOUNT_H_INCLUDED +#include #include #include + #include namespace ripple { -class STAccount final : public STBase +class STAccount final : public STBase, public CountedObject { private: // The original implementation of STAccount kept the value in an STBlob. diff --git a/src/ripple/protocol/STBitString.h b/src/ripple/protocol/STBitString.h index 45d1a3d6f05..decdfa64861 100644 --- a/src/ripple/protocol/STBitString.h +++ b/src/ripple/protocol/STBitString.h @@ -20,6 +20,7 @@ #ifndef RIPPLE_PROTOCOL_STBITSTRING_H_INCLUDED #define RIPPLE_PROTOCOL_STBITSTRING_H_INCLUDED +#include #include #include @@ -30,7 +31,7 @@ namespace ripple { // information of a template parameterized by an unsigned type. This RTTI // information is needed to write gdb pretty printers. template -class STBitString final : public STBase +class STBitString final : public STBase, public CountedObject> { static_assert(Bits > 0, "Number of bits must be positive"); diff --git a/src/ripple/protocol/STBlob.h b/src/ripple/protocol/STBlob.h index 58a61206b1a..3b2731be7f0 100644 --- a/src/ripple/protocol/STBlob.h +++ b/src/ripple/protocol/STBlob.h @@ -21,8 +21,10 @@ #define RIPPLE_PROTOCOL_STBLOB_H_INCLUDED #include +#include #include #include + #include #include #include @@ -30,7 +32,7 @@ namespace ripple { // variable length byte string -class STBlob : public STBase +class STBlob : public STBase, public CountedObject { Buffer value_; @@ -88,7 +90,7 @@ class STBlob : public STBase }; inline STBlob::STBlob(STBlob const& rhs) - : STBase(rhs), value_(rhs.data(), rhs.size()) + : STBase(rhs), CountedObject(rhs), value_(rhs.data(), rhs.size()) { } diff --git a/src/ripple/protocol/STCurrency.h b/src/ripple/protocol/STCurrency.h new file mode 100644 index 00000000000..f855c24832e --- /dev/null +++ b/src/ripple/protocol/STCurrency.h @@ -0,0 +1,138 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_PROTOCOL_STCURRENCY_H_INCLUDED +#define RIPPLE_PROTOCOL_STCURRENCY_H_INCLUDED + +#include +#include +#include +#include +#include + +namespace ripple { + +class STCurrency final : public STBase +{ +private: + Currency currency_{}; + +public: + using value_type = Currency; + + STCurrency() = default; + + explicit STCurrency(SerialIter& sit, SField const& name); + + explicit STCurrency(SField const& name, Currency const& currency); + + explicit STCurrency(SField const& name); + + Currency const& + currency() const; + + Currency const& + value() const noexcept; + + void + setCurrency(Currency const& currency); + + SerializedTypeID + getSType() const override; + + std::string + getText() const override; + + Json::Value getJson(JsonOptions) const override; + + void + add(Serializer& s) const override; + + bool + isEquivalent(const STBase& t) const override; + + bool + isDefault() const override; + +private: + static std::unique_ptr + construct(SerialIter&, SField const& name); + + STBase* + copy(std::size_t n, void* buf) const override; + STBase* + move(std::size_t n, void* buf) override; + + friend class detail::STVar; +}; + +STCurrency +currencyFromJson(SField const& name, Json::Value const& v); + +inline Currency const& +STCurrency::currency() const +{ + return currency_; +} + +inline Currency const& +STCurrency::value() const noexcept +{ + return currency_; +} + +inline void +STCurrency::setCurrency(Currency const& currency) +{ + currency_ = currency; +} + +inline bool +operator==(STCurrency const& lhs, STCurrency const& rhs) +{ + return lhs.currency() == rhs.currency(); +} + +inline bool +operator!=(STCurrency const& lhs, STCurrency const& rhs) +{ + return !operator==(lhs, rhs); +} + +inline bool +operator<(STCurrency const& lhs, STCurrency const& rhs) +{ + return lhs.currency() < rhs.currency(); +} + +inline bool +operator==(STCurrency const& lhs, Currency const& rhs) +{ + return lhs.currency() == rhs; +} + +inline bool +operator<(STCurrency const& lhs, Currency const& rhs) +{ + return lhs.currency() < rhs; +} + +} // namespace ripple + +#endif diff --git a/src/ripple/protocol/STInteger.h b/src/ripple/protocol/STInteger.h index fcd007e4562..aaf0f8c904e 100644 --- a/src/ripple/protocol/STInteger.h +++ b/src/ripple/protocol/STInteger.h @@ -20,12 +20,13 @@ #ifndef RIPPLE_PROTOCOL_STINTEGER_H_INCLUDED #define RIPPLE_PROTOCOL_STINTEGER_H_INCLUDED +#include #include namespace ripple { template -class STInteger : public STBase +class STInteger : public STBase, public CountedObject> { public: using value_type = Integer; diff --git a/src/ripple/protocol/STIssue.h b/src/ripple/protocol/STIssue.h index 80a37b305fc..38ca136e017 100644 --- a/src/ripple/protocol/STIssue.h +++ b/src/ripple/protocol/STIssue.h @@ -28,7 +28,7 @@ namespace ripple { -class STIssue final : public STBase +class STIssue final : public STBase, CountedObject { private: Issue issue_{xrpIssue()}; diff --git a/src/ripple/protocol/STObject.h b/src/ripple/protocol/STObject.h index 3e3862bf6c8..38678f67a55 100644 --- a/src/ripple/protocol/STObject.h +++ b/src/ripple/protocol/STObject.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ namespace ripple { class STArray; +class Rules; inline void throwFieldNotFound(SField const& field) @@ -102,6 +104,9 @@ class STObject : public STBase, public CountedObject STObject(SerialIter&& sit, SField const& name); explicit STObject(SField const& name); + static STObject + makeInnerObject(SField const& name, Rules const& rules); + iterator begin() const; @@ -237,6 +242,8 @@ class STObject : public STBase, public CountedObject getFieldV256(SField const& field) const; const STArray& getFieldArray(SField const& field) const; + const STCurrency& + getFieldCurrency(SField const& field) const; /** Get the value of a field. @param A TypedField built from an SField value representing the desired @@ -339,7 +346,7 @@ class STObject : public STBase, public CountedObject set(std::unique_ptr v); void - set(STBase* v); + set(STBase&& v); void setFieldU8(SField const& field, unsigned char); @@ -366,6 +373,8 @@ class STObject : public STBase, public CountedObject void setFieldIssue(SField const& field, STIssue const&); void + setFieldCurrency(SField const& field, STCurrency const&); + void setFieldPathSet(SField const& field, STPathSet const&); void setFieldV256(SField const& field, STVector256 const& v); diff --git a/src/ripple/protocol/STVector256.h b/src/ripple/protocol/STVector256.h index 87d65036a2a..bf4a1cbec44 100644 --- a/src/ripple/protocol/STVector256.h +++ b/src/ripple/protocol/STVector256.h @@ -20,13 +20,14 @@ #ifndef RIPPLE_PROTOCOL_STVECTOR256_H_INCLUDED #define RIPPLE_PROTOCOL_STVECTOR256_H_INCLUDED +#include #include #include #include namespace ripple { -class STVector256 : public STBase +class STVector256 : public STBase, public CountedObject { std::vector mValue; diff --git a/src/ripple/protocol/STXChainBridge.h b/src/ripple/protocol/STXChainBridge.h index 44cd6a480f7..537a1d160b2 100644 --- a/src/ripple/protocol/STXChainBridge.h +++ b/src/ripple/protocol/STXChainBridge.h @@ -20,6 +20,7 @@ #ifndef RIPPLE_PROTOCOL_STXCHAINBRIDGE_H_INCLUDED #define RIPPLE_PROTOCOL_STXCHAINBRIDGE_H_INCLUDED +#include #include #include #include @@ -29,7 +30,7 @@ namespace ripple { class Serializer; class STObject; -class STXChainBridge final : public STBase +class STXChainBridge final : public STBase, public CountedObject { STAccount lockingChainDoor_{sfLockingChainDoor}; STIssue lockingChainIssue_{sfLockingChainIssue}; diff --git a/src/ripple/protocol/SecretKey.h b/src/ripple/protocol/SecretKey.h index 3026fb9d775..824ae9b1e0f 100644 --- a/src/ripple/protocol/SecretKey.h +++ b/src/ripple/protocol/SecretKey.h @@ -41,7 +41,7 @@ class SecretKey public: using const_iterator = std::uint8_t const*; - SecretKey() = default; + SecretKey() = delete; SecretKey(SecretKey const&) = default; SecretKey& operator=(SecretKey const&) = default; diff --git a/src/ripple/protocol/TER.h b/src/ripple/protocol/TER.h index 23d4fb3ef00..41c23a2d6a8 100644 --- a/src/ripple/protocol/TER.h +++ b/src/ripple/protocol/TER.h @@ -64,7 +64,8 @@ enum TELcodes : TERUnderlyingType { telCAN_NOT_QUEUE_FULL, telWRONG_NETWORK, telREQUIRES_NETWORK_ID, - telNETWORK_ID_MAKES_TX_NON_CANONICAL + telNETWORK_ID_MAKES_TX_NON_CANONICAL, + telENV_RPC_FAILED }; //------------------------------------------------------------------------------ @@ -135,6 +136,9 @@ enum TEMcodes : TERUnderlyingType { temXCHAIN_BRIDGE_BAD_REWARD_AMOUNT, temEMPTY_DID, + + temARRAY_EMPTY, + temARRAY_TOO_LARGE, }; //------------------------------------------------------------------------------ @@ -218,7 +222,6 @@ enum TERcodes : TERUnderlyingType { terQUEUED, // Transaction is being held in TxQ until fee drops terPRE_TICKET, // Ticket is not yet in ledger but might be on its way terNO_AMM, // AMM doesn't exist for the asset pair - terSUBMITTED // Has been submitted async. }; //------------------------------------------------------------------------------ @@ -331,7 +334,11 @@ enum TECcodes : TERUnderlyingType { tecXCHAIN_SELF_COMMIT = 184, tecXCHAIN_BAD_PUBLIC_KEY_ACCOUNT_PAIR = 185, tecXCHAIN_CREATE_ACCOUNT_DISABLED = 186, - tecEMPTY_DID = 187 + tecEMPTY_DID = 187, + tecINVALID_UPDATE_TIME = 188, + tecTOKEN_PAIR_NOT_FOUND = 189, + tecARRAY_EMPTY = 190, + tecARRAY_TOO_LARGE = 191 }; //------------------------------------------------------------------------------ diff --git a/src/ripple/protocol/TxFormats.h b/src/ripple/protocol/TxFormats.h index b12547b0a67..b5afa470f38 100644 --- a/src/ripple/protocol/TxFormats.h +++ b/src/ripple/protocol/TxFormats.h @@ -191,6 +191,12 @@ enum TxType : std::uint16_t ttDID_DELETE = 50, + /** This transaction type creates an Oracle instance */ + ttORACLE_SET = 51, + + /** This transaction type deletes an Oracle instance */ + ttORACLE_DELETE = 52, + /** This system-generated transaction type is used to update the status of the various amendments. For details, see: https://xrpl.org/amendments.html diff --git a/src/ripple/protocol/impl/BuildInfo.cpp b/src/ripple/protocol/impl/BuildInfo.cpp index 8279939e26e..c1907a2e1a6 100644 --- a/src/ripple/protocol/impl/BuildInfo.cpp +++ b/src/ripple/protocol/impl/BuildInfo.cpp @@ -33,7 +33,7 @@ namespace BuildInfo { // and follow the format described at http://semver.org/ //------------------------------------------------------------------------------ // clang-format off -char const* const versionString = "2.0.0-rc3" +char const* const versionString = "2.2.0-b1" // clang-format on #if defined(DEBUG) || defined(SANITIZER) diff --git a/src/ripple/protocol/impl/ErrorCodes.cpp b/src/ripple/protocol/impl/ErrorCodes.cpp index 319bd8e28c2..3af48891c78 100644 --- a/src/ripple/protocol/impl/ErrorCodes.cpp +++ b/src/ripple/protocol/impl/ErrorCodes.cpp @@ -109,7 +109,8 @@ constexpr static ErrorInfo unorderedErrorInfos[]{ {rpcSTREAM_MALFORMED, "malformedStream", "Stream malformed.", 400}, {rpcTOO_BUSY, "tooBusy", "The server is too busy to help you now.", 503}, {rpcTXN_NOT_FOUND, "txnNotFound", "Transaction not found.", 404}, - {rpcUNKNOWN_COMMAND, "unknownCmd", "Unknown method.", 405}}; + {rpcUNKNOWN_COMMAND, "unknownCmd", "Unknown method.", 405}, + {rpcORACLE_MALFORMED, "oracleMalformed", "Oracle request is malformed.", 400}}; // clang-format on // Sort and validate unorderedErrorInfos at compile time. Should be diff --git a/src/ripple/protocol/impl/Feature.cpp b/src/ripple/protocol/impl/Feature.cpp index 25033d4336e..8bd4b7aea27 100644 --- a/src/ripple/protocol/impl/Feature.cpp +++ b/src/ripple/protocol/impl/Feature.cpp @@ -458,7 +458,13 @@ REGISTER_FEATURE(AMM, Supported::yes, VoteBehavior::De REGISTER_FEATURE(XChainBridge, Supported::yes, VoteBehavior::DefaultNo); REGISTER_FIX (fixDisallowIncomingV1, Supported::yes, VoteBehavior::DefaultNo); REGISTER_FEATURE(DID, Supported::yes, VoteBehavior::DefaultNo); -REGISTER_FIX(fixFillOrKill, Supported::yes, VoteBehavior::DefaultNo); +REGISTER_FIX (fixFillOrKill, Supported::yes, VoteBehavior::DefaultNo); +REGISTER_FIX (fixNFTokenReserve, Supported::yes, VoteBehavior::DefaultNo); +REGISTER_FIX (fixInnerObjTemplate, Supported::yes, VoteBehavior::DefaultNo); +REGISTER_FIX (fixAMMOverflowOffer, Supported::yes, VoteBehavior::DefaultYes); +REGISTER_FEATURE(PriceOracle, Supported::yes, VoteBehavior::DefaultNo); +REGISTER_FIX (fixEmptyDID, Supported::yes, VoteBehavior::DefaultNo); +REGISTER_FIX (fixXChainRewardRounding, Supported::yes, VoteBehavior::DefaultNo); // The following amendments are obsolete, but must remain supported // because they could potentially get enabled. diff --git a/src/ripple/protocol/impl/Indexes.cpp b/src/ripple/protocol/impl/Indexes.cpp index 74f6b6492de..0ee52aab297 100644 --- a/src/ripple/protocol/impl/Indexes.cpp +++ b/src/ripple/protocol/impl/Indexes.cpp @@ -72,6 +72,7 @@ enum class LedgerNameSpace : std::uint16_t { XCHAIN_CLAIM_ID = 'Q', XCHAIN_CREATE_ACCOUNT_CLAIM_ID = 'K', DID = 'I', + ORACLE = 'R', // No longer used or supported. Left here to reserve the space // to avoid accidental reuse. @@ -444,6 +445,12 @@ did(AccountID const& account) noexcept return {ltDID, indexHash(LedgerNameSpace::DID, account)}; } +Keylet +oracle(AccountID const& account, std::uint32_t const& documentID) noexcept +{ + return {ltORACLE, indexHash(LedgerNameSpace::ORACLE, account, documentID)}; +} + } // namespace keylet } // namespace ripple diff --git a/src/ripple/protocol/impl/InnerObjectFormats.cpp b/src/ripple/protocol/impl/InnerObjectFormats.cpp index 58f4392f536..edebc57477e 100644 --- a/src/ripple/protocol/impl/InnerObjectFormats.cpp +++ b/src/ripple/protocol/impl/InnerObjectFormats.cpp @@ -25,6 +25,9 @@ namespace ripple { InnerObjectFormats::InnerObjectFormats() { + // inner objects with the default fields have to be + // constructed with STObject::makeInnerObject() + add(sfSignerEntry.jsonName.c_str(), sfSignerEntry.getCode(), { @@ -135,6 +138,15 @@ InnerObjectFormats::InnerObjectFormats() { {sfAccount, soeREQUIRED}, }); + + add(sfPriceData.jsonName.c_str(), + sfPriceData.getCode(), + { + {sfBaseAsset, soeREQUIRED}, + {sfQuoteAsset, soeREQUIRED}, + {sfAssetPrice, soeOPTIONAL}, + {sfScale, soeDEFAULT}, + }); } InnerObjectFormats const& diff --git a/src/ripple/protocol/impl/LedgerFormats.cpp b/src/ripple/protocol/impl/LedgerFormats.cpp index 729ddc1c7bc..26cd7ea69b0 100644 --- a/src/ripple/protocol/impl/LedgerFormats.cpp +++ b/src/ripple/protocol/impl/LedgerFormats.cpp @@ -339,6 +339,22 @@ LedgerFormats::LedgerFormats() {sfPreviousTxnLgrSeq, soeREQUIRED} }, commonFields); + + add(jss::Oracle, + ltORACLE, + { + {sfOwner, soeREQUIRED}, + {sfProvider, soeREQUIRED}, + {sfPriceDataSeries, soeREQUIRED}, + {sfAssetClass, soeREQUIRED}, + {sfLastUpdateTime, soeREQUIRED}, + {sfURI, soeOPTIONAL}, + {sfOwnerNode, soeREQUIRED}, + {sfPreviousTxnID, soeREQUIRED}, + {sfPreviousTxnLgrSeq, soeREQUIRED} + }, + commonFields); + // clang-format on } diff --git a/src/ripple/protocol/impl/PublicKey.cpp b/src/ripple/protocol/impl/PublicKey.cpp index 8ab1bd46cdf..22cb351e61c 100644 --- a/src/ripple/protocol/impl/PublicKey.cpp +++ b/src/ripple/protocol/impl/PublicKey.cpp @@ -24,7 +24,6 @@ #include #include #include -#include namespace ripple { @@ -176,16 +175,19 @@ ed25519Canonical(Slice const& sig) PublicKey::PublicKey(Slice const& slice) { + if (slice.size() < size_) + LogicError( + "PublicKey::PublicKey - Input slice cannot be an undersized " + "buffer"); + if (!publicKeyType(slice)) LogicError("PublicKey::PublicKey invalid type"); - size_ = slice.size(); std::memcpy(buf_, slice.data(), size_); } -PublicKey::PublicKey(PublicKey const& other) : size_(other.size_) +PublicKey::PublicKey(PublicKey const& other) { - if (size_) - std::memcpy(buf_, other.buf_, size_); + std::memcpy(buf_, other.buf_, size_); } PublicKey& @@ -193,9 +195,7 @@ PublicKey::operator=(PublicKey const& other) { if (this != &other) { - size_ = other.size_; - if (size_) - std::memcpy(buf_, other.buf_, size_); + std::memcpy(buf_, other.buf_, size_); } return *this; diff --git a/src/ripple/net/impl/RPCErr.cpp b/src/ripple/protocol/impl/RPCErr.cpp similarity index 97% rename from src/ripple/net/impl/RPCErr.cpp rename to src/ripple/protocol/impl/RPCErr.cpp index 8af2a248c2f..ec0474fe8b0 100644 --- a/src/ripple/net/impl/RPCErr.cpp +++ b/src/ripple/protocol/impl/RPCErr.cpp @@ -17,8 +17,8 @@ */ //============================================================================== -#include #include +#include namespace ripple { diff --git a/src/ripple/protocol/impl/SField.cpp b/src/ripple/protocol/impl/SField.cpp index 027c8ffb9c5..6d034db75ef 100644 --- a/src/ripple/protocol/impl/SField.cpp +++ b/src/ripple/protocol/impl/SField.cpp @@ -91,6 +91,7 @@ CONSTRUCT_UNTYPED_SFIELD(sfMetadata, "Metadata", METADATA CONSTRUCT_TYPED_SFIELD(sfCloseResolution, "CloseResolution", UINT8, 1); CONSTRUCT_TYPED_SFIELD(sfMethod, "Method", UINT8, 2); CONSTRUCT_TYPED_SFIELD(sfTransactionResult, "TransactionResult", UINT8, 3); +CONSTRUCT_TYPED_SFIELD(sfScale, "Scale", UINT8, 4); // 8-bit integers (uncommon) CONSTRUCT_TYPED_SFIELD(sfTickSize, "TickSize", UINT8, 16); @@ -128,6 +129,7 @@ CONSTRUCT_TYPED_SFIELD(sfTransferRate, "TransferRate", UINT32, CONSTRUCT_TYPED_SFIELD(sfWalletSize, "WalletSize", UINT32, 12); CONSTRUCT_TYPED_SFIELD(sfOwnerCount, "OwnerCount", UINT32, 13); CONSTRUCT_TYPED_SFIELD(sfDestinationTag, "DestinationTag", UINT32, 14); +CONSTRUCT_TYPED_SFIELD(sfLastUpdateTime, "LastUpdateTime", UINT32, 15); // 32-bit integers (uncommon) CONSTRUCT_TYPED_SFIELD(sfHighQualityIn, "HighQualityIn", UINT32, 16); @@ -164,6 +166,7 @@ CONSTRUCT_TYPED_SFIELD(sfEmitGeneration, "EmitGeneration", UINT32, // 47 is reserved for LockCount(Hooks) CONSTRUCT_TYPED_SFIELD(sfVoteWeight, "VoteWeight", UINT32, 48); CONSTRUCT_TYPED_SFIELD(sfFirstNFTokenSequence, "FirstNFTokenSequence", UINT32, 50); +CONSTRUCT_TYPED_SFIELD(sfOracleDocumentID, "OracleDocumentID", UINT32, 51); // 64-bit integers (common) CONSTRUCT_TYPED_SFIELD(sfIndexNext, "IndexNext", UINT64, 1); @@ -188,6 +191,7 @@ CONSTRUCT_TYPED_SFIELD(sfReferenceCount, "ReferenceCount", U CONSTRUCT_TYPED_SFIELD(sfXChainClaimID, "XChainClaimID", UINT64, 20); CONSTRUCT_TYPED_SFIELD(sfXChainAccountCreateCount, "XChainAccountCreateCount", UINT64, 21); CONSTRUCT_TYPED_SFIELD(sfXChainAccountClaimCount, "XChainAccountClaimCount", UINT64, 22); +CONSTRUCT_TYPED_SFIELD(sfAssetPrice, "AssetPrice", UINT64, 23); // 128-bit CONSTRUCT_TYPED_SFIELD(sfEmailHash, "EmailHash", UINT128, 1); @@ -300,6 +304,8 @@ CONSTRUCT_TYPED_SFIELD(sfHookParameterName, "HookParameterName", VL, CONSTRUCT_TYPED_SFIELD(sfHookParameterValue, "HookParameterValue", VL, 25); CONSTRUCT_TYPED_SFIELD(sfDIDDocument, "DIDDocument", VL, 26); CONSTRUCT_TYPED_SFIELD(sfData, "Data", VL, 27); +CONSTRUCT_TYPED_SFIELD(sfAssetClass, "AssetClass", VL, 28); +CONSTRUCT_TYPED_SFIELD(sfProvider, "Provider", VL, 29); // account CONSTRUCT_TYPED_SFIELD(sfAccount, "Account", ACCOUNT, 1); @@ -331,6 +337,10 @@ CONSTRUCT_TYPED_SFIELD(sfNFTokenOffers, "NFTokenOffers", VECTOR25 // path set CONSTRUCT_UNTYPED_SFIELD(sfPaths, "Paths", PATHSET, 1); +// currency +CONSTRUCT_TYPED_SFIELD(sfBaseAsset, "BaseAsset", CURRENCY, 1); +CONSTRUCT_TYPED_SFIELD(sfQuoteAsset, "QuoteAsset", CURRENCY, 2); + // issue CONSTRUCT_TYPED_SFIELD(sfLockingChainIssue, "LockingChainIssue", ISSUE, 1); CONSTRUCT_TYPED_SFIELD(sfIssuingChainIssue, "IssuingChainIssue", ISSUE, 2); @@ -379,6 +389,7 @@ CONSTRUCT_UNTYPED_SFIELD(sfXChainClaimAttestationCollectionElement, CONSTRUCT_UNTYPED_SFIELD(sfXChainCreateAccountAttestationCollectionElement, "XChainCreateAccountAttestationCollectionElement", OBJECT, 31); +CONSTRUCT_UNTYPED_SFIELD(sfPriceData, "PriceData", OBJECT, 32); // array of objects // ARRAY/1 is reserved for end of array @@ -406,7 +417,8 @@ CONSTRUCT_UNTYPED_SFIELD(sfXChainClaimAttestations, CONSTRUCT_UNTYPED_SFIELD(sfXChainCreateAccountAttestations, "XChainCreateAccountAttestations", ARRAY, 22); -// 23 and 24 are unused and available for use +// 23 is unused and available for use +CONSTRUCT_UNTYPED_SFIELD(sfPriceDataSeries, "PriceDataSeries", ARRAY, 24); CONSTRUCT_UNTYPED_SFIELD(sfAuthAccounts, "AuthAccounts", ARRAY, 25); // clang-format on diff --git a/src/ripple/protocol/impl/STCurrency.cpp b/src/ripple/protocol/impl/STCurrency.cpp new file mode 100644 index 00000000000..d2bc1b3bea7 --- /dev/null +++ b/src/ripple/protocol/impl/STCurrency.cpp @@ -0,0 +1,114 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +#include + +namespace ripple { + +STCurrency::STCurrency(SField const& name) : STBase{name} +{ +} + +STCurrency::STCurrency(SerialIter& sit, SField const& name) : STBase{name} +{ + currency_ = sit.get160(); +} + +STCurrency::STCurrency(SField const& name, Currency const& currency) + : STBase{name}, currency_{currency} +{ +} + +SerializedTypeID +STCurrency::getSType() const +{ + return STI_CURRENCY; +} + +std::string +STCurrency::getText() const +{ + return to_string(currency_); +} + +Json::Value STCurrency::getJson(JsonOptions) const +{ + return to_string(currency_); +} + +void +STCurrency::add(Serializer& s) const +{ + s.addBitString(currency_); +} + +bool +STCurrency::isEquivalent(const STBase& t) const +{ + const STCurrency* v = dynamic_cast(&t); + return v && (*v == *this); +} + +bool +STCurrency::isDefault() const +{ + return isXRP(currency_); +} + +std::unique_ptr +STCurrency::construct(SerialIter& sit, SField const& name) +{ + return std::make_unique(sit, name); +} + +STBase* +STCurrency::copy(std::size_t n, void* buf) const +{ + return emplace(n, buf, *this); +} + +STBase* +STCurrency::move(std::size_t n, void* buf) +{ + return emplace(n, buf, std::move(*this)); +} + +STCurrency +currencyFromJson(SField const& name, Json::Value const& v) +{ + if (!v.isString()) + { + Throw( + "currencyFromJson currency must be a string Json value"); + } + + auto const currency = to_currency(v.asString()); + if (currency == badCurrency() || currency == noCurrency()) + { + Throw( + "currencyFromJson currency must be a valid currency"); + } + + return STCurrency{name, currency}; +} + +} // namespace ripple diff --git a/src/ripple/protocol/impl/STObject.cpp b/src/ripple/protocol/impl/STObject.cpp index 5bafbcfce54..dbcb47e8794 100644 --- a/src/ripple/protocol/impl/STObject.cpp +++ b/src/ripple/protocol/impl/STObject.cpp @@ -18,10 +18,13 @@ //============================================================================== #include +#include #include +#include #include #include #include +#include #include namespace ripple { @@ -57,6 +60,19 @@ STObject::STObject(SerialIter& sit, SField const& name, int depth) noexcept( set(sit, depth); } +STObject +STObject::makeInnerObject(SField const& name, Rules const& rules) +{ + STObject obj{name}; + if (rules.enabled(fixInnerObjTemplate)) + { + if (SOTemplate const* elements = + InnerObjectFormats::getInstance().findSOTemplateBySField(name)) + obj.set(*elements); + } + return obj; +} + STBase* STObject::copy(std::size_t n, void* buf) const { @@ -627,19 +643,32 @@ STObject::getFieldArray(SField const& field) const return getFieldByConstRef(field, empty); } +STCurrency const& +STObject::getFieldCurrency(SField const& field) const +{ + static STCurrency const empty{}; + return getFieldByConstRef(field, empty); +} + void STObject::set(std::unique_ptr v) { - auto const i = getFieldIndex(v->getFName()); + set(std::move(*v.get())); +} + +void +STObject::set(STBase&& v) +{ + auto const i = getFieldIndex(v.getFName()); if (i != -1) { - v_[i] = std::move(*v); + v_[i] = std::move(v); } else { if (!isFree()) Throw("missing field in templated STObject"); - v_.emplace_back(std::move(*v)); + v_.emplace_back(std::move(v)); } } @@ -709,6 +738,12 @@ STObject::setFieldAmount(SField const& field, STAmount const& v) setFieldUsingAssignment(field, v); } +void +STObject::setFieldCurrency(SField const& field, STCurrency const& v) +{ + setFieldUsingAssignment(field, v); +} + void STObject::setFieldIssue(SField const& field, STIssue const& v) { diff --git a/src/ripple/protocol/impl/STParsedJSON.cpp b/src/ripple/protocol/impl/STParsedJSON.cpp index fb960e6f11e..6727fe7388c 100644 --- a/src/ripple/protocol/impl/STParsedJSON.cpp +++ b/src/ripple/protocol/impl/STParsedJSON.cpp @@ -760,6 +760,19 @@ parseLeaf( } break; + case STI_CURRENCY: + try + { + ret = detail::make_stvar( + currencyFromJson(field, value)); + } + catch (std::exception const&) + { + error = invalid_data(json_name, fieldName); + return ret; + } + break; + default: error = bad_type(json_name, fieldName); return ret; diff --git a/src/ripple/protocol/impl/STVar.cpp b/src/ripple/protocol/impl/STVar.cpp index 2ec55ccaf03..adda165901f 100644 --- a/src/ripple/protocol/impl/STVar.cpp +++ b/src/ripple/protocol/impl/STVar.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -167,6 +168,9 @@ STVar::STVar(SerialIter& sit, SField const& name, int depth) case STI_XCHAIN_BRIDGE: construct(sit, name); return; + case STI_CURRENCY: + construct(sit, name); + return; default: Throw("Unknown object type"); } @@ -228,6 +232,9 @@ STVar::STVar(SerializedTypeID id, SField const& name) case STI_XCHAIN_BRIDGE: construct(name); return; + case STI_CURRENCY: + construct(name); + return; default: Throw("Unknown object type"); } diff --git a/src/ripple/protocol/impl/TER.cpp b/src/ripple/protocol/impl/TER.cpp index 1c2db3feb3b..93bc60a98ba 100644 --- a/src/ripple/protocol/impl/TER.cpp +++ b/src/ripple/protocol/impl/TER.cpp @@ -111,6 +111,10 @@ transResults() MAKE_ERROR(tecXCHAIN_BAD_PUBLIC_KEY_ACCOUNT_PAIR, "Bad public key account pair in an xchain transaction."), MAKE_ERROR(tecXCHAIN_CREATE_ACCOUNT_DISABLED, "This bridge does not support account creation."), MAKE_ERROR(tecEMPTY_DID, "The DID object did not have a URI or DIDDocument field."), + MAKE_ERROR(tecINVALID_UPDATE_TIME, "The Oracle object has invalid LastUpdateTime field."), + MAKE_ERROR(tecTOKEN_PAIR_NOT_FOUND, "Token pair is not found in Oracle object."), + MAKE_ERROR(tecARRAY_EMPTY, "Array is empty."), + MAKE_ERROR(tecARRAY_TOO_LARGE, "Array is too large."), MAKE_ERROR(tefALREADY, "The exact transaction was already in this ledger."), MAKE_ERROR(tefBAD_ADD_AUTH, "Not authorized to add account."), @@ -150,6 +154,7 @@ transResults() MAKE_ERROR(telWRONG_NETWORK, "Transaction specifies a Network ID that differs from that of the local node."), MAKE_ERROR(telREQUIRES_NETWORK_ID, "Transactions submitted to this node/network must include a correct NetworkID field."), MAKE_ERROR(telNETWORK_ID_MAKES_TX_NON_CANONICAL, "Transactions submitted to this node/network must NOT include a NetworkID field."), + MAKE_ERROR(telENV_RPC_FAILED, "Unit test RPC failure."), MAKE_ERROR(temMALFORMED, "Malformed transaction."), MAKE_ERROR(temBAD_AMM_TOKENS, "Malformed: Invalid LPTokens."), @@ -197,6 +202,8 @@ transResults() MAKE_ERROR(temXCHAIN_BRIDGE_NONDOOR_OWNER, "Malformed: Bridge owner must be one of the door accounts."), MAKE_ERROR(temXCHAIN_BRIDGE_BAD_MIN_ACCOUNT_CREATE_AMOUNT, "Malformed: Bad min account create amount."), MAKE_ERROR(temXCHAIN_BRIDGE_BAD_REWARD_AMOUNT, "Malformed: Bad reward amount."), + MAKE_ERROR(temARRAY_EMPTY, "Malformed: Array is empty."), + MAKE_ERROR(temARRAY_TOO_LARGE, "Malformed: Array is too large."), MAKE_ERROR(terRETRY, "Retry transaction."), MAKE_ERROR(terFUNDS_SPENT, "DEPRECATED."), @@ -211,7 +218,6 @@ transResults() MAKE_ERROR(terQUEUED, "Held until escalated fee drops."), MAKE_ERROR(terPRE_TICKET, "Ticket is not yet in ledger."), MAKE_ERROR(terNO_AMM, "AMM doesn't exist for the asset pair."), - MAKE_ERROR(terSUBMITTED, "Transaction has been submitted."), MAKE_ERROR(tesSUCCESS, "The transaction was applied. Only final in a validated ledger."), }; diff --git a/src/ripple/protocol/impl/TxFormats.cpp b/src/ripple/protocol/impl/TxFormats.cpp index 7be8ca741e2..d2bdd4f8aa7 100644 --- a/src/ripple/protocol/impl/TxFormats.cpp +++ b/src/ripple/protocol/impl/TxFormats.cpp @@ -483,6 +483,25 @@ TxFormats::TxFormats() commonFields); add(jss::DIDDelete, ttDID_DELETE, {}, commonFields); + + add(jss::OracleSet, + ttORACLE_SET, + { + {sfOracleDocumentID, soeREQUIRED}, + {sfProvider, soeOPTIONAL}, + {sfURI, soeOPTIONAL}, + {sfAssetClass, soeOPTIONAL}, + {sfLastUpdateTime, soeREQUIRED}, + {sfPriceDataSeries, soeREQUIRED}, + }, + commonFields); + + add(jss::OracleDelete, + ttORACLE_DELETE, + { + {sfOracleDocumentID, soeREQUIRED}, + }, + commonFields); } TxFormats const& diff --git a/src/ripple/protocol/impl/b58_utils.h b/src/ripple/protocol/impl/b58_utils.h new file mode 100644 index 00000000000..c3bb0c03750 --- /dev/null +++ b/src/ripple/protocol/impl/b58_utils.h @@ -0,0 +1,192 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2022 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_PROTOCOL_B58_UTILS_H_INCLUDED +#define RIPPLE_PROTOCOL_B58_UTILS_H_INCLUDED + +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace ripple { + +template +using Result = boost::outcome_v2::result; + +#ifndef _MSC_VER +namespace b58_fast { +namespace detail { + +// This optimizes to what hand written asm would do (single divide) +[[nodiscard]] inline std::tuple +div_rem(std::uint64_t a, std::uint64_t b) +{ + return {a / b, a % b}; +} + +// This optimizes to what hand written asm would do (single multiply) +[[nodiscard]] inline std::tuple +carrying_mul(std::uint64_t a, std::uint64_t b, std::uint64_t carry) +{ + unsigned __int128 const x = a; + unsigned __int128 const y = b; + unsigned __int128 const c = x * y + carry; + return {c & 0xffff'ffff'ffff'ffff, c >> 64}; +} + +[[nodiscard]] inline std::tuple +carrying_add(std::uint64_t a, std::uint64_t b) +{ + unsigned __int128 const x = a; + unsigned __int128 const y = b; + unsigned __int128 const c = x + y; + return {c & 0xffff'ffff'ffff'ffff, c >> 64}; +} + +// Add a u64 to a "big uint" value inplace. +// The bigint value is stored with the smallest coefficients first +// (i.e a[0] is the 2^0 coefficient, a[n] is the 2^(64*n) coefficient) +// panics if overflows (this is a specialized adder for b58 decoding. +// it should never overflow). +inline void +inplace_bigint_add(std::span a, std::uint64_t b) +{ + if (a.size() <= 1) + { + ripple::LogicError("Input span too small for inplace_bigint_add"); + } + + std::uint64_t carry; + std::tie(a[0], carry) = carrying_add(a[0], b); + + for (auto& v : a.subspan(1)) + { + if (!carry) + { + return; + } + std::tie(v, carry) = carrying_add(v, 1); + } + if (carry) + { + LogicError("Overflow in inplace_bigint_add"); + } +} + +inline void +inplace_bigint_mul(std::span a, std::uint64_t b) +{ + if (a.empty()) + { + LogicError("Empty span passed to inplace_bigint_mul"); + } + + auto const last_index = a.size() - 1; + if (a[last_index] != 0) + { + LogicError("Non-zero element in inplace_bigint_mul last index"); + } + + std::uint64_t carry = 0; + for (auto& coeff : a.subspan(0, last_index)) + { + std::tie(coeff, carry) = carrying_mul(coeff, b, carry); + } + a[last_index] = carry; +} +// divide a "big uint" value inplace and return the mod +// numerator is stored so smallest coefficients come first +[[nodiscard]] inline std::uint64_t +inplace_bigint_div_rem(std::span numerator, std::uint64_t divisor) +{ + if (numerator.size() == 0) + { + // should never happen, but if it does then it seems natural to define + // the a null set of numbers to be zero, so the remainder is also zero. + assert(0); + return 0; + } + + auto to_u128 = [](std::uint64_t high, + std::uint64_t low) -> unsigned __int128 { + unsigned __int128 const high128 = high; + unsigned __int128 const low128 = low; + return ((high128 << 64) | low128); + }; + auto div_rem_64 = + [](unsigned __int128 num, + std::uint64_t denom) -> std::tuple { + unsigned __int128 const denom128 = denom; + unsigned __int128 const d = num / denom128; + unsigned __int128 const r = num - (denom128 * d); + assert(d >> 64 == 0); + assert(r >> 64 == 0); + return {static_cast(d), static_cast(r)}; + }; + + std::uint64_t prev_rem = 0; + int const last_index = numerator.size() - 1; + std::tie(numerator[last_index], prev_rem) = + div_rem(numerator[last_index], divisor); + for (int i = last_index - 1; i >= 0; --i) + { + unsigned __int128 const cur_num = to_u128(prev_rem, numerator[i]); + std::tie(numerator[i], prev_rem) = div_rem_64(cur_num, divisor); + } + return prev_rem; +} + +// convert from base 58^10 to base 58 +// put largest coeffs first +// the `_be` suffix stands for "big endian" +[[nodiscard]] inline std::array +b58_10_to_b58_be(std::uint64_t input) +{ + constexpr std::uint64_t B_58_10 = 430804206899405824; // 58^10; + if (input >= B_58_10) + { + LogicError("Input to b58_10_to_b58_be equals or exceeds 58^10."); + } + + constexpr std::size_t resultSize = 10; + std::array result{}; + int i = 0; + while (input > 0) + { + std::uint64_t rem; + std::tie(input, rem) = div_rem(input, 58); + result[resultSize - 1 - i] = rem; + i += 1; + } + + return result; +} +} // namespace detail +} // namespace b58_fast +#endif + +} // namespace ripple +#endif // RIPPLE_PROTOCOL_B58_UTILS_H_INCLUDED diff --git a/src/ripple/protocol/impl/token_errors.h b/src/ripple/protocol/impl/token_errors.h new file mode 100644 index 00000000000..59b09974149 --- /dev/null +++ b/src/ripple/protocol/impl/token_errors.h @@ -0,0 +1,101 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2022 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_PROTOCOL_TOKEN_ERRORS_H_INCLUDED +#define RIPPLE_PROTOCOL_TOKEN_ERRORS_H_INCLUDED + +#include + +namespace ripple { +enum class TokenCodecErrc { + success = 0, + inputTooLarge, + inputTooSmall, + badB58Character, + outputTooSmall, + mismatchedTokenType, + mismatchedChecksum, + invalidEncodingChar, + unknown, +}; +} + +namespace std { +template <> +struct is_error_code_enum : true_type +{ +}; +} // namespace std + +namespace ripple { +namespace detail { +class TokenCodecErrcCategory : public std::error_category +{ +public: + // Return a short descriptive name for the category + virtual const char* + name() const noexcept override final + { + return "TokenCodecError"; + } + // Return what each enum means in text + virtual std::string + message(int c) const override final + { + switch (static_cast(c)) + { + case TokenCodecErrc::success: + return "conversion successful"; + case TokenCodecErrc::inputTooLarge: + return "input too large"; + case TokenCodecErrc::inputTooSmall: + return "input too small"; + case TokenCodecErrc::badB58Character: + return "bad base 58 character"; + case TokenCodecErrc::outputTooSmall: + return "output too small"; + case TokenCodecErrc::mismatchedTokenType: + return "mismatched token type"; + case TokenCodecErrc::mismatchedChecksum: + return "mismatched checksum"; + case TokenCodecErrc::invalidEncodingChar: + return "invalid encoding char"; + case TokenCodecErrc::unknown: + return "unknown"; + default: + return "unknown"; + } + } +}; +} // namespace detail + +inline const ripple::detail::TokenCodecErrcCategory& +TokenCodecErrcCategory() +{ + static ripple::detail::TokenCodecErrcCategory c; + return c; +} + +inline std::error_code +make_error_code(ripple::TokenCodecErrc e) +{ + return {static_cast(e), TokenCodecErrcCategory()}; +} +} // namespace ripple +#endif // TOKEN_ERRORS_H_ diff --git a/src/ripple/protocol/impl/tokens.cpp b/src/ripple/protocol/impl/tokens.cpp index 816d49e40df..8445eec38ca 100644 --- a/src/ripple/protocol/impl/tokens.cpp +++ b/src/ripple/protocol/impl/tokens.cpp @@ -16,11 +16,25 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== +// +/* The base58 encoding & decoding routines in the b58_ref namespace are taken + * from Bitcoin but have been modified from the original. + * + * Copyright (c) 2014 The Bitcoin Core developers + * Distributed under the MIT software license, see the accompanying + * file COPYING or http://www.opensource.org/licenses/mit-license.php. + */ + +#include #include #include -#include +#include + #include +#include +#include + #include #include #include @@ -28,6 +42,97 @@ #include #include +/* +Converting between bases is straight forward. First, some background: + +Given the coefficients C[m], ... ,C[0] and base B, those coefficients represent +the number C[m]*B^m + ... + C[0]*B^0; The following pseudo-code converts the +coefficients to the (infinite precision) integer N: + +``` +N = 0; +i = m ;; N.B. m is the index of the largest coefficient +while (i>=0) + N = N + C[i]*B^i + i = i - 1 +``` + +For example, in base 10, the number 437 represents the integer 4*10^2 + 3*10^1 + +7*10^0. In base 16, 437 is the same as 4*16^2 + 3*16^1 + 7*16^0. + +To find the coefficients that represent the integer N in base B, we start by +computing the lowest order coefficients and work up to the highest order +coefficients. The following pseudo-code converts the (infinite precision) +integer N to the correct coefficients: + +``` +i = 0 +while(N) + C[i] = N mod B + N = floor(N/B) + i = i + 1 +``` + +For example, to find the coefficients of the integer 437 in base 10: + +C[0] is 437 mod 10; C[0] = 7; +N is floor(437/10); N = 43; +C[1] is 43 mod 10; C[1] = 3; +N is floor(43/10); N = 4; +C[2] is 4 mod 10; C[2] = 4; +N is floor(4/10); N = 0; +Since N is 0, the algorithm stops. + + +To convert between a number represented with coefficients from base B1 to that +same number represented with coefficients from base B2, we can use the algorithm +that converts coefficients from base B1 to an integer, and then use the +algorithm that converts a number to coefficients from base B2. + +There is a useful shortcut that can be used if one of the bases is a power of +the other base. If B1 == B2^G, then each coefficient from base B1 can be +converted to base B2 independently to create a a group of "G" B2 coefficient. +These coefficients can be simply concatenated together. Since 16 == 2^4, this +property is what makes base 16 useful when dealing with binary numbers. For +example consider converting the base 16 number "93" to binary. The base 16 +coefficient 9 is represented in base 2 with the coefficients 1,0,0,1. The base +16 coefficient 3 is represented in base 2 with the coefficients 0,0,1,1. To get +the final answer, just concatenate those two independent conversions together. +The base 16 number "93" is the binary number "10010011". + +The original (now reference) algorithm to convert from base 58 to a binary +number used the + +``` +N = 0; +for i in m to 0 inclusive + N = N + C[i]*B^i +``` + +algorithm. + +However, the algorithm above is pseudo-code. In particular, the variable "N" is +an infinite precision integer in that pseudo-code. Real computers do +computations on registers, and these registers have limited length. Modern +computers use 64-bit general purpose registers, and can multiply two 64 bit +numbers and obtain a 128 bit result (in two registers). + +The original algorithm in essence converted from base 58 to base 256 (base +2^8). The new, faster algorithm converts from base 58 to base 58^10 (this is +fast using the shortcut described above), then from base 58^10 to base 2^64 +(this is slow, and requires multi-precision arithmetic), and then from base 2^64 +to base 2^8 (this is fast, using the shortcut described above). Base 58^10 is +chosen because it is the largest power of 58 that will fit into a 64-bit +register. + +While it may seem counter-intuitive that converting from base 58 -> base 58^10 +-> base 2^64 -> base 2^8 is faster than directly converting from base 58 -> base +2^8, it is actually 10x-15x faster. The reason for the speed increase is two of +the conversions are trivial (converting between bases where one base is a power +of another base), and doing the multi-precision computations with larger +coefficients sizes greatly speeds up the multi-precision computations. +*/ + namespace ripple { static constexpr char const* alphabetForward = @@ -86,16 +191,31 @@ checksum(void* out, void const* message, std::size_t size) std::memcpy(out, h.data(), 4); } +[[nodiscard]] std::string +encodeBase58Token(TokenType type, void const* token, std::size_t size) +{ +#ifndef _MSC_VER + return b58_fast::encodeBase58Token(type, token, size); +#else + return b58_ref::encodeBase58Token(type, token, size); +#endif +} + +[[nodiscard]] std::string +decodeBase58Token(std::string const& s, TokenType type) +{ +#ifndef _MSC_VER + return b58_fast::decodeBase58Token(s, type); +#else + return b58_ref::decodeBase58Token(s, type); +#endif +} + +namespace b58_ref { + namespace detail { -/* The base58 encoding & decoding routines in this namespace are taken from - * Bitcoin but have been modified from the original. - * - * Copyright (c) 2014 The Bitcoin Core developers - * Distributed under the MIT software license, see the accompanying - * file COPYING or http://www.opensource.org/licenses/mit-license.php. - */ -static std::string +std::string encodeBase58( void const* message, std::size_t size, @@ -146,7 +266,7 @@ encodeBase58( return str; } -static std::string +std::string decodeBase58(std::string const& s) { auto psz = reinterpret_cast(s.c_str()); @@ -241,5 +361,367 @@ decodeBase58Token(std::string const& s, TokenType type) // Skip the leading type byte and the trailing checksum. return ret.substr(1, ret.size() - 1 - guard.size()); } +} // namespace b58_ref + +#ifndef _MSC_VER +// The algorithms use gcc's int128 (fast MS version will have to wait, in the +// meantime MS falls back to the slower reference implementation) +namespace b58_fast { +namespace detail { +// Note: both the input and output will be BIG ENDIAN +B58Result> +b256_to_b58_be(std::span input, std::span out) +{ + // Max valid input is 38 bytes: + // (33 bytes for nodepublic + 1 byte token + 4 bytes checksum) + if (input.size() > 38) + { + return Unexpected(TokenCodecErrc::inputTooLarge); + }; + + auto count_leading_zeros = + [](std::span const& col) -> std::size_t { + std::size_t count = 0; + for (auto const& c : col) + { + if (c != 0) + { + return count; + } + count += 1; + } + return count; + }; + + auto const input_zeros = count_leading_zeros(input); + input = input.subspan(input_zeros); + + // Allocate enough base 2^64 coeff for encoding 38 bytes + // log(2^(38*8),2^64)) ~= 4.75. So 5 coeff are enough + std::array base_2_64_coeff_buf{}; + std::span const base_2_64_coeff = + [&]() -> std::span { + // convert input from big endian to native u64, lowest coeff first + std::size_t num_coeff = 0; + for (int i = 0; i < base_2_64_coeff_buf.size(); ++i) + { + if (i * 8 >= input.size()) + { + break; + } + auto const src_i_end = input.size() - i * 8; + if (src_i_end >= 8) + { + std::memcpy( + &base_2_64_coeff_buf[num_coeff], &input[src_i_end - 8], 8); + boost::endian::big_to_native_inplace( + base_2_64_coeff_buf[num_coeff]); + } + else + { + std::uint64_t be = 0; + for (int bi = 0; bi < src_i_end; ++bi) + { + be <<= 8; + be |= input[bi]; + } + base_2_64_coeff_buf[num_coeff] = be; + }; + num_coeff += 1; + } + return std::span(base_2_64_coeff_buf.data(), num_coeff); + }(); + + // Allocate enough base 58^10 coeff for encoding 38 bytes + // log(2^(38*8),58^10)) ~= 5.18. So 6 coeff are enough + std::array base_58_10_coeff{}; + constexpr std::uint64_t B_58_10 = 430804206899405824; // 58^10; + std::size_t num_58_10_coeffs = 0; + std::size_t cur_2_64_end = base_2_64_coeff.size(); + // compute the base 58^10 coeffs + while (cur_2_64_end > 0) + { + base_58_10_coeff[num_58_10_coeffs] = + ripple::b58_fast::detail::inplace_bigint_div_rem( + base_2_64_coeff.subspan(0, cur_2_64_end), B_58_10); + num_58_10_coeffs += 1; + if (base_2_64_coeff[cur_2_64_end - 1] == 0) + { + cur_2_64_end -= 1; + } + } + + // Translate the result into the alphabet + // Put all the zeros at the beginning, then all the values from the output + std::fill( + out.begin(), out.begin() + input_zeros, ::ripple::alphabetForward[0]); + + // iterate through the base 58^10 coeff + // convert to base 58 big endian then + // convert to alphabet big endian + bool skip_zeros = true; + auto out_index = input_zeros; + for (int i = num_58_10_coeffs - 1; i >= 0; --i) + { + if (skip_zeros && base_58_10_coeff[i] == 0) + { + continue; + } + std::array const b58_be = + ripple::b58_fast::detail::b58_10_to_b58_be(base_58_10_coeff[i]); + std::size_t to_skip = 0; + std::span b58_be_s{b58_be.data(), b58_be.size()}; + if (skip_zeros) + { + to_skip = count_leading_zeros(b58_be_s); + skip_zeros = false; + if (out.size() < (i + 1) * 10 - to_skip) + { + return Unexpected(TokenCodecErrc::outputTooSmall); + } + } + for (auto b58_coeff : b58_be_s.subspan(to_skip)) + { + out[out_index] = ::ripple::alphabetForward[b58_coeff]; + out_index += 1; + } + } + + return out.subspan(0, out_index); +} + +// Note the input is BIG ENDIAN (some fn in this module use little endian) +B58Result> +b58_to_b256_be(std::string_view input, std::span out) +{ + // Convert from b58 to b 58^10 + + // Max encoded value is 38 bytes + // log(2^(38*8),58) ~= 51.9 + if (input.size() > 52) + { + return Unexpected(TokenCodecErrc::inputTooLarge); + }; + if (out.size() < 8) + { + return Unexpected(TokenCodecErrc::outputTooSmall); + } + + auto count_leading_zeros = [&](auto const& col) -> std::size_t { + std::size_t count = 0; + for (auto const& c : col) + { + if (c != ::ripple::alphabetForward[0]) + { + return count; + } + count += 1; + } + return count; + }; + + auto const input_zeros = count_leading_zeros(input); + + // Allocate enough base 58^10 coeff for encoding 38 bytes + // (33 bytes for nodepublic + 1 byte token + 4 bytes checksum) + // log(2^(38*8),58^10)) ~= 5.18. So 6 coeff are enough + std::array b_58_10_coeff{}; + auto [num_full_coeffs, partial_coeff_len] = + ripple::b58_fast::detail::div_rem(input.size(), 10); + auto const num_partial_coeffs = partial_coeff_len ? 1 : 0; + auto const num_b_58_10_coeffs = num_full_coeffs + num_partial_coeffs; + assert(num_b_58_10_coeffs <= b_58_10_coeff.size()); + for (auto c : input.substr(0, partial_coeff_len)) + { + auto cur_val = ::ripple::alphabetReverse[c]; + if (cur_val < 0) + { + return Unexpected(TokenCodecErrc::invalidEncodingChar); + } + b_58_10_coeff[0] *= 58; + b_58_10_coeff[0] += cur_val; + } + for (int i = 0; i < 10; ++i) + { + for (int j = 0; j < num_full_coeffs; ++j) + { + auto c = input[partial_coeff_len + j * 10 + i]; + auto cur_val = ::ripple::alphabetReverse[c]; + if (cur_val < 0) + { + return Unexpected(TokenCodecErrc::invalidEncodingChar); + } + b_58_10_coeff[num_partial_coeffs + j] *= 58; + b_58_10_coeff[num_partial_coeffs + j] += cur_val; + } + } + + constexpr std::uint64_t B_58_10 = 430804206899405824; // 58^10; + + // log(2^(38*8),2^64) ~= 4.75) + std::array result{}; + result[0] = b_58_10_coeff[0]; + std::size_t cur_result_size = 1; + for (int i = 1; i < num_b_58_10_coeffs; ++i) + { + std::uint64_t const c = b_58_10_coeff[i]; + ripple::b58_fast::detail::inplace_bigint_mul( + std::span(&result[0], cur_result_size + 1), B_58_10); + ripple::b58_fast::detail::inplace_bigint_add( + std::span(&result[0], cur_result_size + 1), c); + if (result[cur_result_size] != 0) + { + cur_result_size += 1; + } + } + std::fill(out.begin(), out.begin() + input_zeros, 0); + auto cur_out_i = input_zeros; + // Don't write leading zeros to the output for the most significant + // coeff + { + std::uint64_t const c = result[cur_result_size - 1]; + auto skip_zero = true; + // start and end of output range + for (int i = 0; i < 8; ++i) + { + std::uint8_t const b = (c >> (8 * (7 - i))) & 0xff; + if (skip_zero) + { + if (b == 0) + { + continue; + } + skip_zero = false; + } + out[cur_out_i] = b; + cur_out_i += 1; + } + } + if ((cur_out_i + 8 * (cur_result_size - 1)) > out.size()) + { + return Unexpected(TokenCodecErrc::outputTooSmall); + } + + for (int i = cur_result_size - 2; i >= 0; --i) + { + auto c = result[i]; + boost::endian::native_to_big_inplace(c); + memcpy(&out[cur_out_i], &c, 8); + cur_out_i += 8; + } + + return out.subspan(0, cur_out_i); +} +} // namespace detail + +B58Result> +encodeBase58Token( + TokenType token_type, + std::span input, + std::span out) +{ + constexpr std::size_t tmpBufSize = 128; + std::array buf; + if (input.size() > tmpBufSize - 5) + { + return Unexpected(TokenCodecErrc::inputTooLarge); + } + if (input.size() == 0) + { + return Unexpected(TokenCodecErrc::inputTooSmall); + } + // + buf[0] = static_cast(token_type); + // buf[1..=input.len()] = input; + memcpy(&buf[1], input.data(), input.size()); + size_t const checksum_i = input.size() + 1; + // buf[checksum_i..checksum_i + 4] = checksum + checksum(buf.data() + checksum_i, buf.data(), checksum_i); + std::span b58Span(buf.data(), input.size() + 5); + return detail::b256_to_b58_be(b58Span, out); +} +// Convert from base 58 to base 256, largest coefficients first +// The input is encoded in XPRL format, with the token in the first +// byte and the checksum in the last four bytes. +// The decoded base 256 value does not include the token type or checksum. +// It is an error if the token type or checksum does not match. +B58Result> +decodeBase58Token( + TokenType type, + std::string_view s, + std::span outBuf) +{ + std::array tmpBuf; + auto const decodeResult = + detail::b58_to_b256_be(s, std::span(tmpBuf.data(), tmpBuf.size())); + + if (!decodeResult) + return decodeResult; + + auto const ret = decodeResult.value(); + + // Reject zero length tokens + if (ret.size() < 6) + return Unexpected(TokenCodecErrc::inputTooSmall); + + // The type must match. + if (type != static_cast(static_cast(ret[0]))) + return Unexpected(TokenCodecErrc::mismatchedTokenType); + + // And the checksum must as well. + std::array guard; + checksum(guard.data(), ret.data(), ret.size() - guard.size()); + if (!std::equal(guard.rbegin(), guard.rend(), ret.rbegin())) + { + return Unexpected(TokenCodecErrc::mismatchedChecksum); + } + + std::size_t const outSize = ret.size() - 1 - guard.size(); + if (outBuf.size() < outSize) + return Unexpected(TokenCodecErrc::outputTooSmall); + // Skip the leading type byte and the trailing checksum. + std::copy(ret.begin() + 1, ret.begin() + outSize + 1, outBuf.begin()); + return outBuf.subspan(0, outSize); +} + +[[nodiscard]] std::string +encodeBase58Token(TokenType type, void const* token, std::size_t size) +{ + std::string sr; + // The largest object encoded as base58 is 33 bytes; This will be encoded in + // at most ceil(log(2^256,58)) bytes, or 46 bytes. 128 is plenty (and + // there's not real benefit making it smaller). Note that 46 bytes may be + // encoded in more than 46 base58 chars. Since decode uses 64 as the + // over-allocation, this function uses 128 (again, over-allocation assuming + // 2 base 58 char per byte) + sr.resize(128); + std::span outSp( + reinterpret_cast(sr.data()), sr.size()); + std::span inSp( + reinterpret_cast(token), size); + auto r = b58_fast::encodeBase58Token(type, inSp, outSp); + if (!r) + return {}; + sr.resize(r.value().size()); + return sr; +} + +[[nodiscard]] std::string +decodeBase58Token(std::string const& s, TokenType type) +{ + std::string sr; + // The largest object encoded as base58 is 33 bytes; 64 is plenty (and + // there's no benefit making it smaller) + sr.resize(64); + std::span outSp( + reinterpret_cast(sr.data()), sr.size()); + auto r = b58_fast::decodeBase58Token(type, s, outSp); + if (!r) + return {}; + sr.resize(r.value().size()); + return sr; +} +} // namespace b58_fast +#endif // _MSC_VER } // namespace ripple diff --git a/src/ripple/protocol/jss.h b/src/ripple/protocol/jss.h index 8a701defad8..2655e73ef57 100644 --- a/src/ripple/protocol/jss.h +++ b/src/ripple/protocol/jss.h @@ -60,8 +60,11 @@ JSS(Amount); // in: TransactionSign; field. JSS(Amount2); // in/out: AMM IOU/XRP pool, deposit, withdraw amount JSS(Asset); // in: AMM Asset1 JSS(Asset2); // in: AMM Asset2 +JSS(AssetClass); // in: Oracle +JSS(AssetPrice); // in: Oracle JSS(AuthAccount); // in: AMM Auction Slot JSS(AuthAccounts); // in: AMM Auction Slot +JSS(BaseAsset); // in: Oracle JSS(Bridge); // ledger type. JSS(Check); // ledger type. JSS(CheckCancel); // transaction type. @@ -89,6 +92,7 @@ JSS(Flags); // in/out: TransactionSign; field. JSS(incomplete_shards); // out: OverlayImpl, PeerImp JSS(Invalid); // JSS(LastLedgerSequence); // in: TransactionSign; field +JSS(LastUpdateTime); // field. JSS(LedgerHashes); // ledger type. JSS(LimitAmount); // field. JSS(BidMax); // in: AMM Bid @@ -108,16 +112,26 @@ JSS(Offer); // ledger type. JSS(OfferCancel); // transaction type. JSS(OfferCreate); // transaction type. JSS(OfferSequence); // field. +JSS(Oracle); // ledger type. +JSS(OracleDelete); // transaction type. +JSS(OracleDocumentID); // field +JSS(OracleSet); // transaction type. +JSS(Owner); // field JSS(Paths); // in/out: TransactionSign JSS(PayChannel); // ledger type. JSS(Payment); // transaction type. JSS(PaymentChannelClaim); // transaction type. JSS(PaymentChannelCreate); // transaction type. JSS(PaymentChannelFund); // transaction type. +JSS(PriceDataSeries); // field. +JSS(PriceData); // field. +JSS(Provider); // field. +JSS(QuoteAsset); // in: Oracle. JSS(RippleState); // ledger type. JSS(SLE_hit_rate); // out: GetCounts. JSS(SetFee); // transaction type. JSS(UNLModify); // transaction type. +JSS(Scale); // field. JSS(SettleDelay); // in: TransactionSign JSS(SendMax); // in: TransactionSign JSS(Sequence); // in/out: TransactionSign; field. @@ -135,6 +149,7 @@ JSS(TradingFee); // in/out: AMM trading fee JSS(TransactionType); // in: TransactionSign. JSS(TransferRate); // in: TransferRate. JSS(TrustSet); // transaction type. +JSS(URI); // field. JSS(VoteSlots); // out: AMM Vote JSS(XChainAddAccountCreateAttestation); // transaction type. JSS(XChainAddClaimAttestation); // transaction type. @@ -202,6 +217,7 @@ JSS(avg_bps_sent); // out: Peers JSS(balance); // out: AccountLines JSS(balances); // out: GatewayBalances JSS(base); // out: LogLevel +JSS(base_asset); // in: get_aggregate_price JSS(base_fee); // out: NetworkOPs JSS(base_fee_xrp); // out: NetworkOPs JSS(bids); // out: Subscribe @@ -299,6 +315,7 @@ JSS(enabled); // out: AmendmentTable JSS(engine_result); // out: NetworkOPs, TransactionSign, Submit JSS(engine_result_code); // out: NetworkOPs, TransactionSign, Submit JSS(engine_result_message); // out: NetworkOPs, TransactionSign, Submit +JSS(entire_set); // out: get_aggregate_price JSS(ephemeral_key); // out: ValidatorInfo // in/out: Manifest JSS(error); // out: error @@ -458,6 +475,8 @@ JSS(max_ledger); // in/out: LedgerCleaner JSS(max_queue_size); // out: TxQ JSS(max_spend_drops); // out: AccountInfo JSS(max_spend_drops_total); // out: AccountInfo +JSS(mean); // out: get_aggregate_price +JSS(median); // out: get_aggregate_price JSS(median_fee); // out: TxQ JSS(median_level); // out: TxQ JSS(message); // error. @@ -515,6 +534,9 @@ JSS(open); // out: handlers/Ledger JSS(open_ledger_cost); // out: SubmitTransaction JSS(open_ledger_fee); // out: TxQ JSS(open_ledger_level); // out: TxQ +JSS(oracle); // in: LedgerEntry +JSS(oracles); // in: get_aggregate_price +JSS(oracle_document_id); // in: get_aggregate_price JSS(owner); // in: LedgerEntry, out: NetworkOPs JSS(owner_funds); // in/out: Ledger, NetworkOPs, AcceptedLedgerTx JSS(page_index); @@ -561,6 +583,7 @@ JSS(queue); // in: AccountInfo JSS(queue_data); // out: AccountInfo JSS(queued); // out: SubmitTransaction JSS(queued_duration_us); +JSS(quote_asset); // in: get_aggregate_price JSS(random); // out: Random JSS(raw_meta); // out: AcceptedLedgerTx JSS(receive_currencies); // out: AccountCurrencies @@ -615,12 +638,14 @@ JSS(signing_keys); // out: ValidatorList JSS(signing_time); // out: NetworkOPs JSS(signer_list); // in: AccountObjects JSS(signer_lists); // in/out: AccountInfo +JSS(size); // out: get_aggregate_price JSS(snapshot); // in: Subscribe JSS(source_account); // in: PathRequest, RipplePathFind JSS(source_amount); // in: PathRequest, RipplePathFind JSS(source_currencies); // in: PathRequest, RipplePathFind JSS(source_tag); // out: AccountChannels JSS(stand_alone); // out: NetworkOPs +JSS(standard_deviation); // out: get_aggregate_price JSS(start); // in: TxHistory JSS(started); JSS(state); // out: Logic.h, ServerState, LedgerData @@ -650,9 +675,12 @@ JSS(ticket_count); // out: AccountInfo JSS(ticket_seq); // in: LedgerEntry JSS(time); JSS(timeouts); // out: InboundLedger +JSS(time_threshold); // in/out: Oracle aggregate JSS(time_interval); // out: AMM Auction Slot JSS(track); // out: PeerImp JSS(traffic); // out: Overlay +JSS(trim); // in: get_aggregate_price +JSS(trimmed_set); // out: get_aggregate_price JSS(total); // out: counters JSS(total_bytes_recv); // out: Peers JSS(total_bytes_sent); // out: Peers diff --git a/src/ripple/protocol/tokens.h b/src/ripple/protocol/tokens.h index 0afb4509f41..f51c3f96f95 100644 --- a/src/ripple/protocol/tokens.h +++ b/src/ripple/protocol/tokens.h @@ -20,12 +20,21 @@ #ifndef RIPPLE_PROTOCOL_TOKENS_H_INCLUDED #define RIPPLE_PROTOCOL_TOKENS_H_INCLUDED +#include +#include +#include + #include #include +#include #include +#include namespace ripple { +template +using B58Result = Expected; + enum class TokenType : std::uint8_t { None = 1, // unused NodePublic = 28, @@ -38,11 +47,11 @@ enum class TokenType : std::uint8_t { }; template -std::optional +[[nodiscard]] std::optional parseBase58(std::string const& s); template -std::optional +[[nodiscard]] std::optional parseBase58(TokenType type, std::string const& s); /** Encode data in Base58Check format using XRPL alphabet @@ -56,20 +65,71 @@ parseBase58(TokenType type, std::string const& s); @return the encoded token. */ -std::string +[[nodiscard]] std::string encodeBase58Token(TokenType type, void const* token, std::size_t size); -/** Decode a token of given type encoded using Base58Check and the XRPL alphabet +[[nodiscard]] std::string +decodeBase58Token(std::string const& s, TokenType type); - @param s The encoded token - @param type The type expected for this token. +namespace b58_ref { +// The reference version does not use gcc extensions (int128 in particular) +[[nodiscard]] std::string +encodeBase58Token(TokenType type, void const* token, std::size_t size); + +[[nodiscard]] std::string +decodeBase58Token(std::string const& s, TokenType type); + +namespace detail { +// Expose detail functions for unit tests only +std::string +encodeBase58( + void const* message, + std::size_t size, + void* temp, + std::size_t temp_size); - @return If the encoded token decodes correctly, the token data without - the type or checksum. And empty string otherwise. -*/ std::string +decodeBase58(std::string const& s); +} // namespace detail +} // namespace b58_ref + +#ifndef _MSC_VER +namespace b58_fast { +// Use the fast version (10-15x faster) is using gcc extensions (int128 in +// particular) +[[nodiscard]] B58Result> +encodeBase58Token( + TokenType token_type, + std::span input, + std::span out); + +[[nodiscard]] B58Result> +decodeBase58Token( + TokenType type, + std::string_view s, + std::span outBuf); + +// This interface matches the old interface, but requires additional allocation +[[nodiscard]] std::string +encodeBase58Token(TokenType type, void const* token, std::size_t size); + +// This interface matches the old interface, but requires additional allocation +[[nodiscard]] std::string decodeBase58Token(std::string const& s, TokenType type); +namespace detail { +// Expose detail functions for unit tests only +B58Result> +b256_to_b58_be( + std::span input, + std::span out); + +B58Result> +b58_to_b256_be(std::string_view input, std::span out); +} // namespace detail + +} // namespace b58_fast +#endif // _MSC_VER } // namespace ripple #endif diff --git a/src/ripple/rpc/handlers/AMMInfo.cpp b/src/ripple/rpc/handlers/AMMInfo.cpp index a1be636cafd..b240c8c2ae6 100644 --- a/src/ripple/rpc/handlers/AMMInfo.cpp +++ b/src/ripple/rpc/handlers/AMMInfo.cpp @@ -20,9 +20,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -200,6 +200,9 @@ doAMMInfo(RPC::JsonContext& context) } if (voteSlots.size() > 0) ammResult[jss::vote_slots] = std::move(voteSlots); + assert( + !ledger->rules().enabled(fixInnerObjTemplate) || + amm->isFieldPresent(sfAuctionSlot)); if (amm->isFieldPresent(sfAuctionSlot)) { auto const& auctionSlot = diff --git a/src/ripple/rpc/handlers/AccountChannels.cpp b/src/ripple/rpc/handlers/AccountChannels.cpp index 8f39bef164d..ebd89b04418 100644 --- a/src/ripple/rpc/handlers/AccountChannels.cpp +++ b/src/ripple/rpc/handlers/AccountChannels.cpp @@ -20,9 +20,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/AccountCurrenciesHandler.cpp b/src/ripple/rpc/handlers/AccountCurrenciesHandler.cpp index 64956a7d0ad..45dc8b545ca 100644 --- a/src/ripple/rpc/handlers/AccountCurrenciesHandler.cpp +++ b/src/ripple/rpc/handlers/AccountCurrenciesHandler.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/AccountLines.cpp b/src/ripple/rpc/handlers/AccountLines.cpp index f30a5d4b0a7..3bfcd225b14 100644 --- a/src/ripple/rpc/handlers/AccountLines.cpp +++ b/src/ripple/rpc/handlers/AccountLines.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/AccountObjects.cpp b/src/ripple/rpc/handlers/AccountObjects.cpp index bbf5b6e126a..2531cd03115 100644 --- a/src/ripple/rpc/handlers/AccountObjects.cpp +++ b/src/ripple/rpc/handlers/AccountObjects.cpp @@ -20,10 +20,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/AccountOffers.cpp b/src/ripple/rpc/handlers/AccountOffers.cpp index 1afd2732550..867f888e241 100644 --- a/src/ripple/rpc/handlers/AccountOffers.cpp +++ b/src/ripple/rpc/handlers/AccountOffers.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/AccountTx.cpp b/src/ripple/rpc/handlers/AccountTx.cpp index 40395aae32f..7fe7472721f 100644 --- a/src/ripple/rpc/handlers/AccountTx.cpp +++ b/src/ripple/rpc/handlers/AccountTx.cpp @@ -28,9 +28,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/BookOffers.cpp b/src/ripple/rpc/handlers/BookOffers.cpp index e85b6029ba6..e21d7047f69 100644 --- a/src/ripple/rpc/handlers/BookOffers.cpp +++ b/src/ripple/rpc/handlers/BookOffers.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/Connect.cpp b/src/ripple/rpc/handlers/Connect.cpp index ed366f64b2b..46fac457cb8 100644 --- a/src/ripple/rpc/handlers/Connect.cpp +++ b/src/ripple/rpc/handlers/Connect.cpp @@ -19,9 +19,9 @@ #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/ConsensusInfo.cpp b/src/ripple/rpc/handlers/ConsensusInfo.cpp index f6847e37afd..ee7eb10e684 100644 --- a/src/ripple/rpc/handlers/ConsensusInfo.cpp +++ b/src/ripple/rpc/handlers/ConsensusInfo.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/src/ripple/rpc/handlers/CrawlShards.cpp b/src/ripple/rpc/handlers/CrawlShards.cpp index 87292e57b93..41b74860c3a 100644 --- a/src/ripple/rpc/handlers/CrawlShards.cpp +++ b/src/ripple/rpc/handlers/CrawlShards.cpp @@ -19,10 +19,10 @@ #include #include -#include #include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/DepositAuthorized.cpp b/src/ripple/rpc/handlers/DepositAuthorized.cpp index a5c9c9a21fe..bb6e3b07a9a 100644 --- a/src/ripple/rpc/handlers/DepositAuthorized.cpp +++ b/src/ripple/rpc/handlers/DepositAuthorized.cpp @@ -18,9 +18,9 @@ //============================================================================== #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/DownloadShard.cpp b/src/ripple/rpc/handlers/DownloadShard.cpp index d646379d75f..eacf499df04 100644 --- a/src/ripple/rpc/handlers/DownloadShard.cpp +++ b/src/ripple/rpc/handlers/DownloadShard.cpp @@ -19,9 +19,9 @@ #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/Feature1.cpp b/src/ripple/rpc/handlers/Feature1.cpp index 1858a062e53..94a205e62f7 100644 --- a/src/ripple/rpc/handlers/Feature1.cpp +++ b/src/ripple/rpc/handlers/Feature1.cpp @@ -20,8 +20,9 @@ #include #include #include -#include #include +#include +#include #include #include @@ -37,6 +38,7 @@ doFeature(RPC::JsonContext& context) if (context.app.config().reporting()) return rpcError(rpcREPORTING_UNSUPPORTED); + bool const isAdmin = context.role == Role::ADMIN; // Get majority amendment status majorityAmendments_t majorities; @@ -47,7 +49,7 @@ doFeature(RPC::JsonContext& context) if (!context.params.isMember(jss::feature)) { - auto features = table.getJson(); + auto features = table.getJson(isAdmin); for (auto const& [h, t] : majorities) { @@ -69,13 +71,18 @@ doFeature(RPC::JsonContext& context) if (context.params.isMember(jss::vetoed)) { + if (!isAdmin) + return rpcError(rpcNO_PERMISSION); + if (context.params[jss::vetoed].asBool()) table.veto(feature); else table.unVeto(feature); } - Json::Value jvReply = table.getJson(feature); + Json::Value jvReply = table.getJson(feature, isAdmin); + if (!jvReply) + return rpcError(rpcBAD_FEATURE); auto m = majorities.find(feature); if (m != majorities.end()) diff --git a/src/ripple/rpc/handlers/FetchInfo.cpp b/src/ripple/rpc/handlers/FetchInfo.cpp index 13ead3d3886..79d82b646a0 100644 --- a/src/ripple/rpc/handlers/FetchInfo.cpp +++ b/src/ripple/rpc/handlers/FetchInfo.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/src/ripple/rpc/handlers/GatewayBalances.cpp b/src/ripple/rpc/handlers/GatewayBalances.cpp index fc6a7b49fd8..89be6290f77 100644 --- a/src/ripple/rpc/handlers/GatewayBalances.cpp +++ b/src/ripple/rpc/handlers/GatewayBalances.cpp @@ -20,10 +20,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/GetAggregatePrice.cpp b/src/ripple/rpc/handlers/GetAggregatePrice.cpp new file mode 100644 index 00000000000..5490cc4fcff --- /dev/null +++ b/src/ripple/rpc/handlers/GetAggregatePrice.cpp @@ -0,0 +1,340 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace ripple { + +using namespace boost::bimaps; +// sorted descending by lastUpdateTime, ascending by AssetPrice +using Prices = bimap< + multiset_of>, + multiset_of>; + +/** Calls callback "f" on the ledger-object sle and up to three previous + * metadata objects. Stops early if the callback returns true. + */ +static void +iteratePriceData( + RPC::JsonContext& context, + std::shared_ptr const& sle, + std::function&& f) +{ + using Meta = std::shared_ptr; + constexpr std::uint8_t maxHistory = 3; + bool isNew = false; + std::uint8_t history = 0; + + // `oracle` points to an object that has an `sfPriceDataSeries` field. + // When this function is called, that is a `PriceOracle` ledger object, + // but after one iteration of the loop below, it is an `sfNewFields` + // / `sfFinalFields` object in a `CreatedNode` / `ModifiedNode` object in + // a transaction's metadata. + + // `chain` points to an object that has `sfPreviousTxnID` and + // `sfPreviousTxnLgrSeq` fields. When this function is called, + // that is the `PriceOracle` ledger object pointed to by `oracle`, + // but after one iteration of the loop below, then it is a `ModifiedNode` + // / `CreatedNode` object in a transaction's metadata. + STObject const* oracle = sle.get(); + STObject const* chain = oracle; + // Use to test an unlikely scenario when CreatedNode / ModifiedNode + // for the Oracle is not found in the inner loop + STObject const* prevChain = nullptr; + + Meta meta = nullptr; + while (true) + { + if (prevChain == chain) + return; + + if (!oracle || f(*oracle) || isNew) + return; + + if (++history > maxHistory) + return; + + uint256 prevTx = chain->getFieldH256(sfPreviousTxnID); + std::uint32_t prevSeq = chain->getFieldU32(sfPreviousTxnLgrSeq); + + auto const ledger = context.ledgerMaster.getLedgerBySeq(prevSeq); + if (!ledger) + return; + + meta = ledger->txRead(prevTx).second; + + for (STObject const& node : meta->getFieldArray(sfAffectedNodes)) + { + if (node.getFieldU16(sfLedgerEntryType) != ltORACLE) + { + continue; + } + + prevChain = chain; + chain = &node; + isNew = node.isFieldPresent(sfNewFields); + // if a meta is for the new and this is the first + // look-up then it's the meta for the tx that + // created the current object; i.e. there is no + // historical data + if (isNew && history == 1) + return; + + oracle = isNew + ? &static_cast(node.peekAtField(sfNewFields)) + : &static_cast( + node.peekAtField(sfFinalFields)); + break; + } + } +} + +// Return avg, sd, data set size +static std::tuple +getStats( + Prices::right_const_iterator const& begin, + Prices::right_const_iterator const& end) +{ + STAmount avg{noIssue(), 0, 0}; + Number sd{0}; + std::uint16_t const size = std::distance(begin, end); + avg = std::accumulate( + begin, end, avg, [&](STAmount const& acc, auto const& it) { + return acc + it.first; + }); + avg = divide(avg, STAmount{noIssue(), size, 0}, noIssue()); + if (size > 1) + { + sd = std::accumulate( + begin, end, sd, [&](Number const& acc, auto const& it) { + return acc + (it.first - avg) * (it.first - avg); + }); + sd = root2(sd / (size - 1)); + } + return {avg, sd, size}; +}; + +/** + * oracles: array of {account, oracle_document_id} + * base_asset: is the asset to be priced + * quote_asset: is the denomination in which the prices are expressed + * trim : percentage of outliers to trim [optional] + * time_threshold : defines a range of prices to include based on the timestamp + * range - {most recent, most recent - time_threshold} [optional] + */ +Json::Value +doGetAggregatePrice(RPC::JsonContext& context) +{ + Json::Value result; + auto const& params(context.params); + + constexpr std::uint16_t maxOracles = 200; + if (!params.isMember(jss::oracles)) + return RPC::missing_field_error(jss::oracles); + if (!params[jss::oracles].isArray() || params[jss::oracles].size() == 0 || + params[jss::oracles].size() > maxOracles) + { + RPC::inject_error(rpcORACLE_MALFORMED, result); + return result; + } + + if (!params.isMember(jss::base_asset)) + return RPC::missing_field_error(jss::base_asset); + + if (!params.isMember(jss::quote_asset)) + return RPC::missing_field_error(jss::quote_asset); + + // Lambda to get `trim` and `time_threshold` fields. If the field + // is not included in the input then a default value is returned. + auto getField = [¶ms]( + Json::StaticString const& field, + unsigned int def = + 0) -> std::variant { + if (params.isMember(field)) + { + if (!params[field].isConvertibleTo(Json::ValueType::uintValue)) + return rpcORACLE_MALFORMED; + return params[field].asUInt(); + } + return def; + }; + + auto const trim = getField(jss::trim); + if (std::holds_alternative(trim)) + { + RPC::inject_error(std::get(trim), result); + return result; + } + if (params.isMember(jss::trim) && + (std::get(trim) == 0 || + std::get(trim) > maxTrim)) + { + RPC::inject_error(rpcINVALID_PARAMS, result); + return result; + } + + auto const timeThreshold = getField(jss::time_threshold, 0); + if (std::holds_alternative(timeThreshold)) + { + RPC::inject_error(std::get(timeThreshold), result); + return result; + } + + auto const& baseAsset = params[jss::base_asset]; + auto const& quoteAsset = params[jss::quote_asset]; + + // Collect the dataset into bimap keyed by lastUpdateTime and + // STAmount (Number is int64 and price is uint64) + Prices prices; + for (auto const& oracle : params[jss::oracles]) + { + if (!oracle.isMember(jss::oracle_document_id) || + !oracle.isMember(jss::account)) + { + RPC::inject_error(rpcORACLE_MALFORMED, result); + return result; + } + auto const documentID = oracle[jss::oracle_document_id].isConvertibleTo( + Json::ValueType::uintValue) + ? std::make_optional(oracle[jss::oracle_document_id].asUInt()) + : std::nullopt; + auto const account = + parseBase58(oracle[jss::account].asString()); + if (!account || account->isZero() || !documentID) + { + RPC::inject_error(rpcINVALID_PARAMS, result); + return result; + } + + std::shared_ptr ledger; + result = RPC::lookupLedger(ledger, context); + if (!ledger) + return result; + + auto const sle = ledger->read(keylet::oracle(*account, *documentID)); + iteratePriceData(context, sle, [&](STObject const& node) { + auto const& series = node.getFieldArray(sfPriceDataSeries); + // find the token pair entry with the price + if (auto iter = std::find_if( + series.begin(), + series.end(), + [&](STObject const& o) -> bool { + return o.getFieldCurrency(sfBaseAsset).getText() == + baseAsset && + o.getFieldCurrency(sfQuoteAsset).getText() == + quoteAsset && + o.isFieldPresent(sfAssetPrice); + }); + iter != series.end()) + { + auto const price = iter->getFieldU64(sfAssetPrice); + auto const scale = iter->isFieldPresent(sfScale) + ? -static_cast(iter->getFieldU8(sfScale)) + : 0; + prices.insert(Prices::value_type( + node.getFieldU32(sfLastUpdateTime), + STAmount{noIssue(), price, scale})); + return true; + } + return false; + }); + } + + if (prices.empty()) + { + RPC::inject_error(rpcOBJECT_NOT_FOUND, result); + return result; + } + + // erase outdated data + // sorted in descending, therefore begin is the latest, end is the oldest + auto const latestTime = prices.left.begin()->first; + if (auto const threshold = std::get(timeThreshold)) + { + // threshold defines an acceptable range {max,min} of lastUpdateTime as + // {latestTime, latestTime - threshold}, the prices with lastUpdateTime + // greater than (latestTime - threshold) are erased. + auto const oldestTime = prices.left.rbegin()->first; + auto const upperBound = + latestTime > threshold ? (latestTime - threshold) : oldestTime; + if (upperBound > oldestTime) + prices.left.erase( + prices.left.upper_bound(upperBound), prices.left.end()); + + if (prices.empty()) + { + RPC::inject_error(rpcOBJECT_NOT_FOUND, result); + return result; + } + } + result[jss::time] = latestTime; + + // calculate stats + auto const [avg, sd, size] = + getStats(prices.right.begin(), prices.right.end()); + result[jss::entire_set][jss::mean] = avg.getText(); + result[jss::entire_set][jss::size] = size; + result[jss::entire_set][jss::standard_deviation] = to_string(sd); + + auto itAdvance = [&](auto it, int distance) { + std::advance(it, distance); + return it; + }; + + auto const median = [&prices, &itAdvance, &size_ = size]() { + auto const middle = size_ / 2; + if ((size_ % 2) == 0) + { + static STAmount two{noIssue(), 2, 0}; + auto it = itAdvance(prices.right.begin(), middle - 1); + auto const& a1 = it->first; + auto const& a2 = (++it)->first; + return divide(a1 + a2, two, noIssue()); + } + return itAdvance(prices.right.begin(), middle)->first; + }(); + result[jss::median] = median.getText(); + + if (std::get(trim) != 0) + { + auto const trimCount = + prices.size() * std::get(trim) / 100; + + auto const [avg, sd, size] = getStats( + itAdvance(prices.right.begin(), trimCount), + itAdvance(prices.right.end(), -trimCount)); + result[jss::trimmed_set][jss::mean] = avg.getText(); + result[jss::trimmed_set][jss::size] = size; + result[jss::trimmed_set][jss::standard_deviation] = to_string(sd); + } + + return result; +} + +} // namespace ripple diff --git a/src/ripple/rpc/handlers/GetCounts.cpp b/src/ripple/rpc/handlers/GetCounts.cpp index cf3e7290202..131cf7d3614 100644 --- a/src/ripple/rpc/handlers/GetCounts.cpp +++ b/src/ripple/rpc/handlers/GetCounts.cpp @@ -26,10 +26,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/Handlers.h b/src/ripple/rpc/handlers/Handlers.h index ba93be54513..6c74c5c7e5c 100644 --- a/src/ripple/rpc/handlers/Handlers.h +++ b/src/ripple/rpc/handlers/Handlers.h @@ -73,6 +73,8 @@ doGatewayBalances(RPC::JsonContext&); Json::Value doGetCounts(RPC::JsonContext&); Json::Value +doGetAggregatePrice(RPC::JsonContext&); +Json::Value doLedgerAccept(RPC::JsonContext&); Json::Value doLedgerCleaner(RPC::JsonContext&); diff --git a/src/ripple/rpc/handlers/LedgerAccept.cpp b/src/ripple/rpc/handlers/LedgerAccept.cpp index 3a01a3950e1..14177791164 100644 --- a/src/ripple/rpc/handlers/LedgerAccept.cpp +++ b/src/ripple/rpc/handlers/LedgerAccept.cpp @@ -21,9 +21,9 @@ #include #include #include -#include #include #include +#include #include #include diff --git a/src/ripple/rpc/handlers/LedgerCurrent.cpp b/src/ripple/rpc/handlers/LedgerCurrent.cpp index fa2620642ff..3539c800946 100644 --- a/src/ripple/rpc/handlers/LedgerCurrent.cpp +++ b/src/ripple/rpc/handlers/LedgerCurrent.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/ripple/rpc/handlers/LedgerEntry.cpp b/src/ripple/rpc/handlers/LedgerEntry.cpp index baff721cc1f..dfbc32e606a 100644 --- a/src/ripple/rpc/handlers/LedgerEntry.cpp +++ b/src/ripple/rpc/handlers/LedgerEntry.cpp @@ -20,11 +20,12 @@ #include #include #include +#include #include #include -#include #include #include +#include #include #include #include @@ -598,6 +599,51 @@ doLedgerEntry(RPC::JsonContext& context) else uNodeIndex = keylet::did(*account).key; } + else if (context.params.isMember(jss::oracle)) + { + expectedType = ltORACLE; + if (!context.params[jss::oracle].isObject()) + { + if (!uNodeIndex.parseHex( + context.params[jss::oracle].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } + } + else if ( + !context.params[jss::oracle].isMember( + jss::oracle_document_id) || + !context.params[jss::oracle].isMember(jss::account)) + { + jvResult[jss::error] = "malformedRequest"; + } + else + { + uNodeIndex = beast::zero; + auto const& oracle = context.params[jss::oracle]; + auto const documentID = [&]() -> std::optional { + auto const& id = oracle[jss::oracle_document_id]; + if (id.isConvertibleTo(Json::ValueType::uintValue)) + return std::make_optional(id.asUInt()); + else if (id.isString()) + { + std::uint32_t v; + if (beast::lexicalCastChecked(v, id.asString())) + return std::make_optional(v); + } + return std::nullopt; + }(); + auto const account = + parseBase58(oracle[jss::account].asString()); + if (!account || account->isZero()) + jvResult[jss::error] = "malformedAddress"; + else if (!documentID) + jvResult[jss::error] = "malformedDocumentID"; + else + uNodeIndex = keylet::oracle(*account, *documentID).key; + } + } else { if (context.params.isMember("params") && diff --git a/src/ripple/rpc/handlers/LedgerRequest.cpp b/src/ripple/rpc/handlers/LedgerRequest.cpp index 88d26176ddc..83e7a2184f1 100644 --- a/src/ripple/rpc/handlers/LedgerRequest.cpp +++ b/src/ripple/rpc/handlers/LedgerRequest.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/LogLevel.cpp b/src/ripple/rpc/handlers/LogLevel.cpp index e847007254c..20931898d20 100644 --- a/src/ripple/rpc/handlers/LogLevel.cpp +++ b/src/ripple/rpc/handlers/LogLevel.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/Manifest.cpp b/src/ripple/rpc/handlers/Manifest.cpp index 50a9edf88f6..22abfde8a24 100644 --- a/src/ripple/rpc/handlers/Manifest.cpp +++ b/src/ripple/rpc/handlers/Manifest.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include #include #include @@ -51,14 +51,13 @@ doManifest(RPC::JsonContext& context) // first attempt to use params as ephemeral key, // if this lookup succeeds master key will be returned, - // else pk will just be returned and we will assume that - // is master key anyways + // else an unseated optional is returned auto const mk = context.app.validatorManifests().getMasterKey(*pk); auto const ek = context.app.validatorManifests().getSigningKey(mk); // if ephemeral key not found, we don't have specified manifest - if (ek == mk) + if (!ek) return ret; if (auto const manifest = context.app.validatorManifests().getManifest(mk)) @@ -66,7 +65,7 @@ doManifest(RPC::JsonContext& context) Json::Value details; details[jss::master_key] = toBase58(TokenType::NodePublic, mk); - details[jss::ephemeral_key] = toBase58(TokenType::NodePublic, ek); + details[jss::ephemeral_key] = toBase58(TokenType::NodePublic, *ek); if (auto const seq = context.app.validatorManifests().getSequence(mk)) details[jss::seq] = *seq; diff --git a/src/ripple/rpc/handlers/NFTOffers.cpp b/src/ripple/rpc/handlers/NFTOffers.cpp index 69a090e27ec..bca862d27e3 100644 --- a/src/ripple/rpc/handlers/NFTOffers.cpp +++ b/src/ripple/rpc/handlers/NFTOffers.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/NoRippleCheck.cpp b/src/ripple/rpc/handlers/NoRippleCheck.cpp index 1942372f3e3..91d69df96bd 100644 --- a/src/ripple/rpc/handlers/NoRippleCheck.cpp +++ b/src/ripple/rpc/handlers/NoRippleCheck.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/NodeToShard.cpp b/src/ripple/rpc/handlers/NodeToShard.cpp index fd48797cb2f..552900d1548 100644 --- a/src/ripple/rpc/handlers/NodeToShard.cpp +++ b/src/ripple/rpc/handlers/NodeToShard.cpp @@ -20,9 +20,9 @@ #include #include #include -#include #include #include +#include #include #include diff --git a/src/ripple/rpc/handlers/OwnerInfo.cpp b/src/ripple/rpc/handlers/OwnerInfo.cpp index 2bd9f258da8..546a2e70980 100644 --- a/src/ripple/rpc/handlers/OwnerInfo.cpp +++ b/src/ripple/rpc/handlers/OwnerInfo.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/PathFind.cpp b/src/ripple/rpc/handlers/PathFind.cpp index 6c3a27302ac..9c8794b5997 100644 --- a/src/ripple/rpc/handlers/PathFind.cpp +++ b/src/ripple/rpc/handlers/PathFind.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/PayChanClaim.cpp b/src/ripple/rpc/handlers/PayChanClaim.cpp index 23a4041bb35..33561463f21 100644 --- a/src/ripple/rpc/handlers/PayChanClaim.cpp +++ b/src/ripple/rpc/handlers/PayChanClaim.cpp @@ -20,9 +20,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -55,11 +55,16 @@ doChannelAuthorize(RPC::JsonContext& context) return RPC::missing_field_error(jss::secret); Json::Value result; - auto const [pk, sk] = + std::optional> const keyPair = RPC::keypairForSignature(params, result, context.apiVersion); - if (RPC::contains_error(result)) + + assert(keyPair || RPC::contains_error(result)); + if (!keyPair || RPC::contains_error(result)) return result; + PublicKey const& pk = keyPair->first; + SecretKey const& sk = keyPair->second; + uint256 channelId; if (!channelId.parseHex(params[jss::channel_id].asString())) return rpcError(rpcCHANNEL_MALFORMED); diff --git a/src/ripple/rpc/handlers/Peers.cpp b/src/ripple/rpc/handlers/Peers.cpp index d617da737ea..4f377da7277 100644 --- a/src/ripple/rpc/handlers/Peers.cpp +++ b/src/ripple/rpc/handlers/Peers.cpp @@ -19,10 +19,10 @@ #include #include -#include #include #include #include +#include #include #include diff --git a/src/ripple/rpc/handlers/Random.cpp b/src/ripple/rpc/handlers/Random.cpp index 35120c48452..5362969660a 100644 --- a/src/ripple/rpc/handlers/Random.cpp +++ b/src/ripple/rpc/handlers/Random.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include namespace ripple { diff --git a/src/ripple/rpc/handlers/Reservations.cpp b/src/ripple/rpc/handlers/Reservations.cpp index 0d4ad65ebb7..ffbfd0f98bf 100644 --- a/src/ripple/rpc/handlers/Reservations.cpp +++ b/src/ripple/rpc/handlers/Reservations.cpp @@ -18,9 +18,9 @@ //============================================================================== #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/RipplePathFind.cpp b/src/ripple/rpc/handlers/RipplePathFind.cpp index 5e23a47bd52..19bce998494 100644 --- a/src/ripple/rpc/handlers/RipplePathFind.cpp +++ b/src/ripple/rpc/handlers/RipplePathFind.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/ServerInfo.cpp b/src/ripple/rpc/handlers/ServerInfo.cpp index 2637c3d51be..83c73e20558 100644 --- a/src/ripple/rpc/handlers/ServerInfo.cpp +++ b/src/ripple/rpc/handlers/ServerInfo.cpp @@ -21,8 +21,9 @@ #include #include #include -#include +#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/ServerState.cpp b/src/ripple/rpc/handlers/ServerState.cpp index d974e7391f7..756a3953968 100644 --- a/src/ripple/rpc/handlers/ServerState.cpp +++ b/src/ripple/rpc/handlers/ServerState.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/Submit.cpp b/src/ripple/rpc/handlers/Submit.cpp index 8e998f1ea6c..a151778fbb8 100644 --- a/src/ripple/rpc/handlers/Submit.cpp +++ b/src/ripple/rpc/handlers/Submit.cpp @@ -21,9 +21,8 @@ #include #include #include -#include -#include #include +#include #include #include #include @@ -49,10 +48,6 @@ doSubmit(RPC::JsonContext& context) { context.loadType = Resource::feeMediumBurdenRPC; - auto const sync = RPC::getSubmitSyncMode(context.params); - if (!sync) - return sync.error(); - if (!context.params.isMember(jss::tx_blob)) { auto const failType = getFailHard(context); @@ -68,8 +63,7 @@ doSubmit(RPC::JsonContext& context) context.role, context.ledgerMaster.getValidatedLedgerAge(), context.app, - RPC::getProcessTxnFn(context.netOps), - *sync); + RPC::getProcessTxnFn(context.netOps)); ret[jss::deprecated] = "Signing support in the 'submit' command has been " @@ -138,7 +132,7 @@ doSubmit(RPC::JsonContext& context) auto const failType = getFailHard(context); context.netOps.processTransaction( - tpTrans, isUnlimited(context.role), *sync, true, failType); + tpTrans, isUnlimited(context.role), true, failType); } catch (std::exception& e) { diff --git a/src/ripple/rpc/handlers/SubmitMultiSigned.cpp b/src/ripple/rpc/handlers/SubmitMultiSigned.cpp index 82fa52a4623..5b9d5b34ac6 100644 --- a/src/ripple/rpc/handlers/SubmitMultiSigned.cpp +++ b/src/ripple/rpc/handlers/SubmitMultiSigned.cpp @@ -18,12 +18,10 @@ //============================================================================== #include -#include #include #include #include #include -#include #include namespace ripple { @@ -39,10 +37,6 @@ doSubmitMultiSigned(RPC::JsonContext& context) auto const failHard = context.params[jss::fail_hard].asBool(); auto const failType = NetworkOPs::doFailHard(failHard); - auto const sync = RPC::getSubmitSyncMode(context.params); - if (!sync) - return sync.error(); - return RPC::transactionSubmitMultiSigned( context.params, context.apiVersion, @@ -50,8 +44,7 @@ doSubmitMultiSigned(RPC::JsonContext& context) context.role, context.ledgerMaster.getValidatedLedgerAge(), context.app, - RPC::getProcessTxnFn(context.netOps), - *sync); + RPC::getProcessTxnFn(context.netOps)); } } // namespace ripple diff --git a/src/ripple/rpc/handlers/Subscribe.cpp b/src/ripple/rpc/handlers/Subscribe.cpp index e48cfe5049a..24465a2c3a5 100644 --- a/src/ripple/rpc/handlers/Subscribe.cpp +++ b/src/ripple/rpc/handlers/Subscribe.cpp @@ -22,9 +22,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/Tx.cpp b/src/ripple/rpc/handlers/Tx.cpp index 0237fef22ac..d4c9c95a341 100644 --- a/src/ripple/rpc/handlers/Tx.cpp +++ b/src/ripple/rpc/handlers/Tx.cpp @@ -24,9 +24,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/TxHistory.cpp b/src/ripple/rpc/handlers/TxHistory.cpp index 0f3e353fcbc..8759d4af1b9 100644 --- a/src/ripple/rpc/handlers/TxHistory.cpp +++ b/src/ripple/rpc/handlers/TxHistory.cpp @@ -25,8 +25,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/UnlList.cpp b/src/ripple/rpc/handlers/UnlList.cpp index b3beca28307..0dc4dfc3776 100644 --- a/src/ripple/rpc/handlers/UnlList.cpp +++ b/src/ripple/rpc/handlers/UnlList.cpp @@ -19,8 +19,8 @@ #include #include -#include #include +#include #include #include diff --git a/src/ripple/rpc/handlers/Unsubscribe.cpp b/src/ripple/rpc/handlers/Unsubscribe.cpp index 8a606a26dca..512790f2a10 100644 --- a/src/ripple/rpc/handlers/Unsubscribe.cpp +++ b/src/ripple/rpc/handlers/Unsubscribe.cpp @@ -19,8 +19,8 @@ #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/ValidationCreate.cpp b/src/ripple/rpc/handlers/ValidationCreate.cpp index ae1ee33d4c6..f07cbe9cdfa 100644 --- a/src/ripple/rpc/handlers/ValidationCreate.cpp +++ b/src/ripple/rpc/handlers/ValidationCreate.cpp @@ -18,8 +18,8 @@ //============================================================================== #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/handlers/ValidatorInfo.cpp b/src/ripple/rpc/handlers/ValidatorInfo.cpp index 93859dd65d5..910c5e9740f 100644 --- a/src/ripple/rpc/handlers/ValidatorInfo.cpp +++ b/src/ripple/rpc/handlers/ValidatorInfo.cpp @@ -30,24 +30,23 @@ Json::Value doValidatorInfo(RPC::JsonContext& context) { // return error if not configured as validator - if (context.app.getValidationPublicKey().empty()) + auto const validationPK = context.app.getValidationPublicKey(); + if (!validationPK) return RPC::not_validator_error(); Json::Value ret; - auto const pk = context.app.getValidationPublicKey(); - - // assume pk is ephemeral key, get master key - auto const mk = context.app.validatorManifests().getMasterKey(pk); + // assume validationPK is ephemeral key, get master key + auto const mk = + context.app.validatorManifests().getMasterKey(*validationPK); ret[jss::master_key] = toBase58(TokenType::NodePublic, mk); - // pk is maskter key, eg no ephemeral key, eg no manifest, just return - if (mk == pk) + // validationPK is master key, this implies that there is no ephemeral + // key, no manifest, hence return + if (mk == validationPK) return ret; - // lookup ephemeral key - auto const ek = context.app.validatorManifests().getSigningKey(mk); - ret[jss::ephemeral_key] = toBase58(TokenType::NodePublic, ek); + ret[jss::ephemeral_key] = toBase58(TokenType::NodePublic, *validationPK); if (auto const manifest = context.app.validatorManifests().getManifest(mk)) ret[jss::manifest] = base64_encode(*manifest); diff --git a/src/ripple/rpc/handlers/ValidatorListSites.cpp b/src/ripple/rpc/handlers/ValidatorListSites.cpp index 81972c5a2d4..4800a5e8661 100644 --- a/src/ripple/rpc/handlers/ValidatorListSites.cpp +++ b/src/ripple/rpc/handlers/ValidatorListSites.cpp @@ -19,8 +19,8 @@ #include #include -#include #include +#include #include namespace ripple { diff --git a/src/ripple/rpc/handlers/Validators.cpp b/src/ripple/rpc/handlers/Validators.cpp index bce9a311d13..39306612bdf 100644 --- a/src/ripple/rpc/handlers/Validators.cpp +++ b/src/ripple/rpc/handlers/Validators.cpp @@ -19,8 +19,8 @@ #include #include -#include #include +#include #include namespace ripple { diff --git a/src/ripple/rpc/handlers/WalletPropose.cpp b/src/ripple/rpc/handlers/WalletPropose.cpp index 624c5c83c2b..25aaa3bbb8e 100644 --- a/src/ripple/rpc/handlers/WalletPropose.cpp +++ b/src/ripple/rpc/handlers/WalletPropose.cpp @@ -18,10 +18,10 @@ //============================================================================== #include -#include #include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/impl/DeliveredAmount.cpp b/src/ripple/rpc/impl/DeliveredAmount.cpp index 03a36b9a15f..59316d91cf5 100644 --- a/src/ripple/rpc/impl/DeliveredAmount.cpp +++ b/src/ripple/rpc/impl/DeliveredAmount.cpp @@ -23,9 +23,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/src/ripple/rpc/impl/Handler.cpp b/src/ripple/rpc/impl/Handler.cpp index d05c3279800..1fc160dc4db 100644 --- a/src/ripple/rpc/impl/Handler.cpp +++ b/src/ripple/rpc/impl/Handler.cpp @@ -105,15 +105,19 @@ Handler const handlerArray[]{ Role::USER, NO_CONDITION}, {"download_shard", byRef(&doDownloadShard), Role::ADMIN, NO_CONDITION}, + {"feature", byRef(&doFeature), Role::USER, NO_CONDITION}, + {"fee", byRef(&doFee), Role::USER, NEEDS_CURRENT_LEDGER}, + {"fetch_info", byRef(&doFetchInfo), Role::ADMIN, NO_CONDITION}, #ifdef RIPPLED_REPORTING {"gateway_balances", byRef(&doGatewayBalances), Role::ADMIN, NO_CONDITION}, #else {"gateway_balances", byRef(&doGatewayBalances), Role::USER, NO_CONDITION}, #endif {"get_counts", byRef(&doGetCounts), Role::ADMIN, NO_CONDITION}, - {"feature", byRef(&doFeature), Role::ADMIN, NO_CONDITION}, - {"fee", byRef(&doFee), Role::USER, NEEDS_CURRENT_LEDGER}, - {"fetch_info", byRef(&doFetchInfo), Role::ADMIN, NO_CONDITION}, + {"get_aggregate_price", + byRef(&doGetAggregatePrice), + Role::USER, + NO_CONDITION}, {"ledger_accept", byRef(&doLedgerAccept), Role::ADMIN, diff --git a/src/ripple/rpc/impl/RPCHandler.cpp b/src/ripple/rpc/impl/RPCHandler.cpp index c7984f8309c..9c5aa6d465b 100644 --- a/src/ripple/rpc/impl/RPCHandler.cpp +++ b/src/ripple/rpc/impl/RPCHandler.cpp @@ -31,8 +31,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/src/ripple/rpc/impl/RPCHelpers.cpp b/src/ripple/rpc/impl/RPCHelpers.cpp index 672095fe950..58503b0bf4e 100644 --- a/src/ripple/rpc/impl/RPCHelpers.cpp +++ b/src/ripple/rpc/impl/RPCHelpers.cpp @@ -25,9 +25,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -792,7 +792,7 @@ getSeedFromRPC(Json::Value const& params, Json::Value& error) return seed; } -std::pair +std::optional> keypairForSignature( Json::Value const& params, Json::Value& error, @@ -934,7 +934,7 @@ chooseLedgerEntryType(Json::Value const& params) std::pair result{RPC::Status::OK, ltANY}; if (params.isMember(jss::type)) { - static constexpr std::array, 20> + static constexpr std::array, 21> types{ {{jss::account, ltACCOUNT_ROOT}, {jss::amendments, ltAMENDMENTS}, @@ -956,7 +956,8 @@ chooseLedgerEntryType(Json::Value const& params) {jss::xchain_owned_claim_id, ltXCHAIN_OWNED_CLAIM_ID}, {jss::xchain_owned_create_account_claim_id, ltXCHAIN_OWNED_CREATE_ACCOUNT_CLAIM_ID}, - {jss::did, ltDID}}}; + {jss::did, ltDID}, + {jss::oracle, ltORACLE}}}; auto const& p = params[jss::type]; if (!p.isString()) @@ -1125,26 +1126,5 @@ getLedgerByContext(RPC::JsonContext& context) return RPC::make_error( rpcNOT_READY, "findCreate failed to return an inbound ledger"); } - -ripple::Expected -getSubmitSyncMode(Json::Value const& params) -{ - using enum RPC::SubmitSync; - if (params.isMember(jss::sync_mode)) - { - std::string const syncMode = params[jss::sync_mode].asString(); - if (syncMode == "async") - return async; - else if (syncMode == "wait") - return wait; - else if (syncMode != "sync") - return Unexpected(RPC::make_error( - rpcINVALID_PARAMS, - "sync_mode parameter must be one of \"sync\", \"async\", or " - "\"wait\".")); - } - return sync; -} - } // namespace RPC } // namespace ripple diff --git a/src/ripple/rpc/impl/RPCHelpers.h b/src/ripple/rpc/impl/RPCHelpers.h index 0c2a299d8ab..e003773e50c 100644 --- a/src/ripple/rpc/impl/RPCHelpers.h +++ b/src/ripple/rpc/impl/RPCHelpers.h @@ -22,16 +22,16 @@ #include #include +#include #include #include #include -#include -#include #include #include #include #include + #include #include @@ -209,41 +209,6 @@ extern beast::SemanticVersion const firstVersion; extern beast::SemanticVersion const goodVersion; extern beast::SemanticVersion const lastVersion; -/** - * API version numbers used in later API versions - * - * Requests with a version number in the range - * [apiMinimumSupportedVersion, apiMaximumSupportedVersion] - * are supported. - * - * If [beta_rpc_api] is enabled in config, the version numbers - * in the range [apiMinimumSupportedVersion, apiBetaVersion] - * are supported. - * - * Network Requests without explicit version numbers use - * apiVersionIfUnspecified. apiVersionIfUnspecified is 1, - * because all the RPC requests with a version >= 2 must - * explicitly specify the version in the requests. - * Note that apiVersionIfUnspecified will be lower than - * apiMinimumSupportedVersion when we stop supporting API - * version 1. - * - * Command line Requests use apiMaximumSupportedVersion. - */ - -constexpr unsigned int apiInvalidVersion = 0; -constexpr unsigned int apiVersionIfUnspecified = 1; -constexpr unsigned int apiMinimumSupportedVersion = 1; -constexpr unsigned int apiMaximumSupportedVersion = 2; -constexpr unsigned int apiCommandLineVersion = 1; // TODO Bump to 2 later -constexpr unsigned int apiBetaVersion = 3; -constexpr unsigned int apiMaximumValidVersion = apiBetaVersion; - -static_assert(apiMinimumSupportedVersion >= apiVersionIfUnspecified); -static_assert(apiMaximumSupportedVersion >= apiMinimumSupportedVersion); -static_assert(apiBetaVersion >= apiMaximumSupportedVersion); -static_assert(apiMaximumValidVersion >= apiMaximumSupportedVersion); - template void setVersion(Object& parent, unsigned int apiVersion, bool betaEnabled) @@ -258,7 +223,7 @@ setVersion(Object& parent, unsigned int apiVersion, bool betaEnabled) } else { - object[jss::first] = apiMinimumSupportedVersion; + object[jss::first] = apiMinimumSupportedVersion.value; object[jss::last] = betaEnabled ? apiBetaVersion : apiMaximumSupportedVersion; } @@ -289,19 +254,11 @@ getAPIVersionNumber(const Json::Value& value, bool betaEnabled); std::variant, Json::Value> getLedgerByContext(RPC::JsonContext& context); -std::pair +std::optional> keypairForSignature( Json::Value const& params, Json::Value& error, unsigned int apiVersion = apiVersionIfUnspecified); -/** Helper to parse submit_mode parameter to RPC submit. - * - * @param params RPC parameters - * @return Either the mode or an error object. - */ -ripple::Expected -getSubmitSyncMode(Json::Value const& params); - } // namespace RPC } // namespace ripple diff --git a/src/ripple/rpc/impl/ServerHandler.cpp b/src/ripple/rpc/impl/ServerHandler.cpp index f33ecd625aa..d643bdb6331 100644 --- a/src/ripple/rpc/impl/ServerHandler.cpp +++ b/src/ripple/rpc/impl/ServerHandler.cpp @@ -27,12 +27,13 @@ #include #include #include +#include #include #include #include -#include #include #include +#include #include #include #include @@ -46,9 +47,7 @@ #include #include #include -#include #include -#include #include namespace ripple { @@ -644,7 +643,7 @@ ServerHandler::processRequest( continue; } - auto apiVersion = RPC::apiVersionIfUnspecified; + unsigned apiVersion = RPC::apiVersionIfUnspecified; if (jsonRPC.isMember(jss::params) && jsonRPC[jss::params].isArray() && jsonRPC[jss::params].size() > 0 && jsonRPC[jss::params][0u].isObject()) @@ -1149,6 +1148,12 @@ parse_Ports(Config const& config, std::ostream& log) log << "Missing section: [" << name << "]"; Throw(); } + + // grpc ports are parsed by GRPCServer class. Do not validate + // grpc port information in this file. + if (name == SECTION_PORT_GRPC) + continue; + ParsedPort parsed = common; parsed.name = name; parse_Port(parsed, config[name], log); diff --git a/src/ripple/rpc/impl/TransactionSign.cpp b/src/ripple/rpc/impl/TransactionSign.cpp index 48a9c66d81c..3f4407c86e7 100644 --- a/src/ripple/rpc/impl/TransactionSign.cpp +++ b/src/ripple/rpc/impl/TransactionSign.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -52,35 +53,25 @@ class SigningForParams { private: AccountID const* const multiSigningAcctID_; - PublicKey* const multiSignPublicKey_; - Buffer* const multiSignature_; + std::optional multiSignPublicKey_; + Buffer multiSignature_; public: - explicit SigningForParams() - : multiSigningAcctID_(nullptr) - , multiSignPublicKey_(nullptr) - , multiSignature_(nullptr) + explicit SigningForParams() : multiSigningAcctID_(nullptr) { } SigningForParams(SigningForParams const& rhs) = delete; - SigningForParams( - AccountID const& multiSigningAcctID, - PublicKey& multiSignPublicKey, - Buffer& multiSignature) + SigningForParams(AccountID const& multiSigningAcctID) : multiSigningAcctID_(&multiSigningAcctID) - , multiSignPublicKey_(&multiSignPublicKey) - , multiSignature_(&multiSignature) { } bool isMultiSigning() const { - return ( - (multiSigningAcctID_ != nullptr) && - (multiSignPublicKey_ != nullptr) && (multiSignature_ != nullptr)); + return multiSigningAcctID_ != nullptr; } bool @@ -96,23 +87,46 @@ class SigningForParams return !isMultiSigning(); } + bool + validMultiSign() const + { + return isMultiSigning() && multiSignPublicKey_ && + multiSignature_.size(); + } + // Don't call this method unless isMultiSigning() returns true. AccountID const& - getSigner() + getSigner() const { + if (!multiSigningAcctID_) + LogicError("Accessing unknown SigningForParams::getSigner()"); return *multiSigningAcctID_; } + PublicKey const& + getPublicKey() const + { + if (!multiSignPublicKey_) + LogicError("Accessing unknown SigningForParams::getPublicKey()"); + return *multiSignPublicKey_; + } + + Buffer const& + getSignature() const + { + return multiSignature_; + } + void setPublicKey(PublicKey const& multiSignPublicKey) { - *multiSignPublicKey_ = multiSignPublicKey; + multiSignPublicKey_ = multiSignPublicKey; } void moveMultiSignature(Buffer&& multiSignature) { - *multiSignature_ = std::move(multiSignature); + multiSignature_ = std::move(multiSignature); } }; @@ -385,10 +399,14 @@ transactionPreProcessImpl( auto j = app.journal("RPCHandler"); Json::Value jvResult; - auto const [pk, sk] = keypairForSignature(params, jvResult); - if (contains_error(jvResult)) + std::optional> keyPair = + keypairForSignature(params, jvResult); + if (!keyPair || contains_error(jvResult)) return jvResult; + PublicKey const& pk = keyPair->first; + SecretKey const& sk = keyPair->second; + bool const verify = !(params.isMember(jss::offline) && params[jss::offline].asBool()); @@ -659,6 +677,11 @@ transactionFormatResultImpl(Transaction::pointer tpTrans, unsigned apiVersion) else jvResult[jss::tx_json] = tpTrans->getJson(JsonOptions::none); + RPC::insertDeliverMax( + jvResult[jss::tx_json], + tpTrans->getSTransaction()->getTxnType(), + apiVersion); + jvResult[jss::tx_blob] = strHex(tpTrans->getSTransaction()->getSerializer().peekData()); @@ -828,8 +851,7 @@ transactionSubmit( Role role, std::chrono::seconds validatedLedgerAge, Application& app, - ProcessTransactionFn const& processTransaction, - RPC::SubmitSync sync) + ProcessTransactionFn const& processTransaction) { using namespace detail; @@ -855,7 +877,8 @@ transactionSubmit( // Finally, submit the transaction. try { - processTransaction(txn.second, isUnlimited(role), sync, failType); + // FIXME: For performance, should use asynch interface + processTransaction(txn.second, isUnlimited(role), true, failType); } catch (std::exception&) { @@ -1003,10 +1026,7 @@ transactionSignFor( } // Add and amend fields based on the transaction type. - Buffer multiSignature; - PublicKey multiSignPubKey; - SigningForParams signForParams( - *signerAccountID, multiSignPubKey, multiSignature); + SigningForParams signForParams(*signerAccountID); transactionPreProcessResult preprocResult = transactionPreProcessImpl( jvRequest, role, signForParams, validatedLedgerAge, app); @@ -1014,12 +1034,14 @@ transactionSignFor( if (!preprocResult.second) return preprocResult.first; + assert(signForParams.validMultiSign()); + { std::shared_ptr account_state = ledger->read(keylet::account(*signerAccountID)); // Make sure the account and secret belong together. - auto const err = - acctMatchesPubKey(account_state, *signerAccountID, multiSignPubKey); + auto const err = acctMatchesPubKey( + account_state, *signerAccountID, signForParams.getPublicKey()); if (err != rpcSUCCESS) return rpcError(err); @@ -1031,8 +1053,9 @@ transactionSignFor( // Make the signer object that we'll inject. STObject signer(sfSigner); signer[sfAccount] = *signerAccountID; - signer.setFieldVL(sfTxnSignature, multiSignature); - signer.setFieldVL(sfSigningPubKey, multiSignPubKey.slice()); + signer.setFieldVL(sfTxnSignature, signForParams.getSignature()); + signer.setFieldVL( + sfSigningPubKey, signForParams.getPublicKey().slice()); // If there is not yet a Signers array, make one. if (!sttx->isFieldPresent(sfSigners)) @@ -1066,8 +1089,7 @@ transactionSubmitMultiSigned( Role role, std::chrono::seconds validatedLedgerAge, Application& app, - ProcessTransactionFn const& processTransaction, - RPC::SubmitSync sync) + ProcessTransactionFn const& processTransaction) { auto const& ledger = app.openLedger().current(); auto j = app.journal("RPCHandler"); @@ -1240,7 +1262,7 @@ transactionSubmitMultiSigned( try { // FIXME: For performance, should use asynch interface - processTransaction(txn.second, isUnlimited(role), sync, failType); + processTransaction(txn.second, isUnlimited(role), true, failType); } catch (std::exception&) { diff --git a/src/ripple/rpc/impl/TransactionSign.h b/src/ripple/rpc/impl/TransactionSign.h index 48d2859ccf5..2a38031f50a 100644 --- a/src/ripple/rpc/impl/TransactionSign.h +++ b/src/ripple/rpc/impl/TransactionSign.h @@ -21,7 +21,6 @@ #define RIPPLE_RPC_TRANSACTIONSIGN_H_INCLUDED #include -#include #include #include @@ -76,7 +75,7 @@ checkFee( using ProcessTransactionFn = std::function& transaction, bool bUnlimited, - RPC::SubmitSync sync, + bool bLocal, NetworkOPs::FailHard failType)>; inline ProcessTransactionFn @@ -85,10 +84,9 @@ getProcessTxnFn(NetworkOPs& netOPs) return [&netOPs]( std::shared_ptr& transaction, bool bUnlimited, - RPC::SubmitSync sync, + bool bLocal, NetworkOPs::FailHard failType) { - netOPs.processTransaction( - transaction, bUnlimited, sync, true, failType); + netOPs.processTransaction(transaction, bUnlimited, bLocal, failType); }; } @@ -111,8 +109,7 @@ transactionSubmit( Role role, std::chrono::seconds validatedLedgerAge, Application& app, - ProcessTransactionFn const& processTransaction, - RPC::SubmitSync sync); + ProcessTransactionFn const& processTransaction); /** Returns a Json::objectValue. */ Json::Value @@ -133,8 +130,7 @@ transactionSubmitMultiSigned( Role role, std::chrono::seconds validatedLedgerAge, Application& app, - ProcessTransactionFn const& processTransaction, - RPC::SubmitSync sync); + ProcessTransactionFn const& processTransaction); } // namespace RPC } // namespace ripple diff --git a/src/ripple/server/impl/BaseWSPeer.h b/src/ripple/server/impl/BaseWSPeer.h index b2ba7dd6572..fd1722df003 100644 --- a/src/ripple/server/impl/BaseWSPeer.h +++ b/src/ripple/server/impl/BaseWSPeer.h @@ -29,6 +29,7 @@ #include #include #include + #include #include @@ -52,6 +53,9 @@ class BaseWSPeer : public BasePeer, public WSSession boost::beast::multi_buffer rb_; boost::beast::multi_buffer wb_; std::list> wq_; + /// The socket has been closed, or will close after the next write + /// finishes. Do not do any more writes, and don't try to close + /// again. bool do_close_ = false; boost::beast::websocket::close_reason cr_; waitable_timer timer_; @@ -256,6 +260,8 @@ BaseWSPeer::close( return post(strand_, [self = impl().shared_from_this(), reason] { self->close(reason); }); + if (do_close_) + return; do_close_ = true; if (wq_.empty()) { @@ -348,6 +354,7 @@ BaseWSPeer::on_write_fin(error_code const& ec) return fail(ec, "write_fin"); wq_.pop_front(); if (do_close_) + { impl().ws_.async_close( cr_, bind_executor( @@ -356,6 +363,7 @@ BaseWSPeer::on_write_fin(error_code const& ec) &BaseWSPeer::on_close, impl().shared_from_this(), std::placeholders::_1))); + } else if (!wq_.empty()) on_write({}); } diff --git a/src/secp256k1/src/CMakeLists.txt b/src/secp256k1/src/CMakeLists.txt index 93844caa7f2..dace09201f8 100644 --- a/src/secp256k1/src/CMakeLists.txt +++ b/src/secp256k1/src/CMakeLists.txt @@ -64,9 +64,16 @@ elseif(APPLE) endif() elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(${PROJECT_NAME}_windows "secp256k1") - if(MSVC) - set(${PROJECT_NAME}_windows "${PROJECT_NAME}") - endif() + # This step is commented out from the original. It is bad practice to change + # the binary base name depending on the platform. CMake already manipulates + # the base name into a final name that fits the conventions of the platform. + # Linkers accept base names on the command line and then look for + # conventional names on disk. This way, developers can use base names + # everywhere (in the CMake and Conan they write) and the tools will do the + # right thing. + # if(MSVC) + # set(${PROJECT_NAME}_windows "${PROJECT_NAME}") + # endif() set_target_properties(secp256k1 PROPERTIES ARCHIVE_OUTPUT_NAME "${${PROJECT_NAME}_windows}" RUNTIME_OUTPUT_NAME "${${PROJECT_NAME}_windows}-${${PROJECT_NAME}_soversion}" diff --git a/src/test/app/AMM_test.cpp b/src/test/app/AMM_test.cpp index b0e5106f9eb..b6828fab773 100644 --- a/src/test/app/AMM_test.cpp +++ b/src/test/app/AMM_test.cpp @@ -4789,6 +4789,106 @@ struct AMM_test : public jtx::AMMTest } } + void + testFixDefaultInnerObj() + { + testcase("Fix Default Inner Object"); + using namespace jtx; + FeatureBitset const all{supported_amendments()}; + + auto test = [&](FeatureBitset features, + TER const& err1, + TER const& err2, + TER const& err3, + TER const& err4, + std::uint16_t tfee, + bool closeLedger, + std::optional extra = std::nullopt) { + Env env(*this, features); + fund(env, gw, {alice}, XRP(1'000), {USD(10)}); + AMM amm( + env, + gw, + XRP(10), + USD(10), + {.tfee = tfee, .close = closeLedger}); + amm.deposit(alice, USD(10), XRP(10)); + amm.vote(VoteArg{.account = alice, .tfee = tfee, .err = ter(err1)}); + amm.withdraw(WithdrawArg{ + .account = gw, .asset1Out = USD(1), .err = ter(err2)}); + // with the amendment disabled and ledger not closed, + // second vote succeeds if the first vote sets the trading fee + // to non-zero; if the first vote sets the trading fee to >0 && <9 + // then the second withdraw succeeds if the second vote sets + // the trading fee so that the discounted fee is non-zero + amm.vote(VoteArg{.account = alice, .tfee = 20, .err = ter(err3)}); + amm.withdraw(WithdrawArg{ + .account = gw, .asset1Out = USD(2), .err = ter(err4)}); + }; + + // ledger is closed after each transaction, vote/withdraw don't fail + // regardless whether the amendment is enabled or not + test(all, tesSUCCESS, tesSUCCESS, tesSUCCESS, tesSUCCESS, 0, true); + test( + all - fixInnerObjTemplate, + tesSUCCESS, + tesSUCCESS, + tesSUCCESS, + tesSUCCESS, + 0, + true); + // ledger is not closed after each transaction + // vote/withdraw don't fail if the amendment is enabled + test(all, tesSUCCESS, tesSUCCESS, tesSUCCESS, tesSUCCESS, 0, false); + // vote/withdraw fail if the amendment is not enabled + // second vote/withdraw still fail: second vote fails because + // the initial trading fee is 0, consequently second withdraw fails + // because the second vote fails + test( + all - fixInnerObjTemplate, + tefEXCEPTION, + tefEXCEPTION, + tefEXCEPTION, + tefEXCEPTION, + 0, + false); + // if non-zero trading/discounted fee then vote/withdraw + // don't fail whether the ledger is closed or not and + // the amendment is enabled or not + test(all, tesSUCCESS, tesSUCCESS, tesSUCCESS, tesSUCCESS, 10, true); + test( + all - fixInnerObjTemplate, + tesSUCCESS, + tesSUCCESS, + tesSUCCESS, + tesSUCCESS, + 10, + true); + test(all, tesSUCCESS, tesSUCCESS, tesSUCCESS, tesSUCCESS, 10, false); + test( + all - fixInnerObjTemplate, + tesSUCCESS, + tesSUCCESS, + tesSUCCESS, + tesSUCCESS, + 10, + false); + // non-zero trading fee but discounted fee is 0, vote doesn't fail + // but withdraw fails + test(all, tesSUCCESS, tesSUCCESS, tesSUCCESS, tesSUCCESS, 9, false); + // second vote sets the trading fee to non-zero, consequently + // second withdraw doesn't fail even if the amendment is not + // enabled and the ledger is not closed + test( + all - fixInnerObjTemplate, + tesSUCCESS, + tefEXCEPTION, + tesSUCCESS, + tesSUCCESS, + 9, + false); + } + void testCore() { @@ -4815,6 +4915,7 @@ struct AMM_test : public jtx::AMMTest testClawback(); testAMMID(); testSelection(); + testFixDefaultInnerObj(); } void diff --git a/src/test/app/AmendmentTable_test.cpp b/src/test/app/AmendmentTable_test.cpp index 64e6a038653..238e05ba523 100644 --- a/src/test/app/AmendmentTable_test.cpp +++ b/src/test/app/AmendmentTable_test.cpp @@ -288,7 +288,7 @@ class AmendmentTable_test final : public beast::unit_test::suite uint256 const unsupportedID = amendmentId(unsupported_[0]); { Json::Value const unsupp = - table->getJson(unsupportedID)[to_string(unsupportedID)]; + table->getJson(unsupportedID, true)[to_string(unsupportedID)]; BEAST_EXPECT(unsupp.size() == 0); } @@ -296,7 +296,7 @@ class AmendmentTable_test final : public beast::unit_test::suite table->veto(unsupportedID); { Json::Value const unsupp = - table->getJson(unsupportedID)[to_string(unsupportedID)]; + table->getJson(unsupportedID, true)[to_string(unsupportedID)]; BEAST_EXPECT(unsupp[jss::vetoed].asBool()); } } @@ -638,6 +638,22 @@ class AmendmentTable_test final : public beast::unit_test::suite BEAST_EXPECT(enabled.empty()); BEAST_EXPECT(majority.empty()); + uint256 const unsupportedID = amendmentId(unsupported_[0]); + { + Json::Value const unsupp = + table->getJson(unsupportedID, false)[to_string(unsupportedID)]; + BEAST_EXPECT(unsupp.size() == 0); + } + + table->veto(unsupportedID); + { + Json::Value const unsupp = + table->getJson(unsupportedID, false)[to_string(unsupportedID)]; + BEAST_EXPECT(!unsupp[jss::vetoed].asBool()); + } + + votes.emplace_back(testAmendment, validators.size()); + votes.emplace_back(testAmendment, validators.size()); doRound( diff --git a/src/test/app/DID_test.cpp b/src/test/app/DID_test.cpp index 3aa27978bfe..82e77a20264 100644 --- a/src/test/app/DID_test.cpp +++ b/src/test/app/DID_test.cpp @@ -31,7 +31,7 @@ namespace ripple { namespace test { // Helper function that returns the owner count of an account root. -std::uint32_t +static std::uint32_t ownerCount(test::jtx::Env const& env, test::jtx::Account const& acct) { std::uint32_t ret{0}; @@ -130,7 +130,7 @@ struct DID_test : public beast::unit_test::suite using namespace jtx; using namespace std::chrono; - Env env(*this); + Env env{*this, features}; Account const alice{"alice"}; env.fund(XRP(5000), alice); env.close(); @@ -177,6 +177,16 @@ struct DID_test : public beast::unit_test::suite env.close(); BEAST_EXPECT(ownerCount(env, alice) == 0); + // some empty fields, some optional fields + // pre-fix amendment + auto const fixEnabled = env.current()->rules().enabled(fixEmptyDID); + env(did::set(alice), + did::uri(""), + fixEnabled ? ter(tecEMPTY_DID) : ter(tesSUCCESS)); + env.close(); + auto const expectedOwnerReserve = fixEnabled ? 0 : 1; + BEAST_EXPECT(ownerCount(env, alice) == expectedOwnerReserve); + // Modifying a DID to become empty is checked in testSetModify } @@ -188,7 +198,7 @@ struct DID_test : public beast::unit_test::suite using namespace jtx; using namespace std::chrono; - Env env(*this); + Env env{*this, features}; Account const alice{"alice"}; env.fund(XRP(5000), alice); env.close(); @@ -219,7 +229,7 @@ struct DID_test : public beast::unit_test::suite using namespace jtx; using namespace std::chrono; - Env env(*this); + Env env{*this, features}; Account const alice{"alice"}; Account const bob{"bob"}; Account const charlie{"charlie"}; @@ -273,7 +283,7 @@ struct DID_test : public beast::unit_test::suite using namespace jtx; using namespace std::chrono; - Env env(*this); + Env env{*this, features}; Account const alice{"alice"}; env.fund(XRP(5000), alice); env.close(); @@ -390,13 +400,19 @@ struct DID_test : public beast::unit_test::suite run() override { using namespace test::jtx; - FeatureBitset const all{ - supported_amendments() | FeatureBitset{featureDID}}; + FeatureBitset const all{supported_amendments()}; + FeatureBitset const emptyDID{fixEmptyDID}; testEnabled(all); testAccountReserve(all); testSetInvalid(all); testDeleteInvalid(all); testSetModify(all); + + testEnabled(all - emptyDID); + testAccountReserve(all - emptyDID); + testSetInvalid(all - emptyDID); + testDeleteInvalid(all - emptyDID); + testSetModify(all - emptyDID); } }; diff --git a/src/test/app/LedgerLoad_test.cpp b/src/test/app/LedgerLoad_test.cpp index 93fc002ae2a..f06e7d0bf01 100644 --- a/src/test/app/LedgerLoad_test.cpp +++ b/src/test/app/LedgerLoad_test.cpp @@ -63,21 +63,21 @@ class LedgerLoad_test : public beast::unit_test::suite retval.ledgerFile = td.file("ledgerdata.json"); Env env{*this}; - Account prev; + std::optional prev; for (auto i = 0; i < 20; ++i) { Account acct{"A" + std::to_string(i)}; env.fund(XRP(10000), acct); env.close(); - if (i > 0) + if (i > 0 && BEAST_EXPECT(prev)) { - env.trust(acct["USD"](1000), prev); - env(pay(acct, prev, acct["USD"](5))); + env.trust(acct["USD"](1000), *prev); + env(pay(acct, *prev, acct["USD"](5))); } env(offer(acct, XRP(100), acct["USD"](1))); env.close(); - prev = std::move(acct); + prev.emplace(std::move(acct)); } retval.ledger = env.rpc("ledger", "current", "full")[jss::result]; diff --git a/src/test/app/LedgerReplay_test.cpp b/src/test/app/LedgerReplay_test.cpp index b535739353b..aee24cd7d57 100644 --- a/src/test/app/LedgerReplay_test.cpp +++ b/src/test/app/LedgerReplay_test.cpp @@ -173,6 +173,12 @@ class MagicInboundLedgers : public InboundLedgers { } + virtual size_t + cacheSize() override + { + return 0; + } + LedgerMaster& ledgerSource; LedgerMaster& ledgerSink; InboundLedgersBehavior bhvr; @@ -191,7 +197,9 @@ enum class PeerFeature { class TestPeer : public Peer { public: - TestPeer(bool enableLedgerReplay) : ledgerReplayEnabled_(enableLedgerReplay) + TestPeer(bool enableLedgerReplay) + : ledgerReplayEnabled_(enableLedgerReplay) + , nodePublicKey_(derivePublicKey(KeyType::ed25519, randomSecretKey())) { } @@ -231,8 +239,7 @@ class TestPeer : public Peer PublicKey const& getNodePublic() const override { - static PublicKey key{}; - return key; + return nodePublicKey_; } Json::Value json() override @@ -308,6 +315,7 @@ class TestPeer : public Peer } bool ledgerReplayEnabled_; + PublicKey nodePublicKey_; }; enum class PeerSetBehavior { diff --git a/src/test/app/Manifest_test.cpp b/src/test/app/Manifest_test.cpp index db66e09e518..b72623309e9 100644 --- a/src/test/app/Manifest_test.cpp +++ b/src/test/app/Manifest_test.cpp @@ -238,12 +238,8 @@ class Manifest_test : public beast::unit_test::suite Manifest clone(Manifest const& m) { - Manifest m2; - m2.serialized = m.serialized; - m2.masterKey = m.masterKey; - m2.signingKey = m.signingKey; - m2.sequence = m.sequence; - m2.domain = m.domain; + Manifest m2( + m.serialized, m.masterKey, m.signingKey, m.sequence, m.domain); return m2; } @@ -313,14 +309,13 @@ class Manifest_test : public beast::unit_test::suite } { // save should store all trusted master keys to db - PublicKey emptyLocalKey; std::vector s1; std::vector keys; std::string cfgManifest; for (auto const& man : inManifests) s1.push_back( toBase58(TokenType::NodePublic, man->masterKey)); - unl->load(emptyLocalKey, s1, keys); + unl->load({}, s1, keys); m.save( *dbCon, @@ -852,7 +847,10 @@ class Manifest_test : public beast::unit_test::suite BEAST_EXPECT(manifest); BEAST_EXPECT(manifest->masterKey == pk); - BEAST_EXPECT(manifest->signingKey == PublicKey()); + + // Since this manifest is revoked, it should not have + // a signingKey + BEAST_EXPECT(!manifest->signingKey); BEAST_EXPECT(manifest->revoked()); BEAST_EXPECT(manifest->domain.empty()); BEAST_EXPECT(manifest->serialized == m); diff --git a/src/test/app/MultiSign_test.cpp b/src/test/app/MultiSign_test.cpp index 433cf2f7cd8..6e918e36c79 100644 --- a/src/test/app/MultiSign_test.cpp +++ b/src/test/app/MultiSign_test.cpp @@ -247,7 +247,10 @@ class MultiSign_test : public beast::unit_test::suite // Duplicate signers should fail. aliceSeq = env.seq(alice); - env(noop(alice), msig(demon, demon), fee(3 * baseFee), ter(temINVALID)); + env(noop(alice), + msig(demon, demon), + fee(3 * baseFee), + ter(telENV_RPC_FAILED)); env.close(); BEAST_EXPECT(env.seq(alice) == aliceSeq); @@ -358,7 +361,7 @@ class MultiSign_test : public beast::unit_test::suite msig phantoms{bogie, demon}; std::reverse(phantoms.signers.begin(), phantoms.signers.end()); std::uint32_t const aliceSeq = env.seq(alice); - env(noop(alice), phantoms, ter(temINVALID)); + env(noop(alice), phantoms, ter(telENV_RPC_FAILED)); env.close(); BEAST_EXPECT(env.seq(alice) == aliceSeq); } @@ -1634,7 +1637,10 @@ class MultiSign_test : public beast::unit_test::suite // Duplicate signers should fail. aliceSeq = env.seq(alice); - env(noop(alice), msig(demon, demon), fee(3 * baseFee), ter(temINVALID)); + env(noop(alice), + msig(demon, demon), + fee(3 * baseFee), + ter(telENV_RPC_FAILED)); env.close(); BEAST_EXPECT(env.seq(alice) == aliceSeq); diff --git a/src/test/app/NFTokenBurn_test.cpp b/src/test/app/NFTokenBurn_test.cpp index 2bd7074b2a8..abd9ed56e83 100644 --- a/src/test/app/NFTokenBurn_test.cpp +++ b/src/test/app/NFTokenBurn_test.cpp @@ -26,7 +26,7 @@ namespace ripple { -class NFTokenBurn0_test : public beast::unit_test::suite +class NFTokenBurnBaseUtil_test : public beast::unit_test::suite { // Helper function that returns the owner count of an account root. static std::uint32_t @@ -815,39 +815,39 @@ class NFTokenBurn0_test : public beast::unit_test::suite } }; -class NFTokenBurn1_test : public NFTokenBurn0_test +class NFTokenBurnWOfixFungTokens_test : public NFTokenBurnBaseUtil_test { public: void run() override { - NFTokenBurn0_test::run(1); + NFTokenBurnBaseUtil_test::run(1); } }; -class NFTokenBurn2_test : public NFTokenBurn0_test +class NFTokenBurnWOFixTokenRemint_test : public NFTokenBurnBaseUtil_test { public: void run() override { - NFTokenBurn0_test::run(2); + NFTokenBurnBaseUtil_test::run(2); } }; -class NFTokenBurn3_test : public NFTokenBurn0_test +class NFTokenBurnAllFeatures_test : public NFTokenBurnBaseUtil_test { public: void run() override { - NFTokenBurn0_test::run(3, true); + NFTokenBurnBaseUtil_test::run(3, true); } }; -BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurn0, tx, ripple, 3); -BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurn1, tx, ripple, 3); -BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurn2, tx, ripple, 3); -BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurn3, tx, ripple, 3); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurnBaseUtil, tx, ripple, 3); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurnWOfixFungTokens, tx, ripple, 3); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurnWOFixTokenRemint, tx, ripple, 3); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBurnAllFeatures, tx, ripple, 3); } // namespace ripple diff --git a/src/test/app/NFToken_test.cpp b/src/test/app/NFToken_test.cpp index 2f029c5db1f..8740b521132 100644 --- a/src/test/app/NFToken_test.cpp +++ b/src/test/app/NFToken_test.cpp @@ -27,7 +27,7 @@ namespace ripple { -class NFToken0_test : public beast::unit_test::suite +class NFTokenBaseUtil_test : public beast::unit_test::suite { FeatureBitset const disallowIncoming{featureDisallowIncoming}; @@ -6798,6 +6798,320 @@ class NFToken0_test : public beast::unit_test::suite } } + void + testFixNFTokenBuyerReserve(FeatureBitset features) + { + testcase("Test buyer reserve when accepting an offer"); + + using namespace test::jtx; + + // Lambda that mints an NFT and then creates a sell offer + auto mintAndCreateSellOffer = [](test::jtx::Env& env, + test::jtx::Account const& acct, + STAmount const amt) -> uint256 { + // acct mints a NFT + uint256 const nftId{ + token::getNextID(env, acct, 0u, tfTransferable)}; + env(token::mint(acct, 0u), txflags(tfTransferable)); + env.close(); + + // acct makes an sell offer + uint256 const sellOfferIndex = + keylet::nftoffer(acct, env.seq(acct)).key; + env(token::createOffer(acct, nftId, amt), txflags(tfSellNFToken)); + env.close(); + + return sellOfferIndex; + }; + + // Test the behaviors when the buyer makes an accept offer, both before + // and after enabling the amendment. Exercises the precise number of + // reserve in drops that's required to accept the offer + { + Account const alice{"alice"}; + Account const bob{"bob"}; + + Env env{*this, features}; + auto const acctReserve = env.current()->fees().accountReserve(0); + auto const incReserve = env.current()->fees().increment; + + env.fund(XRP(10000), alice); + env.close(); + + // Bob is funded with minimum XRP reserve + env.fund(acctReserve, bob); + env.close(); + + // alice mints an NFT and create a sell offer for 0 XRP + auto const sellOfferIndex = + mintAndCreateSellOffer(env, alice, XRP(0)); + + // Bob owns no object + BEAST_EXPECT(ownerCount(env, bob) == 0); + + // Without fixNFTokenReserve amendment, when bob accepts an NFT sell + // offer, he can get the NFT free of reserve + if (!features[fixNFTokenReserve]) + { + // Bob is able to accept the offer + env(token::acceptSellOffer(bob, sellOfferIndex)); + env.close(); + + // Bob now owns an extra objects + BEAST_EXPECT(ownerCount(env, bob) == 1); + + // This is the wrong behavior, since Bob should need at least + // one incremental reserve. + } + // With fixNFTokenReserve, bob can no longer accept the offer unless + // there is enough reserve. A detail to note is that NFTs(sell + // offer) will not allow one to go below the reserve requirement, + // because buyer's balance is computed after the transaction fee is + // deducted. This means that the reserve requirement will be 10 + // drops higher than normal. + else + { + // Bob is not able to accept the offer with only the account + // reserve (200,000,000 drops) + env(token::acceptSellOffer(bob, sellOfferIndex), + ter(tecINSUFFICIENT_RESERVE)); + env.close(); + + // after prev transaction, Bob owns 199,999,990 drops due to + // burnt tx fee + + BEAST_EXPECT(ownerCount(env, bob) == 0); + + // Send bob an increment reserve and 10 drops (to make up for + // the transaction fee burnt from the prev failed tx) Bob now + // owns 250,000,000 drops + env(pay(env.master, bob, incReserve + drops(10))); + env.close(); + + // However, this transaction will still fail because the reserve + // requirement is 10 drops higher + env(token::acceptSellOffer(bob, sellOfferIndex), + ter(tecINSUFFICIENT_RESERVE)); + env.close(); + + // Send bob an increment reserve and 20 drops + // Bob now owns 250,000,010 drops + env(pay(env.master, bob, drops(20))); + env.close(); + + // Bob is now able to accept the offer + env(token::acceptSellOffer(bob, sellOfferIndex)); + env.close(); + + BEAST_EXPECT(ownerCount(env, bob) == 1); + } + } + + // Now exercise the scenario when the buyer accepts + // many sell offers + { + Account const alice{"alice"}; + Account const bob{"bob"}; + + Env env{*this, features}; + auto const acctReserve = env.current()->fees().accountReserve(0); + auto const incReserve = env.current()->fees().increment; + + env.fund(XRP(10000), alice); + env.close(); + + env.fund(acctReserve + XRP(1), bob); + env.close(); + + if (!features[fixNFTokenReserve]) + { + // Bob can accept many NFTs without having a single reserve! + for (size_t i = 0; i < 200; i++) + { + // alice mints an NFT and creates a sell offer for 0 XRP + auto const sellOfferIndex = + mintAndCreateSellOffer(env, alice, XRP(0)); + + // Bob is able to accept the offer + env(token::acceptSellOffer(bob, sellOfferIndex)); + env.close(); + } + } + else + { + // alice mints the first NFT and creates a sell offer for 0 XRP + auto const sellOfferIndex1 = + mintAndCreateSellOffer(env, alice, XRP(0)); + + // Bob cannot accept this offer because he doesn't have the + // reserve for the NFT + env(token::acceptSellOffer(bob, sellOfferIndex1), + ter(tecINSUFFICIENT_RESERVE)); + env.close(); + + // Give bob enough reserve + env(pay(env.master, bob, drops(incReserve))); + env.close(); + + BEAST_EXPECT(ownerCount(env, bob) == 0); + + // Bob now owns his first NFT + env(token::acceptSellOffer(bob, sellOfferIndex1)); + env.close(); + + BEAST_EXPECT(ownerCount(env, bob) == 1); + + // alice now mints 31 more NFTs and creates an offer for each + // NFT, then sells to bob + for (size_t i = 0; i < 31; i++) + { + // alice mints an NFT and creates a sell offer for 0 XRP + auto const sellOfferIndex = + mintAndCreateSellOffer(env, alice, XRP(0)); + + // Bob can accept the offer because the new NFT is stored in + // an existing NFTokenPage so no new reserve is requried + env(token::acceptSellOffer(bob, sellOfferIndex)); + env.close(); + } + + BEAST_EXPECT(ownerCount(env, bob) == 1); + + // alice now mints the 33rd NFT and creates an sell offer for 0 + // XRP + auto const sellOfferIndex33 = + mintAndCreateSellOffer(env, alice, XRP(0)); + + // Bob fails to accept this NFT because he does not have enough + // reserve for a new NFTokenPage + env(token::acceptSellOffer(bob, sellOfferIndex33), + ter(tecINSUFFICIENT_RESERVE)); + env.close(); + + // Send bob incremental reserve + env(pay(env.master, bob, drops(incReserve))); + env.close(); + + // Bob now has enough reserve to accept the offer and now + // owns one more NFTokenPage + env(token::acceptSellOffer(bob, sellOfferIndex33)); + env.close(); + + BEAST_EXPECT(ownerCount(env, bob) == 2); + } + } + + // Test the behavior when the seller accepts a buy offer. + // The behavior should not change regardless whether fixNFTokenReserve + // is enabled or not, since the ledger is able to guard against + // free NFTokenPages when buy offer is accepted. This is merely an + // additional test to exercise existing offer behavior. + { + Account const alice{"alice"}; + Account const bob{"bob"}; + + Env env{*this, features}; + auto const acctReserve = env.current()->fees().accountReserve(0); + auto const incReserve = env.current()->fees().increment; + + env.fund(XRP(10000), alice); + env.close(); + + // Bob is funded with account reserve + increment reserve + 1 XRP + // increment reserve is for the buy offer, and 1 XRP is for offer + // price + env.fund(acctReserve + incReserve + XRP(1), bob); + env.close(); + + // Alice mints a NFT + uint256 const nftId{ + token::getNextID(env, alice, 0u, tfTransferable)}; + env(token::mint(alice, 0u), txflags(tfTransferable)); + env.close(); + + // Bob makes a buy offer for 1 XRP + auto const buyOfferIndex = keylet::nftoffer(bob, env.seq(bob)).key; + env(token::createOffer(bob, nftId, XRP(1)), token::owner(alice)); + env.close(); + + // accepting the buy offer fails because bob's balance is 10 drops + // lower than the required amount, since the previous tx burnt 10 + // drops for tx fee. + env(token::acceptBuyOffer(alice, buyOfferIndex), + ter(tecINSUFFICIENT_FUNDS)); + env.close(); + + // send Bob 10 drops + env(pay(env.master, bob, drops(10))); + env.close(); + + // Now bob can buy the offer + env(token::acceptBuyOffer(alice, buyOfferIndex)); + env.close(); + } + + // Test the reserve behavior in brokered mode. + // The behavior should not change regardless whether fixNFTokenReserve + // is enabled or not, since the ledger is able to guard against + // free NFTokenPages in brokered mode. This is merely an + // additional test to exercise existing offer behavior. + { + Account const alice{"alice"}; + Account const bob{"bob"}; + Account const broker{"broker"}; + + Env env{*this, features}; + auto const acctReserve = env.current()->fees().accountReserve(0); + auto const incReserve = env.current()->fees().increment; + + env.fund(XRP(10000), alice, broker); + env.close(); + + // Bob is funded with account reserve + incr reserve + 1 XRP(offer + // price) + env.fund(acctReserve + incReserve + XRP(1), bob); + env.close(); + + // Alice mints a NFT + uint256 const nftId{ + token::getNextID(env, alice, 0u, tfTransferable)}; + env(token::mint(alice, 0u), txflags(tfTransferable)); + env.close(); + + // Alice creates sell offer and set broker as destination + uint256 const offerAliceToBroker = + keylet::nftoffer(alice, env.seq(alice)).key; + env(token::createOffer(alice, nftId, XRP(1)), + token::destination(broker), + txflags(tfSellNFToken)); + env.close(); + + // Bob creates buy offer + uint256 const offerBobToBroker = + keylet::nftoffer(bob, env.seq(bob)).key; + env(token::createOffer(bob, nftId, XRP(1)), token::owner(alice)); + env.close(); + + // broker offers. + // Returns insufficient funds, because bob burnt tx fee when he + // created his buy offer, which makes his spendable balance to be + // less than the required amount. + env(token::brokerOffers( + broker, offerBobToBroker, offerAliceToBroker), + ter(tecINSUFFICIENT_FUNDS)); + env.close(); + + // send Bob 10 drops + env(pay(env.master, bob, drops(10))); + env.close(); + + // broker offers. + env(token::brokerOffers( + broker, offerBobToBroker, offerAliceToBroker)); + env.close(); + } + } + void testWithFeats(FeatureBitset features) { @@ -6831,6 +7145,7 @@ class NFToken0_test : public beast::unit_test::suite testBrokeredSaleToSelf(features); testFixNFTokenRemint(features); testTxJsonMetaFields(features); + testFixNFTokenBuyerReserve(features); } public: @@ -6841,12 +7156,15 @@ class NFToken0_test : public beast::unit_test::suite static FeatureBitset const all{supported_amendments()}; static FeatureBitset const fixNFTDir{fixNFTokenDirV1}; - static std::array const feats{ - all - fixNFTDir - fixNonFungibleTokensV1_2 - fixNFTokenRemint, + static std::array const feats{ + all - fixNFTDir - fixNonFungibleTokensV1_2 - fixNFTokenRemint - + fixNFTokenReserve, all - disallowIncoming - fixNonFungibleTokensV1_2 - - fixNFTokenRemint, - all - fixNonFungibleTokensV1_2 - fixNFTokenRemint, - all - fixNFTokenRemint, + fixNFTokenRemint - fixNFTokenReserve, + all - fixNonFungibleTokensV1_2 - fixNFTokenRemint - + fixNFTokenReserve, + all - fixNFTokenRemint - fixNFTokenReserve, + all - fixNFTokenReserve, all}; if (BEAST_EXPECT(instance < feats.size())) @@ -6863,46 +7181,56 @@ class NFToken0_test : public beast::unit_test::suite } }; -class NFToken1_test : public NFToken0_test +class NFTokenDisallowIncoming_test : public NFTokenBaseUtil_test +{ + void + run() override + { + NFTokenBaseUtil_test::run(1); + } +}; + +class NFTokenWOfixV1_test : public NFTokenBaseUtil_test { void run() override { - NFToken0_test::run(1); + NFTokenBaseUtil_test::run(2); } }; -class NFToken2_test : public NFToken0_test +class NFTokenWOTokenRemint_test : public NFTokenBaseUtil_test { void run() override { - NFToken0_test::run(2); + NFTokenBaseUtil_test::run(3); } }; -class NFToken3_test : public NFToken0_test +class NFTokenWOTokenReserve_test : public NFTokenBaseUtil_test { void run() override { - NFToken0_test::run(3); + NFTokenBaseUtil_test::run(4); } }; -class NFToken4_test : public NFToken0_test +class NFTokenAllFeatures_test : public NFTokenBaseUtil_test { void run() override { - NFToken0_test::run(4, true); + NFTokenBaseUtil_test::run(5, true); } }; -BEAST_DEFINE_TESTSUITE_PRIO(NFToken0, tx, ripple, 2); -BEAST_DEFINE_TESTSUITE_PRIO(NFToken1, tx, ripple, 2); -BEAST_DEFINE_TESTSUITE_PRIO(NFToken2, tx, ripple, 2); -BEAST_DEFINE_TESTSUITE_PRIO(NFToken3, tx, ripple, 2); -BEAST_DEFINE_TESTSUITE_PRIO(NFToken4, tx, ripple, 2); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenBaseUtil, tx, ripple, 2); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenDisallowIncoming, tx, ripple, 2); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenWOfixV1, tx, ripple, 2); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenWOTokenRemint, tx, ripple, 2); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenWOTokenReserve, tx, ripple, 2); +BEAST_DEFINE_TESTSUITE_PRIO(NFTokenAllFeatures, tx, ripple, 2); } // namespace ripple diff --git a/src/test/app/Offer_test.cpp b/src/test/app/Offer_test.cpp index 25ca5a4b2dc..95ffd9f3aee 100644 --- a/src/test/app/Offer_test.cpp +++ b/src/test/app/Offer_test.cpp @@ -27,7 +27,7 @@ namespace ripple { namespace test { -class Offer0_test : public beast::unit_test::suite +class OfferBaseUtil_test : public beast::unit_test::suite { XRPAmount reserve(jtx::Env& env, std::uint32_t count) @@ -5371,52 +5371,52 @@ class Offer0_test : public beast::unit_test::suite } }; -class Offer1_test : public Offer0_test +class OfferWOFlowCross_test : public OfferBaseUtil_test { void run() override { - Offer0_test::run(1); + OfferBaseUtil_test::run(1); } }; -class Offer2_test : public Offer0_test +class OfferWTakerDryOffer_test : public OfferBaseUtil_test { void run() override { - Offer0_test::run(2); + OfferBaseUtil_test::run(2); } }; -class Offer3_test : public Offer0_test +class OfferWOSmallQOffers_test : public OfferBaseUtil_test { void run() override { - Offer0_test::run(3); + OfferBaseUtil_test::run(3); } }; -class Offer4_test : public Offer0_test +class OfferWOFillOrKill_test : public OfferBaseUtil_test { void run() override { - Offer0_test::run(4); + OfferBaseUtil_test::run(4); } }; -class Offer5_test : public Offer0_test +class OfferAllFeatures_test : public OfferBaseUtil_test { void run() override { - Offer0_test::run(5, true); + OfferBaseUtil_test::run(5, true); } }; -class Offer_manual_test : public Offer0_test +class Offer_manual_test : public OfferBaseUtil_test { void run() override @@ -5439,12 +5439,12 @@ class Offer_manual_test : public Offer0_test } }; -BEAST_DEFINE_TESTSUITE_PRIO(Offer0, tx, ripple, 4); -BEAST_DEFINE_TESTSUITE_PRIO(Offer1, tx, ripple, 4); -BEAST_DEFINE_TESTSUITE_PRIO(Offer2, tx, ripple, 4); -BEAST_DEFINE_TESTSUITE_PRIO(Offer3, tx, ripple, 4); -BEAST_DEFINE_TESTSUITE_PRIO(Offer4, tx, ripple, 4); -BEAST_DEFINE_TESTSUITE_PRIO(Offer5, tx, ripple, 4); +BEAST_DEFINE_TESTSUITE_PRIO(OfferBaseUtil, tx, ripple, 4); +BEAST_DEFINE_TESTSUITE_PRIO(OfferWOFlowCross, tx, ripple, 4); +BEAST_DEFINE_TESTSUITE_PRIO(OfferWTakerDryOffer, tx, ripple, 4); +BEAST_DEFINE_TESTSUITE_PRIO(OfferWOSmallQOffers, tx, ripple, 4); +BEAST_DEFINE_TESTSUITE_PRIO(OfferWOFillOrKill, tx, ripple, 4); +BEAST_DEFINE_TESTSUITE_PRIO(OfferAllFeatures, tx, ripple, 4); BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(Offer_manual, tx, ripple, 20); } // namespace test diff --git a/src/test/app/Oracle_test.cpp b/src/test/app/Oracle_test.cpp new file mode 100644 index 00000000000..f5488c793a1 --- /dev/null +++ b/src/test/app/Oracle_test.cpp @@ -0,0 +1,698 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +namespace ripple { +namespace test { +namespace jtx { +namespace oracle { + +struct Oracle_test : public beast::unit_test::suite +{ +private: + // Helper function that returns the owner count of an account root. + static std::uint32_t + ownerCount(jtx::Env const& env, jtx::Account const& acct) + { + std::uint32_t ret{0}; + if (auto const sleAcct = env.le(acct)) + ret = sleAcct->at(sfOwnerCount); + return ret; + } + + void + testInvalidSet() + { + testcase("Invalid Set"); + + using namespace jtx; + Account const owner("owner"); + + { + // Invalid account + Env env(*this); + Account const bad("bad"); + env.memoize(bad); + Oracle oracle( + env, {.owner = bad, .seq = seq(1), .err = ter(terNO_ACCOUNT)}); + } + + // Insufficient reserve + { + Env env(*this); + env.fund(env.current()->fees().accountReserve(0), owner); + Oracle oracle( + env, {.owner = owner, .err = ter(tecINSUFFICIENT_RESERVE)}); + } + // Insufficient reserve if the data series extends to greater than 5 + { + Env env(*this); + env.fund( + env.current()->fees().accountReserve(1) + + env.current()->fees().base * 2, + owner); + Oracle oracle(env, {.owner = owner}); + BEAST_EXPECT(oracle.exists()); + oracle.set(UpdateArg{ + .series = + { + {"XRP", "EUR", 740, 1}, + {"XRP", "GBP", 740, 1}, + {"XRP", "CNY", 740, 1}, + {"XRP", "CAD", 740, 1}, + {"XRP", "AUD", 740, 1}, + }, + .err = ter(tecINSUFFICIENT_RESERVE)}); + } + + { + Env env(*this); + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner}, false); + + // Invalid flag + oracle.set( + CreateArg{.flags = tfSellNFToken, .err = ter(temINVALID_FLAG)}); + + // Duplicate token pair + oracle.set(CreateArg{ + .series = {{"XRP", "USD", 740, 1}, {"XRP", "USD", 750, 1}}, + .err = ter(temMALFORMED)}); + + // Price is not included + oracle.set(CreateArg{ + .series = + {{"XRP", "USD", 740, 1}, {"XRP", "EUR", std::nullopt, 1}}, + .err = ter(temMALFORMED)}); + + // Token pair is in update and delete + oracle.set(CreateArg{ + .series = + {{"XRP", "USD", 740, 1}, {"XRP", "USD", std::nullopt, 1}}, + .err = ter(temMALFORMED)}); + // Token pair is in add and delete + oracle.set(CreateArg{ + .series = + {{"XRP", "EUR", 740, 1}, {"XRP", "EUR", std::nullopt, 1}}, + .err = ter(temMALFORMED)}); + + // Array of token pair is 0 or exceeds 10 + oracle.set(CreateArg{ + .series = + {{"XRP", "US1", 740, 1}, + {"XRP", "US2", 750, 1}, + {"XRP", "US3", 740, 1}, + {"XRP", "US4", 750, 1}, + {"XRP", "US5", 740, 1}, + {"XRP", "US6", 750, 1}, + {"XRP", "US7", 740, 1}, + {"XRP", "US8", 750, 1}, + {"XRP", "US9", 740, 1}, + {"XRP", "U10", 750, 1}, + {"XRP", "U11", 740, 1}}, + .err = ter(temARRAY_TOO_LARGE)}); + oracle.set(CreateArg{.series = {}, .err = ter(temARRAY_EMPTY)}); + } + + // Array of token pair exceeds 10 after update + { + Env env{*this}; + env.fund(XRP(1'000), owner); + + Oracle oracle( + env, + CreateArg{ + .owner = owner, .series = {{{"XRP", "USD", 740, 1}}}}); + oracle.set(UpdateArg{ + .series = + { + {"XRP", "US1", 740, 1}, + {"XRP", "US2", 750, 1}, + {"XRP", "US3", 740, 1}, + {"XRP", "US4", 750, 1}, + {"XRP", "US5", 740, 1}, + {"XRP", "US6", 750, 1}, + {"XRP", "US7", 740, 1}, + {"XRP", "US8", 750, 1}, + {"XRP", "US9", 740, 1}, + {"XRP", "U10", 750, 1}, + }, + .err = ter(tecARRAY_TOO_LARGE)}); + } + + { + Env env(*this); + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner}, false); + + // Symbol class or provider not included on create + oracle.set(CreateArg{ + .assetClass = std::nullopt, + .provider = "provider", + .err = ter(temMALFORMED)}); + oracle.set(CreateArg{ + .assetClass = "currency", + .provider = std::nullopt, + .uri = "URI", + .err = ter(temMALFORMED)}); + + // Symbol class or provider are included on update + // and don't match the current values + oracle.set(CreateArg{}); + BEAST_EXPECT(oracle.exists()); + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .provider = "provider1", + .err = ter(temMALFORMED)}); + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .assetClass = "currency1", + .err = ter(temMALFORMED)}); + } + + { + Env env(*this); + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner}, false); + + // Fields too long + // Symbol class + std::string assetClass(17, '0'); + oracle.set( + CreateArg{.assetClass = assetClass, .err = ter(temMALFORMED)}); + // provider + std::string const large(257, '0'); + oracle.set(CreateArg{.provider = large, .err = ter(temMALFORMED)}); + // URI + oracle.set(CreateArg{.uri = large, .err = ter(temMALFORMED)}); + } + + { + // Different owner creates a new object and fails because + // of missing fields currency/provider + Env env(*this); + Account const some("some"); + env.fund(XRP(1'000), owner); + env.fund(XRP(1'000), some); + Oracle oracle(env, {.owner = owner}); + BEAST_EXPECT(oracle.exists()); + oracle.set(UpdateArg{ + .owner = some, + .series = {{"XRP", "USD", 740, 1}}, + .err = ter(temMALFORMED)}); + } + + { + // Invalid update time + using namespace std::chrono; + Env env(*this); + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner}); + BEAST_EXPECT(oracle.exists()); + env.close(seconds(400)); + // Less than the last close time - 300s + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .lastUpdateTime = testStartTime.count() + 400 - 301, + .err = ter(tecINVALID_UPDATE_TIME)}); + // Greater than last close time + 300s + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .lastUpdateTime = testStartTime.count() + 400 + 301, + .err = ter(tecINVALID_UPDATE_TIME)}); + oracle.set(UpdateArg{.series = {{"XRP", "USD", 740, 1}}}); + BEAST_EXPECT( + oracle.expectLastUpdateTime(testStartTime.count() + 450)); + // Less than the previous lastUpdateTime + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .lastUpdateTime = testStartTime.count() + 449, + .err = ter(tecINVALID_UPDATE_TIME)}); + } + + { + // delete token pair that doesn't exist + Env env(*this); + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner}); + BEAST_EXPECT(oracle.exists()); + oracle.set(UpdateArg{ + .series = {{"XRP", "EUR", std::nullopt, std::nullopt}}, + .err = ter(tecTOKEN_PAIR_NOT_FOUND)}); + // delete all token pairs + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", std::nullopt, std::nullopt}}, + .err = ter(tecARRAY_EMPTY)}); + } + + { + // same BaseAsset and QuoteAsset + Env env(*this); + env.fund(XRP(1'000), owner); + Oracle oracle( + env, + {.owner = owner, + .series = {{"USD", "USD", 740, 1}}, + .err = ter(temMALFORMED)}); + } + + { + // Scale is greater than maxPriceScale + Env env(*this); + env.fund(XRP(1'000), owner); + Oracle oracle( + env, + {.owner = owner, + .series = {{"USD", "BTC", 740, maxPriceScale + 1}}, + .err = ter(temMALFORMED)}); + } + } + + void + testCreate() + { + testcase("Create"); + using namespace jtx; + Account const owner("owner"); + + auto test = [&](Env& env, DataSeries const& series, std::uint16_t adj) { + env.fund(XRP(1'000), owner); + auto const count = ownerCount(env, owner); + Oracle oracle(env, {.owner = owner, .series = series}); + BEAST_EXPECT(oracle.exists()); + BEAST_EXPECT(ownerCount(env, owner) == (count + adj)); + BEAST_EXPECT(oracle.expectLastUpdateTime(946694810)); + }; + + { + // owner count is adjusted by 1 + Env env(*this); + test(env, {{"XRP", "USD", 740, 1}}, 1); + } + + { + // owner count is adjusted by 2 + Env env(*this); + test( + env, + {{"XRP", "USD", 740, 1}, + {"BTC", "USD", 740, 1}, + {"ETH", "USD", 740, 1}, + {"CAN", "USD", 740, 1}, + {"YAN", "USD", 740, 1}, + {"GBP", "USD", 740, 1}}, + 2); + } + + { + // Different owner creates a new object + Env env(*this); + Account const some("some"); + env.fund(XRP(1'000), owner); + env.fund(XRP(1'000), some); + Oracle oracle(env, {.owner = owner}); + BEAST_EXPECT(oracle.exists()); + oracle.set(CreateArg{ + .owner = some, .series = {{"912810RR9", "USD", 740, 1}}}); + BEAST_EXPECT(Oracle::exists(env, some, oracle.documentID())); + } + } + + void + testInvalidDelete() + { + testcase("Invalid Delete"); + + using namespace jtx; + Env env(*this); + Account const owner("owner"); + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner}); + BEAST_EXPECT(oracle.exists()); + + { + // Invalid account + Account const bad("bad"); + env.memoize(bad); + oracle.remove( + {.owner = bad, .seq = seq(1), .err = ter(terNO_ACCOUNT)}); + } + + // Invalid Sequence + oracle.remove({.documentID = 2, .err = ter(tecNO_ENTRY)}); + + // Invalid owner + Account const invalid("invalid"); + env.fund(XRP(1'000), invalid); + oracle.remove({.owner = invalid, .err = ter(tecNO_ENTRY)}); + } + + void + testDelete() + { + testcase("Delete"); + using namespace jtx; + Account const owner("owner"); + + auto test = [&](Env& env, DataSeries const& series, std::uint16_t adj) { + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner, .series = series}); + auto const count = ownerCount(env, owner); + BEAST_EXPECT(oracle.exists()); + oracle.remove({}); + BEAST_EXPECT(!oracle.exists()); + BEAST_EXPECT(ownerCount(env, owner) == (count - adj)); + }; + + { + // owner count is adjusted by 1 + Env env(*this); + test(env, {{"XRP", "USD", 740, 1}}, 1); + } + + { + // owner count is adjusted by 2 + Env env(*this); + test( + env, + { + {"XRP", "USD", 740, 1}, + {"BTC", "USD", 740, 1}, + {"ETH", "USD", 740, 1}, + {"CAN", "USD", 740, 1}, + {"YAN", "USD", 740, 1}, + {"GBP", "USD", 740, 1}, + }, + 2); + } + + { + // deleting the account deletes the oracles + Env env(*this); + auto const alice = Account("alice"); + auto const acctDelFee{drops(env.current()->fees().increment)}; + env.fund(XRP(1'000), owner); + env.fund(XRP(1'000), alice); + Oracle oracle( + env, {.owner = owner, .series = {{"XRP", "USD", 740, 1}}}); + Oracle oracle1( + env, + {.owner = owner, + .documentID = 2, + .series = {{"XRP", "EUR", 740, 1}}}); + BEAST_EXPECT(ownerCount(env, owner) == 2); + BEAST_EXPECT(oracle.exists()); + BEAST_EXPECT(oracle1.exists()); + auto const index = env.closed()->seq(); + auto const hash = env.closed()->info().hash; + for (int i = 0; i < 256; ++i) + env.close(); + env(acctdelete(owner, alice), fee(acctDelFee)); + env.close(); + BEAST_EXPECT(!oracle.exists()); + BEAST_EXPECT(!oracle1.exists()); + + // can still get the oracles via the ledger index or hash + auto verifyLedgerData = [&](auto const& field, auto const& value) { + Json::Value jvParams; + jvParams[field] = value; + jvParams[jss::binary] = false; + jvParams[jss::type] = jss::oracle; + Json::Value jrr = env.rpc( + "json", + "ledger_data", + boost::lexical_cast(jvParams)); + BEAST_EXPECT(jrr[jss::result][jss::state].size() == 2); + }; + verifyLedgerData(jss::ledger_index, index); + verifyLedgerData(jss::ledger_hash, to_string(hash)); + } + } + + void + testUpdate() + { + testcase("Update"); + using namespace jtx; + Account const owner("owner"); + + { + Env env(*this); + env.fund(XRP(1'000), owner); + auto count = ownerCount(env, owner); + Oracle oracle(env, {.owner = owner}); + BEAST_EXPECT(oracle.exists()); + + // update existing pair + oracle.set(UpdateArg{.series = {{"XRP", "USD", 740, 2}}}); + BEAST_EXPECT(oracle.expectPrice({{"XRP", "USD", 740, 2}})); + // owner count is increased by 1 since the oracle object is added + // with one token pair + count += 1; + BEAST_EXPECT(ownerCount(env, owner) == count); + + // add new pairs, not-included pair is reset + oracle.set(UpdateArg{.series = {{"XRP", "EUR", 700, 2}}}); + BEAST_EXPECT(oracle.expectPrice( + {{"XRP", "USD", 0, 0}, {"XRP", "EUR", 700, 2}})); + // owner count is not changed since the number of pairs is 2 + BEAST_EXPECT(ownerCount(env, owner) == count); + + // update both pairs + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 741, 2}, {"XRP", "EUR", 710, 2}}}); + BEAST_EXPECT(oracle.expectPrice( + {{"XRP", "USD", 741, 2}, {"XRP", "EUR", 710, 2}})); + // owner count is not changed since the number of pairs is 2 + BEAST_EXPECT(ownerCount(env, owner) == count); + + // owner count is increased by 1 since the number of pairs is 6 + oracle.set(UpdateArg{ + .series = { + {"BTC", "USD", 741, 2}, + {"ETH", "EUR", 710, 2}, + {"YAN", "EUR", 710, 2}, + {"CAN", "EUR", 710, 2}, + }}); + count += 1; + BEAST_EXPECT(ownerCount(env, owner) == count); + + // update two pairs and delete four + oracle.set(UpdateArg{ + .series = {{"BTC", "USD", std::nullopt, std::nullopt}}}); + oracle.set(UpdateArg{ + .series = { + {"XRP", "USD", 742, 2}, + {"XRP", "EUR", 711, 2}, + {"ETH", "EUR", std::nullopt, std::nullopt}, + {"YAN", "EUR", std::nullopt, std::nullopt}, + {"CAN", "EUR", std::nullopt, std::nullopt}}}); + BEAST_EXPECT(oracle.expectPrice( + {{"XRP", "USD", 742, 2}, {"XRP", "EUR", 711, 2}})); + // owner count is decreased by 1 since the number of pairs is 2 + count -= 1; + BEAST_EXPECT(ownerCount(env, owner) == count); + } + + // Min reserve to create and update + { + Env env(*this); + env.fund( + env.current()->fees().accountReserve(1) + + env.current()->fees().base * 2, + owner); + Oracle oracle(env, {.owner = owner}); + oracle.set(UpdateArg{.series = {{"XRP", "USD", 742, 2}}}); + } + } + + void + testMultisig(FeatureBitset features) + { + testcase("Multisig"); + using namespace jtx; + Oracle::setFee(100'000); + + Env env(*this, features); + Account const alice{"alice", KeyType::secp256k1}; + Account const bogie{"bogie", KeyType::secp256k1}; + Account const ed{"ed", KeyType::secp256k1}; + Account const becky{"becky", KeyType::ed25519}; + Account const zelda{"zelda", KeyType::secp256k1}; + Account const bob{"bob", KeyType::secp256k1}; + env.fund(XRP(10'000), alice, becky, zelda, ed, bob); + + // alice uses a regular key with the master disabled. + Account const alie{"alie", KeyType::secp256k1}; + env(regkey(alice, alie)); + env(fset(alice, asfDisableMaster), sig(alice)); + + // Attach signers to alice. + env(signers(alice, 2, {{becky, 1}, {bogie, 1}, {ed, 2}}), sig(alie)); + env.close(); + // if multiSignReserve disabled then its 2 + 1 per signer + int const signerListOwners{features[featureMultiSignReserve] ? 1 : 5}; + env.require(owners(alice, signerListOwners)); + + // Create + // Force close (true) and time advancement because the close time + // is no longer 0. + Oracle oracle(env, CreateArg{.owner = alice, .close = true}, false); + oracle.set(CreateArg{.msig = msig(becky), .err = ter(tefBAD_QUORUM)}); + oracle.set( + CreateArg{.msig = msig(zelda), .err = ter(tefBAD_SIGNATURE)}); + oracle.set(CreateArg{.msig = msig(becky, bogie)}); + BEAST_EXPECT(oracle.exists()); + + // Update + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .msig = msig(becky), + .err = ter(tefBAD_QUORUM)}); + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .msig = msig(zelda), + .err = ter(tefBAD_SIGNATURE)}); + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 741, 1}}, .msig = msig(becky, bogie)}); + BEAST_EXPECT(oracle.expectPrice({{"XRP", "USD", 741, 1}})); + // remove the signer list + env(signers(alice, jtx::none), sig(alie)); + env.close(); + env.require(owners(alice, 1)); + // create new signer list + env(signers(alice, 2, {{zelda, 1}, {bob, 1}, {ed, 2}}), sig(alie)); + env.close(); + // old list fails + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 740, 1}}, + .msig = msig(becky, bogie), + .err = ter(tefBAD_SIGNATURE)}); + // updated list succeeds + oracle.set(UpdateArg{ + .series = {{"XRP", "USD", 7412, 2}}, .msig = msig(zelda, bob)}); + BEAST_EXPECT(oracle.expectPrice({{"XRP", "USD", 7412, 2}})); + oracle.set( + UpdateArg{.series = {{"XRP", "USD", 74245, 3}}, .msig = msig(ed)}); + BEAST_EXPECT(oracle.expectPrice({{"XRP", "USD", 74245, 3}})); + + // Remove + oracle.remove({.msig = msig(bob), .err = ter(tefBAD_QUORUM)}); + oracle.remove({.msig = msig(becky), .err = ter(tefBAD_SIGNATURE)}); + oracle.remove({.msig = msig(ed)}); + BEAST_EXPECT(!oracle.exists()); + } + + void + testAmendment() + { + testcase("Amendment"); + using namespace jtx; + + auto const features = supported_amendments() - featurePriceOracle; + Account const owner("owner"); + Env env(*this, features); + + env.fund(XRP(1'000), owner); + { + Oracle oracle(env, {.owner = owner, .err = ter(temDISABLED)}); + } + + { + Oracle oracle(env, {.owner = owner}, false); + oracle.remove({.err = ter(temDISABLED)}); + } + } + + void + testLedgerEntry() + { + testcase("Ledger Entry"); + using namespace jtx; + + Env env(*this); + std::vector accounts; + std::vector oracles; + for (int i = 0; i < 10; ++i) + { + Account const owner(std::string("owner") + std::to_string(i)); + env.fund(XRP(1'000), owner); + // different accounts can have the same asset pair + Oracle oracle(env, {.owner = owner, .documentID = i}); + accounts.push_back(owner.id()); + oracles.push_back(oracle.documentID()); + // same account can have different asset pair + Oracle oracle1(env, {.owner = owner, .documentID = i + 10}); + accounts.push_back(owner.id()); + oracles.push_back(oracle1.documentID()); + } + for (int i = 0; i < accounts.size(); ++i) + { + auto const jv = [&]() { + // document id is uint32 + if (i % 2) + return Oracle::ledgerEntry(env, accounts[i], oracles[i]); + // document id is string + return Oracle::ledgerEntry( + env, accounts[i], std::to_string(oracles[i])); + }(); + try + { + BEAST_EXPECT( + jv[jss::node][jss::Owner] == to_string(accounts[i])); + } + catch (...) + { + fail(); + } + } + } + +public: + void + run() override + { + using namespace jtx; + auto const all = supported_amendments(); + testInvalidSet(); + testInvalidDelete(); + testCreate(); + testDelete(); + testUpdate(); + testAmendment(); + for (auto const& features : + {all, + all - featureMultiSignReserve - featureExpandedSignerList, + all - featureExpandedSignerList}) + testMultisig(features); + testLedgerEntry(); + } +}; + +BEAST_DEFINE_TESTSUITE(Oracle, app, ripple); + +} // namespace oracle + +} // namespace jtx + +} // namespace test + +} // namespace ripple diff --git a/src/test/app/PayChan_test.cpp b/src/test/app/PayChan_test.cpp index 02a13de5c21..a479e43b170 100644 --- a/src/test/app/PayChan_test.cpp +++ b/src/test/app/PayChan_test.cpp @@ -1088,10 +1088,7 @@ struct PayChan_test : public beast::unit_test::suite args[jss::amount] = 51110000; // test for all api versions - for (auto apiVersion = RPC::apiMinimumSupportedVersion; - apiVersion <= RPC::apiBetaVersion; - ++apiVersion) - { + forAllApiVersions([&, this](unsigned apiVersion) { testcase( "PayChan Channel_Auth RPC Api " + std::to_string(apiVersion)); args[jss::api_version] = apiVersion; @@ -1101,7 +1098,7 @@ struct PayChan_test : public beast::unit_test::suite args.toStyledString())[jss::result]; auto const error = apiVersion < 2u ? "invalidParams" : "badKeyType"; BEAST_EXPECT(rs[jss::error] == error); - } + }); } void diff --git a/src/test/app/Regression_test.cpp b/src/test/app/Regression_test.cpp index 6431f81dbd6..e2c4b355a9d 100644 --- a/src/test/app/Regression_test.cpp +++ b/src/test/app/Regression_test.cpp @@ -149,7 +149,7 @@ struct Regression_test : public beast::unit_test::suite secp256r1Sig->setFieldVL(sfSigningPubKey, *pubKeyBlob); jt.stx.reset(secp256r1Sig.release()); - env(jt, ter(temINVALID)); + env(jt, ter(telENV_RPC_FAILED)); }; Account const alice{"alice", KeyType::secp256k1}; diff --git a/src/test/app/Transaction_ordering_test.cpp b/src/test/app/Transaction_ordering_test.cpp index 01f870d0668..0353df90663 100644 --- a/src/test/app/Transaction_ordering_test.cpp +++ b/src/test/app/Transaction_ordering_test.cpp @@ -15,7 +15,6 @@ */ //============================================================================== -#include #include #include #include @@ -92,7 +91,6 @@ struct Transaction_ordering_test : public beast::unit_test::suite env(tx2, ter(terPRE_SEQ)); BEAST_EXPECT(env.seq(alice) == aliceSequence); env(tx1); - BEAST_EXPECT(env.app().getOPs().transactionBatch(false)); env.app().getJobQueue().rendezvous(); BEAST_EXPECT(env.seq(alice) == aliceSequence + 2); @@ -145,8 +143,6 @@ struct Transaction_ordering_test : public beast::unit_test::suite } env(tx[0]); - // Apply until no more deferred/held transactions. - BEAST_EXPECT(env.app().getOPs().transactionBatch(true)); env.app().getJobQueue().rendezvous(); BEAST_EXPECT(env.seq(alice) == aliceSequence + 5); diff --git a/src/test/app/TxQ_test.cpp b/src/test/app/TxQ_test.cpp index ac6bf56f06c..31e0b7b8124 100644 --- a/src/test/app/TxQ_test.cpp +++ b/src/test/app/TxQ_test.cpp @@ -35,7 +35,7 @@ namespace ripple { namespace test { -class TxQ1_test : public beast::unit_test::suite +class TxQPosNegFlows_test : public beast::unit_test::suite { void checkMetrics( @@ -1019,6 +1019,9 @@ class TxQ1_test : public beast::unit_test::suite // Fail in preflight env(pay(alice, bob, XRP(-1000)), ter(temBAD_AMOUNT)); + // Fail in preflight + env(pay(alice, alice, XRP(100)), ter(temREDUNDANT)); + // Fail in preclaim env(noop(alice), fee(XRP(100000)), ter(terINSUF_FEE_B)); } @@ -4949,7 +4952,7 @@ class TxQ1_test : public beast::unit_test::suite } void - run2() + runMetaInfo() { testAcctInQueueButEmpty(); testRPC(); @@ -4970,17 +4973,17 @@ class TxQ1_test : public beast::unit_test::suite } }; -class TxQ2_test : public TxQ1_test +class TxQMetaInfo_test : public TxQPosNegFlows_test { void run() override { - run2(); + runMetaInfo(); } }; -BEAST_DEFINE_TESTSUITE_PRIO(TxQ1, app, ripple, 1); -BEAST_DEFINE_TESTSUITE_PRIO(TxQ2, app, ripple, 1); +BEAST_DEFINE_TESTSUITE_PRIO(TxQPosNegFlows, app, ripple, 1); +BEAST_DEFINE_TESTSUITE_PRIO(TxQMetaInfo, app, ripple, 1); } // namespace test } // namespace ripple diff --git a/src/test/app/ValidatorKeys_test.cpp b/src/test/app/ValidatorKeys_test.cpp index 3943fd85881..99bd6188261 100644 --- a/src/test/app/ValidatorKeys_test.cpp +++ b/src/test/app/ValidatorKeys_test.cpp @@ -107,7 +107,7 @@ class ValidatorKeys_test : public beast::unit_test::suite // No config -> no key but valid Config c; ValidatorKeys k{c, journal}; - BEAST_EXPECT(k.publicKey.size() == 0); + BEAST_EXPECT(!k.keys); BEAST_EXPECT(k.manifest.empty()); BEAST_EXPECT(!k.configInvalid()); } @@ -117,8 +117,11 @@ class ValidatorKeys_test : public beast::unit_test::suite c.section(SECTION_VALIDATION_SEED).append(seed); ValidatorKeys k{c, journal}; - BEAST_EXPECT(k.publicKey == seedPublicKey); - BEAST_EXPECT(k.secretKey == seedSecretKey); + if (BEAST_EXPECT(k.keys)) + { + BEAST_EXPECT(k.keys->publicKey == seedPublicKey); + BEAST_EXPECT(k.keys->secretKey == seedSecretKey); + } BEAST_EXPECT(k.nodeID == seedNodeID); BEAST_EXPECT(k.manifest.empty()); BEAST_EXPECT(!k.configInvalid()); @@ -131,7 +134,7 @@ class ValidatorKeys_test : public beast::unit_test::suite ValidatorKeys k{c, journal}; BEAST_EXPECT(k.configInvalid()); - BEAST_EXPECT(k.publicKey.size() == 0); + BEAST_EXPECT(!k.keys); BEAST_EXPECT(k.manifest.empty()); } @@ -141,8 +144,11 @@ class ValidatorKeys_test : public beast::unit_test::suite c.section(SECTION_VALIDATOR_TOKEN).append(tokenBlob); ValidatorKeys k{c, journal}; - BEAST_EXPECT(k.publicKey == tokenPublicKey); - BEAST_EXPECT(k.secretKey == tokenSecretKey); + if (BEAST_EXPECT(k.keys)) + { + BEAST_EXPECT(k.keys->publicKey == tokenPublicKey); + BEAST_EXPECT(k.keys->secretKey == tokenSecretKey); + } BEAST_EXPECT(k.nodeID == tokenNodeID); BEAST_EXPECT(k.manifest == tokenManifest); BEAST_EXPECT(!k.configInvalid()); @@ -153,7 +159,7 @@ class ValidatorKeys_test : public beast::unit_test::suite c.section(SECTION_VALIDATOR_TOKEN).append("badtoken"); ValidatorKeys k{c, journal}; BEAST_EXPECT(k.configInvalid()); - BEAST_EXPECT(k.publicKey.size() == 0); + BEAST_EXPECT(!k.keys); BEAST_EXPECT(k.manifest.empty()); } @@ -165,7 +171,7 @@ class ValidatorKeys_test : public beast::unit_test::suite ValidatorKeys k{c, journal}; BEAST_EXPECT(k.configInvalid()); - BEAST_EXPECT(k.publicKey.size() == 0); + BEAST_EXPECT(!k.keys); BEAST_EXPECT(k.manifest.empty()); } @@ -176,7 +182,7 @@ class ValidatorKeys_test : public beast::unit_test::suite ValidatorKeys k{c, journal}; BEAST_EXPECT(k.configInvalid()); - BEAST_EXPECT(k.publicKey.size() == 0); + BEAST_EXPECT(!k.keys); BEAST_EXPECT(k.manifest.empty()); } } diff --git a/src/test/app/ValidatorList_test.cpp b/src/test/app/ValidatorList_test.cpp index ff9b57f3ced..638c2060d32 100644 --- a/src/test/app/ValidatorList_test.cpp +++ b/src/test/app/ValidatorList_test.cpp @@ -221,7 +221,6 @@ class ValidatorList_test : public beast::unit_test::suite jtx::Env env( *this, jtx::envconfig(), nullptr, beast::severities::kDisabled); auto& app = env.app(); - PublicKey emptyLocalKey; std::vector const emptyCfgKeys; std::vector const emptyCfgPublishers; @@ -278,8 +277,8 @@ class ValidatorList_test : public beast::unit_test::suite env.journal); // Correct (empty) configuration - BEAST_EXPECT(trustedKeys->load( - emptyLocalKey, emptyCfgKeys, emptyCfgPublishers)); + BEAST_EXPECT( + trustedKeys->load({}, emptyCfgKeys, emptyCfgPublishers)); // load local validator key with or without manifest BEAST_EXPECT(trustedKeys->load( @@ -303,8 +302,7 @@ class ValidatorList_test : public beast::unit_test::suite app.config().legacy("database_path"), env.journal); - BEAST_EXPECT( - trustedKeys->load(emptyLocalKey, cfgKeys, emptyCfgPublishers)); + BEAST_EXPECT(trustedKeys->load({}, cfgKeys, emptyCfgPublishers)); for (auto const& n : configList) BEAST_EXPECT(trustedKeys->listed(n)); @@ -315,23 +313,21 @@ class ValidatorList_test : public beast::unit_test::suite std::vector cfgMasterKeys( {format(masterNode1), format(masterNode2, " Comment")}); - BEAST_EXPECT(trustedKeys->load( - emptyLocalKey, cfgMasterKeys, emptyCfgPublishers)); + BEAST_EXPECT( + trustedKeys->load({}, cfgMasterKeys, emptyCfgPublishers)); BEAST_EXPECT(trustedKeys->listed(masterNode1)); BEAST_EXPECT(trustedKeys->listed(masterNode2)); // load should reject invalid config keys + BEAST_EXPECT( + !trustedKeys->load({}, {"NotAPublicKey"}, emptyCfgPublishers)); BEAST_EXPECT(!trustedKeys->load( - emptyLocalKey, {"NotAPublicKey"}, emptyCfgPublishers)); - BEAST_EXPECT(!trustedKeys->load( - emptyLocalKey, - {format(randomNode(), "!")}, - emptyCfgPublishers)); + {}, {format(randomNode(), "!")}, emptyCfgPublishers)); // load terminates when encountering an invalid entry auto const goodKey = randomNode(); BEAST_EXPECT(!trustedKeys->load( - emptyLocalKey, + {}, {format(randomNode(), "!"), format(goodKey)}, emptyCfgPublishers)); BEAST_EXPECT(!trustedKeys->listed(goodKey)); @@ -408,8 +404,7 @@ class ValidatorList_test : public beast::unit_test::suite // load should reject invalid validator list signing keys std::vector badPublishers({"NotASigningKey"}); - BEAST_EXPECT( - !trustedKeys->load(emptyLocalKey, emptyCfgKeys, badPublishers)); + BEAST_EXPECT(!trustedKeys->load({}, emptyCfgKeys, badPublishers)); // load should reject validator list signing keys with invalid // encoding @@ -419,8 +414,7 @@ class ValidatorList_test : public beast::unit_test::suite for (auto const& key : keys) badPublishers.push_back(toBase58(TokenType::NodePublic, key)); - BEAST_EXPECT( - !trustedKeys->load(emptyLocalKey, emptyCfgKeys, badPublishers)); + BEAST_EXPECT(!trustedKeys->load({}, emptyCfgKeys, badPublishers)); for (auto const& key : keys) BEAST_EXPECT(!trustedKeys->trustedPublisher(key)); @@ -429,8 +423,7 @@ class ValidatorList_test : public beast::unit_test::suite for (auto const& key : keys) cfgPublishers.push_back(strHex(key)); - BEAST_EXPECT( - trustedKeys->load(emptyLocalKey, emptyCfgKeys, cfgPublishers)); + BEAST_EXPECT(trustedKeys->load({}, emptyCfgKeys, cfgPublishers)); for (auto const& key : keys) BEAST_EXPECT(trustedKeys->trustedPublisher(key)); } @@ -464,8 +457,7 @@ class ValidatorList_test : public beast::unit_test::suite std::vector cfgPublishers = { strHex(pubRevokedPublic), strHex(legitKey)}; - BEAST_EXPECT( - trustedKeys->load(emptyLocalKey, emptyCfgKeys, cfgPublishers)); + BEAST_EXPECT(trustedKeys->load({}, emptyCfgKeys, cfgPublishers)); BEAST_EXPECT(!trustedKeys->trustedPublisher(pubRevokedPublic)); BEAST_EXPECT(trustedKeys->trustedPublisher(legitKey)); @@ -569,10 +561,9 @@ class ValidatorList_test : public beast::unit_test::suite 1)); std::vector cfgKeys1({strHex(publisherPublic)}); - PublicKey emptyLocalKey; std::vector emptyCfgKeys; - BEAST_EXPECT(trustedKeys->load(emptyLocalKey, emptyCfgKeys, cfgKeys1)); + BEAST_EXPECT(trustedKeys->load({}, emptyCfgKeys, cfgKeys1)); std::map> const lists = []() { auto constexpr listSize = 20; @@ -954,10 +945,9 @@ class ValidatorList_test : public beast::unit_test::suite 1)); std::vector cfgKeys1({strHex(publisherPublic)}); - PublicKey emptyLocalKey; std::vector emptyCfgKeys; - BEAST_EXPECT(trustedKeys->load(emptyLocalKey, emptyCfgKeys, cfgKeys1)); + BEAST_EXPECT(trustedKeys->load({}, emptyCfgKeys, cfgKeys1)); std::vector const list = []() { auto constexpr listSize = 20; @@ -1066,7 +1056,6 @@ class ValidatorList_test : public beast::unit_test::suite std::string const siteUri = "testUpdateTrusted.test"; - PublicKey emptyLocalKeyOuter; ManifestCache manifestsOuter; jtx::Env env(*this); auto& app = env.app(); @@ -1096,8 +1085,8 @@ class ValidatorList_test : public beast::unit_test::suite unseenValidators.emplace(calcNodeID(valKey)); } - BEAST_EXPECT(trustedKeysOuter->load( - emptyLocalKeyOuter, cfgKeys, cfgPublishersOuter)); + BEAST_EXPECT( + trustedKeysOuter->load({}, cfgKeys, cfgPublishersOuter)); // updateTrusted should make all configured validators trusted // even if they are not active/seen @@ -1147,8 +1136,8 @@ class ValidatorList_test : public beast::unit_test::suite std::vector cfgKeys( {toBase58(TokenType::NodePublic, masterPublic)}); - BEAST_EXPECT(trustedKeysOuter->load( - emptyLocalKeyOuter, cfgKeys, cfgPublishersOuter)); + BEAST_EXPECT( + trustedKeysOuter->load({}, cfgKeys, cfgPublishersOuter)); auto const signingKeys1 = randomKeyPair(KeyType::secp256k1); auto const signingPublic1 = signingKeys1.first; @@ -1260,8 +1249,7 @@ class ValidatorList_test : public beast::unit_test::suite std::vector cfgPublishers({strHex(publisherPublic)}); std::vector emptyCfgKeys; - BEAST_EXPECT(trustedKeys->load( - emptyLocalKeyOuter, emptyCfgKeys, cfgPublishers)); + BEAST_EXPECT(trustedKeys->load({}, emptyCfgKeys, cfgPublishers)); TrustChanges changes = trustedKeys->updateTrusted( activeValidatorsOuter, @@ -1305,8 +1293,7 @@ class ValidatorList_test : public beast::unit_test::suite toBeSeen = calcNodeID(valKey); } - BEAST_EXPECT(trustedKeys->load( - emptyLocalKeyOuter, cfgKeys, cfgPublishersOuter)); + BEAST_EXPECT(trustedKeys->load({}, cfgKeys, cfgPublishersOuter)); TrustChanges changes = trustedKeys->updateTrusted( activeValidators, @@ -1339,7 +1326,6 @@ class ValidatorList_test : public beast::unit_test::suite app.config().legacy("database_path"), env.journal); - PublicKey emptyLocalKey; std::vector emptyCfgKeys; auto const publisherKeys = randomKeyPair(KeyType::secp256k1); auto const pubSigningKeys = randomKeyPair(KeyType::secp256k1); @@ -1352,8 +1338,7 @@ class ValidatorList_test : public beast::unit_test::suite std::vector cfgKeys({strHex(publisherKeys.first)}); - BEAST_EXPECT( - trustedKeys->load(emptyLocalKey, emptyCfgKeys, cfgKeys)); + BEAST_EXPECT(trustedKeys->load({}, emptyCfgKeys, cfgKeys)); std::vector list({randomValidator(), randomValidator()}); hash_set activeValidators( @@ -1463,8 +1448,7 @@ class ValidatorList_test : public beast::unit_test::suite cfgKeys.push_back(toBase58(TokenType::NodePublic, valKey)); activeValidators.emplace(calcNodeID(valKey)); activeKeys.emplace(valKey); - BEAST_EXPECT(trustedKeys->load( - emptyLocalKeyOuter, cfgKeys, cfgPublishers)); + BEAST_EXPECT(trustedKeys->load({}, cfgKeys, cfgPublishers)); TrustChanges changes = trustedKeys->updateTrusted( activeValidators, env.timeKeeper().now(), @@ -1564,11 +1548,10 @@ class ValidatorList_test : public beast::unit_test::suite std::vector cfgPublishers( {strHex(publisherPublic)}); - PublicKey emptyLocalKey; std::vector emptyCfgKeys; - BEAST_EXPECT(trustedKeys->load( - emptyLocalKey, emptyCfgKeys, cfgPublishers)); + BEAST_EXPECT( + trustedKeys->load({}, emptyCfgKeys, cfgPublishers)); auto const version = 1; auto const sequence = 1; @@ -1640,9 +1623,8 @@ class ValidatorList_test : public beast::unit_test::suite BEAST_EXPECT(trustedKeys->expires() == std::nullopt); // Config listed keys have maximum expiry - PublicKey emptyLocalKey; PublicKey localCfgListed = randomNode(); - trustedKeys->load(emptyLocalKey, {toStr(localCfgListed)}, {}); + trustedKeys->load({}, {toStr(localCfgListed)}, {}); BEAST_EXPECT( trustedKeys->expires() && trustedKeys->expires().value() == NetClock::time_point::max()); @@ -1688,11 +1670,10 @@ class ValidatorList_test : public beast::unit_test::suite std::vector cfgPublishers( {strHex(publisherPublic)}); - PublicKey emptyLocalKey; std::vector emptyCfgKeys; - BEAST_EXPECT(trustedKeys->load( - emptyLocalKey, emptyCfgKeys, cfgPublishers)); + BEAST_EXPECT( + trustedKeys->load({}, emptyCfgKeys, cfgPublishers)); auto const version = 2; auto const sequence1 = 1; @@ -1795,7 +1776,6 @@ class ValidatorList_test : public beast::unit_test::suite { testcase("NegativeUNL"); jtx::Env env(*this); - PublicKey emptyLocalKey; ManifestCache manifests; auto createValidatorList = @@ -1820,7 +1800,7 @@ class ValidatorList_test : public beast::unit_test::suite cfgKeys.push_back(toBase58(TokenType::NodePublic, valKey)); activeValidators.emplace(calcNodeID(valKey)); } - if (trustedKeys->load(emptyLocalKey, cfgKeys, cfgPublishers)) + if (trustedKeys->load({}, cfgKeys, cfgPublishers)) { trustedKeys->updateTrusted( activeValidators, diff --git a/src/test/app/ValidatorSite_test.cpp b/src/test/app/ValidatorSite_test.cpp index 8ec86feadce..00512d157ec 100644 --- a/src/test/app/ValidatorSite_test.cpp +++ b/src/test/app/ValidatorSite_test.cpp @@ -172,7 +172,6 @@ class ValidatorSite_test : public beast::unit_test::suite test::StreamSink sink; beast::Journal journal{sink}; - PublicKey emptyLocalKey; std::vector emptyCfgKeys; struct publisher { @@ -229,8 +228,7 @@ class ValidatorSite_test : public beast::unit_test::suite item.uri = uri.str(); } - BEAST_EXPECT( - trustedKeys.load(emptyLocalKey, emptyCfgKeys, cfgPublishers)); + BEAST_EXPECT(trustedKeys.load({}, emptyCfgKeys, cfgPublishers)); // Normally, tests will only need a fraction of this time, // but sometimes DNS resolution takes an inordinate amount @@ -239,7 +237,10 @@ class ValidatorSite_test : public beast::unit_test::suite std::vector uris; for (auto const& u : servers) + { + log << "Testing " << u.uri << std::endl; uris.push_back(u.uri); + } sites->load(uris); sites->start(); sites->join(); diff --git a/src/test/app/XChain_test.cpp b/src/test/app/XChain_test.cpp index e6e193adf10..e2816f06096 100644 --- a/src/test/app/XChain_test.cpp +++ b/src/test/app/XChain_test.cpp @@ -728,7 +728,7 @@ struct XChain_test : public beast::unit_test::suite, // Locking chain is XRP, // - Issuing chain is XRP with issuing chain is the root account. // --------------------------------------------------------------------- - Account a, b; + Account a("a"), b("b"); Issue ia, ib; std::tuple lcs{ @@ -5005,7 +5005,8 @@ struct XChainSim_test : public beast::unit_test::suite, // create 10 accounts + door funded on both chains, and store // in ChainStateTracker the initial amount of these accounts - Account doorXRPLocking, doorUSDLocking, doorUSDIssuing; + Account doorXRPLocking("doorXRPLocking"), + doorUSDLocking("doorUSDLocking"), doorUSDIssuing("doorUSDIssuing"); constexpr size_t num_acct = 10; auto a = [&doorXRPLocking, &doorUSDLocking, &doorUSDIssuing]() { diff --git a/src/test/basics/base58_test.cpp b/src/test/basics/base58_test.cpp new file mode 100644 index 00000000000..6f3d495d7a9 --- /dev/null +++ b/src/test/basics/base58_test.cpp @@ -0,0 +1,439 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2022 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef _MSC_VER + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { +namespace { + +[[nodiscard]] inline auto +randEngine() -> std::mt19937& +{ + static std::mt19937 r = [] { + std::random_device rd; + return std::mt19937{rd()}; + }(); + return r; +} + +constexpr int numTokenTypeIndexes = 9; + +[[nodiscard]] inline auto +tokenTypeAndSize(int i) -> std::tuple +{ + assert(i < numTokenTypeIndexes); + + switch (i) + { + using enum ripple::TokenType; + case 0: + return {None, 20}; + case 1: + return {NodePublic, 32}; + case 2: + return {NodePublic, 33}; + case 3: + return {NodePrivate, 32}; + case 4: + return {AccountID, 20}; + case 5: + return {AccountPublic, 32}; + case 6: + return {AccountPublic, 33}; + case 7: + return {AccountSecret, 32}; + case 8: + return {FamilySeed, 16}; + default: + throw std::invalid_argument( + "Invalid token selection passed to tokenTypeAndSize() " + "in " __FILE__); + } +} + +[[nodiscard]] inline auto +randomTokenTypeAndSize() -> std::tuple +{ + using namespace ripple; + auto& rng = randEngine(); + std::uniform_int_distribution<> d(0, 8); + return tokenTypeAndSize(d(rng)); +} + +// Return the token type and subspan of `d` to use as test data. +[[nodiscard]] inline auto +randomB256TestData(std::span d) + -> std::tuple> +{ + auto& rng = randEngine(); + std::uniform_int_distribution dist(0, 255); + auto [tokType, tokSize] = randomTokenTypeAndSize(); + std::generate(d.begin(), d.begin() + tokSize, [&] { return dist(rng); }); + return {tokType, d.subspan(0, tokSize)}; +} + +inline void +printAsChar(std::span a, std::span b) +{ + auto asString = [](std::span s) { + std::string r; + r.resize(s.size()); + std::copy(s.begin(), s.end(), r.begin()); + return r; + }; + auto sa = asString(a); + auto sb = asString(b); + std::cerr << "\n\n" << sa << "\n" << sb << "\n"; +} + +inline void +printAsInt(std::span a, std::span b) +{ + auto asString = [](std::span s) -> std::string { + std::stringstream sstr; + for (auto i : s) + { + sstr << std::setw(3) << int(i) << ','; + } + return sstr.str(); + }; + auto sa = asString(a); + auto sb = asString(b); + std::cerr << "\n\n" << sa << "\n" << sb << "\n"; +} + +} // namespace + +namespace multiprecision_utils { + +boost::multiprecision::checked_uint512_t +toBoostMP(std::span in) +{ + boost::multiprecision::checked_uint512_t mbp = 0; + for (auto i = in.rbegin(); i != in.rend(); ++i) + { + mbp <<= 64; + mbp += *i; + } + return mbp; +} + +std::vector +randomBigInt(std::uint8_t minSize = 1, std::uint8_t maxSize = 5) +{ + auto eng = randEngine(); + std::uniform_int_distribution numCoeffDist(minSize, maxSize); + std::uniform_int_distribution dist; + auto const numCoeff = numCoeffDist(eng); + std::vector coeffs; + coeffs.reserve(numCoeff); + for (int i = 0; i < numCoeff; ++i) + { + coeffs.push_back(dist(eng)); + } + return coeffs; +} +} // namespace multiprecision_utils + +class base58_test : public beast::unit_test::suite +{ + void + testMultiprecision() + { + testcase("b58_multiprecision"); + + using namespace boost::multiprecision; + + constexpr std::size_t iters = 100000; + auto eng = randEngine(); + std::uniform_int_distribution dist; + for (int i = 0; i < iters; ++i) + { + std::uint64_t const d = dist(eng); + if (!d) + continue; + auto bigInt = multiprecision_utils::randomBigInt(); + auto const boostBigInt = multiprecision_utils::toBoostMP( + std::span(bigInt.data(), bigInt.size())); + + auto const refDiv = boostBigInt / d; + auto const refMod = boostBigInt % d; + + auto const mod = b58_fast::detail::inplace_bigint_div_rem( + std::span(bigInt.data(), bigInt.size()), d); + auto const foundDiv = multiprecision_utils::toBoostMP(bigInt); + BEAST_EXPECT(refMod.convert_to() == mod); + BEAST_EXPECT(foundDiv == refDiv); + } + for (int i = 0; i < iters; ++i) + { + std::uint64_t const d = dist(eng); + auto bigInt = multiprecision_utils::randomBigInt(/*minSize*/ 2); + if (bigInt[bigInt.size() - 1] == + std::numeric_limits::max()) + { + bigInt[bigInt.size() - 1] -= 1; // Prevent overflow + } + auto const boostBigInt = multiprecision_utils::toBoostMP( + std::span(bigInt.data(), bigInt.size())); + + auto const refAdd = boostBigInt + d; + + b58_fast::detail::inplace_bigint_add( + std::span(bigInt.data(), bigInt.size()), d); + auto const foundAdd = multiprecision_utils::toBoostMP(bigInt); + BEAST_EXPECT(refAdd == foundAdd); + } + for (int i = 0; i < iters; ++i) + { + std::uint64_t const d = dist(eng); + auto bigInt = multiprecision_utils::randomBigInt(/* minSize */ 2); + // inplace mul requires the most significant coeff to be zero to + // hold the result. + bigInt[bigInt.size() - 1] = 0; + auto const boostBigInt = multiprecision_utils::toBoostMP( + std::span(bigInt.data(), bigInt.size())); + + auto const refMul = boostBigInt * d; + + b58_fast::detail::inplace_bigint_mul( + std::span(bigInt.data(), bigInt.size()), d); + auto const foundMul = multiprecision_utils::toBoostMP(bigInt); + BEAST_EXPECT(refMul == foundMul); + } + } + + void + testFastMatchesRef() + { + testcase("fast_matches_ref"); + auto testRawEncode = [&](std::span const& b256Data) { + std::array b58ResultBuf[2]; + std::array, 2> b58Result; + + std::array b256ResultBuf[2]; + std::array, 2> b256Result; + for (int i = 0; i < 2; ++i) + { + std::span const outBuf{b58ResultBuf[i]}; + if (i == 0) + { + auto const r = ripple::b58_fast::detail::b256_to_b58_be( + b256Data, outBuf); + BEAST_EXPECT(r); + b58Result[i] = r.value(); + } + else + { + std::array tmpBuf; + std::string const s = ripple::b58_ref::detail::encodeBase58( + b256Data.data(), + b256Data.size(), + tmpBuf.data(), + tmpBuf.size()); + BEAST_EXPECT(s.size()); + b58Result[i] = outBuf.subspan(0, s.size()); + std::copy(s.begin(), s.end(), b58Result[i].begin()); + } + } + if (BEAST_EXPECT(b58Result[0].size() == b58Result[1].size())) + { + if (!BEAST_EXPECT( + memcmp( + b58Result[0].data(), + b58Result[1].data(), + b58Result[0].size()) == 0)) + { + printAsChar(b58Result[0], b58Result[1]); + } + } + + for (int i = 0; i < 2; ++i) + { + std::span const outBuf{ + b256ResultBuf[i].data(), b256ResultBuf[i].size()}; + if (i == 0) + { + std::string const in( + b58Result[i].data(), + b58Result[i].data() + b58Result[i].size()); + auto const r = + ripple::b58_fast::detail::b58_to_b256_be(in, outBuf); + BEAST_EXPECT(r); + b256Result[i] = r.value(); + } + else + { + std::string const st( + b58Result[i].begin(), b58Result[i].end()); + std::string const s = + ripple::b58_ref::detail::decodeBase58(st); + BEAST_EXPECT(s.size()); + b256Result[i] = outBuf.subspan(0, s.size()); + std::copy(s.begin(), s.end(), b256Result[i].begin()); + } + } + + if (BEAST_EXPECT(b256Result[0].size() == b256Result[1].size())) + { + if (!BEAST_EXPECT( + memcmp( + b256Result[0].data(), + b256Result[1].data(), + b256Result[0].size()) == 0)) + { + printAsInt(b256Result[0], b256Result[1]); + } + } + }; + + auto testTokenEncode = [&](ripple::TokenType const tokType, + std::span const& b256Data) { + std::array b58ResultBuf[2]; + std::array, 2> b58Result; + + std::array b256ResultBuf[2]; + std::array, 2> b256Result; + for (int i = 0; i < 2; ++i) + { + std::span const outBuf{ + b58ResultBuf[i].data(), b58ResultBuf[i].size()}; + if (i == 0) + { + auto const r = ripple::b58_fast::encodeBase58Token( + tokType, b256Data, outBuf); + BEAST_EXPECT(r); + b58Result[i] = r.value(); + } + else + { + std::string const s = ripple::b58_ref::encodeBase58Token( + tokType, b256Data.data(), b256Data.size()); + BEAST_EXPECT(s.size()); + b58Result[i] = outBuf.subspan(0, s.size()); + std::copy(s.begin(), s.end(), b58Result[i].begin()); + } + } + if (BEAST_EXPECT(b58Result[0].size() == b58Result[1].size())) + { + if (!BEAST_EXPECT( + memcmp( + b58Result[0].data(), + b58Result[1].data(), + b58Result[0].size()) == 0)) + { + printAsChar(b58Result[0], b58Result[1]); + } + } + + for (int i = 0; i < 2; ++i) + { + std::span const outBuf{ + b256ResultBuf[i].data(), b256ResultBuf[i].size()}; + if (i == 0) + { + std::string const in( + b58Result[i].data(), + b58Result[i].data() + b58Result[i].size()); + auto const r = ripple::b58_fast::decodeBase58Token( + tokType, in, outBuf); + BEAST_EXPECT(r); + b256Result[i] = r.value(); + } + else + { + std::string const st( + b58Result[i].begin(), b58Result[i].end()); + std::string const s = + ripple::b58_ref::decodeBase58Token(st, tokType); + BEAST_EXPECT(s.size()); + b256Result[i] = outBuf.subspan(0, s.size()); + std::copy(s.begin(), s.end(), b256Result[i].begin()); + } + } + + if (BEAST_EXPECT(b256Result[0].size() == b256Result[1].size())) + { + if (!BEAST_EXPECT( + memcmp( + b256Result[0].data(), + b256Result[1].data(), + b256Result[0].size()) == 0)) + { + printAsInt(b256Result[0], b256Result[1]); + } + } + }; + + auto testIt = [&](ripple::TokenType const tokType, + std::span const& b256Data) { + testRawEncode(b256Data); + testTokenEncode(tokType, b256Data); + }; + + // test every token type with data where every byte is the same and the + // bytes range from 0-255 + for (int i = 0; i < numTokenTypeIndexes; ++i) + { + std::array b256DataBuf; + auto const [tokType, tokSize] = tokenTypeAndSize(i); + for (int d = 0; d <= 255; ++d) + { + memset(b256DataBuf.data(), d, tokSize); + testIt(tokType, std::span(b256DataBuf.data(), tokSize)); + } + } + + // test with random data + constexpr std::size_t iters = 100000; + for (int i = 0; i < iters; ++i) + { + std::array b256DataBuf; + auto const [tokType, b256Data] = randomB256TestData(b256DataBuf); + testIt(tokType, b256Data); + } + } + + void + run() override + { + testMultiprecision(); + testFastMatchesRef(); + } +}; + +BEAST_DEFINE_TESTSUITE(base58, ripple_basics, ripple); + +} // namespace test +} // namespace ripple +#endif // _MSC_VER diff --git a/src/test/consensus/Consensus_test.cpp b/src/test/consensus/Consensus_test.cpp index 6b0b4817875..5c7dc2626fe 100644 --- a/src/test/consensus/Consensus_test.cpp +++ b/src/test/consensus/Consensus_test.cpp @@ -44,35 +44,34 @@ class Consensus_test : public beast::unit_test::suite // Use default parameters ConsensusParms const p{}; - std::optional delay; // Bizarre times forcibly close BEAST_EXPECT(shouldCloseLedger( - true, 10, 10, 10, -10s, 10s, 1s, delay, 1s, p, journal_)); + true, 10, 10, 10, -10s, 10s, 1s, 1s, p, journal_)); BEAST_EXPECT(shouldCloseLedger( - true, 10, 10, 10, 100h, 10s, 1s, delay, 1s, p, journal_)); + true, 10, 10, 10, 100h, 10s, 1s, 1s, p, journal_)); BEAST_EXPECT(shouldCloseLedger( - true, 10, 10, 10, 10s, 100h, 1s, delay, 1s, p, journal_)); + true, 10, 10, 10, 10s, 100h, 1s, 1s, p, journal_)); // Rest of network has closed - BEAST_EXPECT(shouldCloseLedger( - true, 10, 3, 5, 10s, 10s, 10s, delay, 10s, p, journal_)); + BEAST_EXPECT( + shouldCloseLedger(true, 10, 3, 5, 10s, 10s, 10s, 10s, p, journal_)); // No transactions means wait until end of internval - BEAST_EXPECT(!shouldCloseLedger( - false, 10, 0, 0, 1s, 1s, 1s, delay, 10s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger( - false, 10, 0, 0, 1s, 10s, 1s, delay, 10s, p, journal_)); + BEAST_EXPECT( + !shouldCloseLedger(false, 10, 0, 0, 1s, 1s, 1s, 10s, p, journal_)); + BEAST_EXPECT( + shouldCloseLedger(false, 10, 0, 0, 1s, 10s, 1s, 10s, p, journal_)); // Enforce minimum ledger open time - BEAST_EXPECT(!shouldCloseLedger( - true, 10, 0, 0, 10s, 10s, 1s, delay, 10s, p, journal_)); + BEAST_EXPECT( + !shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 1s, 10s, p, journal_)); // Don't go too much faster than last time - BEAST_EXPECT(!shouldCloseLedger( - true, 10, 0, 0, 10s, 10s, 3s, delay, 10s, p, journal_)); + BEAST_EXPECT( + !shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 3s, 10s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger( - true, 10, 0, 0, 10s, 10s, 10s, delay, 10s, p, journal_)); + BEAST_EXPECT( + shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 10s, 10s, p, journal_)); } void @@ -110,10 +109,15 @@ class Consensus_test : public beast::unit_test::suite ConsensusState::MovedOn == checkConsensus(10, 2, 1, 8, 3s, 10s, p, true, journal_)); - // No peers makes it easy to agree + // If no peers, don't agree until time has passed. BEAST_EXPECT( - ConsensusState::Yes == + ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, p, true, journal_)); + + // Agree if no peers and enough time has passed. + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(0, 0, 0, 0, 3s, 16s, p, true, journal_)); } void diff --git a/src/test/consensus/NegativeUNL_test.cpp b/src/test/consensus/NegativeUNL_test.cpp index fee790281a0..8cbb57444bd 100644 --- a/src/test/consensus/NegativeUNL_test.cpp +++ b/src/test/consensus/NegativeUNL_test.cpp @@ -778,8 +778,10 @@ class NegativeUNLVoteInternal_test : public beast::unit_test::suite // one add, one remove auto txSet = std::make_shared( SHAMapType::TRANSACTION, env.app().getNodeFamily()); - PublicKey toDisableKey; - PublicKey toReEnableKey; + PublicKey toDisableKey( + derivePublicKey(KeyType::ed25519, randomSecretKey())); + PublicKey toReEnableKey( + derivePublicKey(KeyType::ed25519, randomSecretKey())); LedgerIndex seq(1234); BEAST_EXPECT(countTx(txSet) == 0); vote.addTx(seq, toDisableKey, NegativeUNLVote::ToDisable, txSet); diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 29c0d0ba75e..6d3008f7348 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -19,9 +19,6 @@ #ifndef RIPPLE_TEST_CSF_PEER_H_INCLUDED #define RIPPLE_TEST_CSF_PEER_H_INCLUDED -#include -#include -#include #include #include #include @@ -29,10 +26,6 @@ #include #include #include -#include -#include -#include -#include #include #include #include @@ -40,7 +33,6 @@ #include #include #include -#include namespace ripple { namespace test { @@ -166,13 +158,10 @@ struct Peer using NodeID_t = PeerID; using NodeKey_t = PeerKey; using TxSet_t = TxSet; - using CanonicalTxSet_t = TxSet; using PeerPosition_t = Position; using Result = ConsensusResult; using NodeKey = Validation::NodeKey; - using clock_type = Stopwatch; - //! Logging support that prefixes messages with the peer ID beast::WrappedSink sink; beast::Journal j; @@ -251,7 +240,7 @@ struct Peer // Quorum of validations needed for a ledger to be fully validated // TODO: Use the logic in ValidatorList to set this dynamically - std::size_t q = 0; + std::size_t quorum = 0; hash_set trustedKeys; @@ -261,16 +250,6 @@ struct Peer //! The collectors to report events to CollectorRefs& collectors; - mutable std::recursive_mutex mtx; - - std::optional delay; - - struct Null_test : public beast::unit_test::suite - { - void - run() override{}; - }; - /** Constructor @param i Unique PeerID @@ -517,8 +496,7 @@ struct Peer onClose( Ledger const& prevLedger, NetClock::time_point closeTime, - ConsensusMode mode, - clock_type& clock) + ConsensusMode mode) { issue(CloseLedger{prevLedger, openTxs}); @@ -530,9 +508,7 @@ struct Peer TxSet::calcID(openTxs), closeTime, now(), - id, - prevLedger.seq() + typename Ledger_t::Seq{1}, - scheduler.clock())); + id)); } void @@ -544,10 +520,11 @@ struct Peer ConsensusMode const& mode, Json::Value&& consensusJson) { - buildAndValidate( + onAccept( result, prevLedger, closeResolution, + rawCloseTimes, mode, std::move(consensusJson)); } @@ -555,18 +532,9 @@ struct Peer void onAccept( Result const& result, - ConsensusCloseTimes const& rawCloseTimes, - ConsensusMode const& mode, - Json::Value&& consensusJson, - std::pair&& txsBuilt) - { - } - - std::pair - buildAndValidate( - Result const& result, - Ledger_t const& prevLedger, + Ledger const& prevLedger, NetClock::duration const& closeResolution, + ConsensusCloseTimes const& rawCloseTimes, ConsensusMode const& mode, Json::Value&& consensusJson) { @@ -631,8 +599,6 @@ struct Peer startRound(); } }); - - return {}; } // Earliest allowed sequence number when checking for ledgers with more @@ -728,8 +694,8 @@ struct Peer std::size_t const count = validations.numTrustedForLedger(ledger.id()); std::size_t const numTrustedPeers = trustGraph.graph().outDegree(this); - q = static_cast(std::ceil(numTrustedPeers * 0.8)); - if (count >= q && ledger.isAncestor(fullyValidatedLedger)) + quorum = static_cast(std::ceil(numTrustedPeers * 0.8)); + if (count >= quorum && ledger.isAncestor(fullyValidatedLedger)) { issue(FullyValidateLedger{ledger, fullyValidatedLedger}); fullyValidatedLedger = ledger; @@ -884,13 +850,7 @@ struct Peer hash_set keys; for (auto const p : trustGraph.trustedPeers(this)) keys.insert(p->key); - return {q, keys}; - } - - std::size_t - quorum() const - { - return q; + return {quorum, keys}; } std::size_t @@ -1013,70 +973,6 @@ struct Peer return TxSet{res}; } - - LedgerMaster& - getLedgerMaster() const - { - Null_test test; - jtx::Env env(test); - - return env.app().getLedgerMaster(); - } - - void - clearValidating() - { - } - - bool - retryAccept( - Ledger_t const& newLedger, - std::optional>& - start) const - { - return false; - } - - std::recursive_mutex& - peekMutex() const - { - return mtx; - } - - void - endConsensus() const - { - } - - bool - validating() const - { - return false; - } - - std::optional - getValidationDelay() const - { - return delay; - } - - void - setValidationDelay( - std::optional vd = std::nullopt) const - { - } - - std::optional - getTimerDelay() const - { - return delay; - } - - void - setTimerDelay( - std::optional vd = std::nullopt) const - { - } }; } // namespace csf diff --git a/src/test/csf/Proposal.h b/src/test/csf/Proposal.h index 76f36877c81..d1cee16c1a7 100644 --- a/src/test/csf/Proposal.h +++ b/src/test/csf/Proposal.h @@ -30,7 +30,7 @@ namespace csf { /** Proposal is a position taken in the consensus process and is represented directly from the generic types. */ -using Proposal = ConsensusProposal; +using Proposal = ConsensusProposal; } // namespace csf } // namespace test diff --git a/src/test/json/MultivarJson_test.cpp b/src/test/json/MultivarJson_test.cpp deleted file mode 100644 index 8cb3a49aff2..00000000000 --- a/src/test/json/MultivarJson_test.cpp +++ /dev/null @@ -1,293 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/XRPLF/rippled/ - Copyright (c) 2023 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include - -#include -#include "ripple/beast/unit_test/suite.hpp" -#include "ripple/json/json_value.h" -#include -#include -#include -#include - -namespace ripple { -namespace test { - -struct MultivarJson_test : beast::unit_test::suite -{ - void - run() override - { - constexpr static Json::StaticString string1("string1"); - static Json::Value const str1{string1}; - - static Json::Value const obj1{[]() { - Json::Value obj1(Json::objectValue); - obj1["one"] = 1; - return obj1; - }()}; - - static Json::Value const jsonNull{}; - - MultivarJson<3> const subject({str1, obj1}); - static_assert(sizeof(subject) == sizeof(subject.val)); - static_assert(subject.size == subject.val.size()); - static_assert( - std::is_same_v>); - - BEAST_EXPECT(subject.val.size() == 3); - BEAST_EXPECT( - (subject.val == std::array{str1, obj1, jsonNull})); - BEAST_EXPECT( - (MultivarJson<3>({obj1, str1}).val == - std::array{obj1, str1, jsonNull})); - BEAST_EXPECT( - (MultivarJson<3>({jsonNull, obj1, str1}).val == - std::array{jsonNull, obj1, str1})); - - { - testcase("default copy construction / assignment"); - - MultivarJson<3> x{subject}; - - BEAST_EXPECT(x.val.size() == subject.val.size()); - BEAST_EXPECT(x.val[0] == subject.val[0]); - BEAST_EXPECT(x.val[1] == subject.val[1]); - BEAST_EXPECT(x.val[2] == subject.val[2]); - BEAST_EXPECT(x.val == subject.val); - BEAST_EXPECT(&x.val[0] != &subject.val[0]); - BEAST_EXPECT(&x.val[1] != &subject.val[1]); - BEAST_EXPECT(&x.val[2] != &subject.val[2]); - - MultivarJson<3> y; - BEAST_EXPECT((y.val == std::array{})); - y = subject; - BEAST_EXPECT(y.val == subject.val); - BEAST_EXPECT(&y.val[0] != &subject.val[0]); - BEAST_EXPECT(&y.val[1] != &subject.val[1]); - BEAST_EXPECT(&y.val[2] != &subject.val[2]); - - y = std::move(x); - BEAST_EXPECT(y.val == subject.val); - BEAST_EXPECT(&y.val[0] != &subject.val[0]); - BEAST_EXPECT(&y.val[1] != &subject.val[1]); - BEAST_EXPECT(&y.val[2] != &subject.val[2]); - } - - { - testcase("select"); - - BEAST_EXPECT( - subject.select([]() -> std::size_t { return 0; }) == str1); - BEAST_EXPECT( - subject.select([]() -> std::size_t { return 1; }) == obj1); - BEAST_EXPECT( - subject.select([]() -> std::size_t { return 2; }) == jsonNull); - - // Tests of requires clause - these are expected to match - static_assert([](auto&& v) { - return requires - { - v.select([]() -> std::size_t { return 0; }); - }; - }(subject)); - static_assert([](auto&& v) { - return requires - { - v.select([]() constexpr->std::size_t { return 0; }); - }; - }(subject)); - static_assert([](auto&& v) { - return requires - { - v.select([]() mutable -> std::size_t { return 0; }); - }; - }(subject)); - - // Tests of requires clause - these are expected NOT to match - static_assert([](auto&& a) { - return !requires - { - subject.select([]() -> int { return 0; }); - }; - }(subject)); - static_assert([](auto&& v) { - return !requires - { - v.select([]() -> void {}); - }; - }(subject)); - static_assert([](auto&& v) { - return !requires - { - v.select([]() -> bool { return false; }); - }; - }(subject)); - } - - { - struct foo_t final - { - }; - testcase("set"); - - auto x = MultivarJson<2>{{Json::objectValue, Json::objectValue}}; - x.set("name1", 42); - BEAST_EXPECT(x.val[0].isMember("name1")); - BEAST_EXPECT(x.val[1].isMember("name1")); - BEAST_EXPECT(x.val[0]["name1"].isInt()); - BEAST_EXPECT(x.val[1]["name1"].isInt()); - BEAST_EXPECT(x.val[0]["name1"].asInt() == 42); - BEAST_EXPECT(x.val[1]["name1"].asInt() == 42); - - x.set("name2", "bar"); - BEAST_EXPECT(x.val[0].isMember("name2")); - BEAST_EXPECT(x.val[1].isMember("name2")); - BEAST_EXPECT(x.val[0]["name2"].isString()); - BEAST_EXPECT(x.val[1]["name2"].isString()); - BEAST_EXPECT(x.val[0]["name2"].asString() == "bar"); - BEAST_EXPECT(x.val[1]["name2"].asString() == "bar"); - - // Tests of requires clause - these are expected to match - static_assert([](auto&& v) { - return requires - { - v.set("name", Json::nullValue); - }; - }(x)); - static_assert([](auto&& v) { - return requires - { - v.set("name", "value"); - }; - }(x)); - static_assert([](auto&& v) { - return requires - { - v.set("name", true); - }; - }(x)); - static_assert([](auto&& v) { - return requires - { - v.set("name", 42); - }; - }(x)); - - // Tests of requires clause - these are expected NOT to match - static_assert([](auto&& v) { - return !requires - { - v.set("name", foo_t{}); - }; - }(x)); - static_assert([](auto&& v) { - return !requires - { - v.set("name", std::nullopt); - }; - }(x)); - } - - { - testcase("isMember"); - - // Well defined behaviour even if we have different types of members - BEAST_EXPECT(subject.isMember("foo") == decltype(subject)::none); - - auto const makeJson = [](const char* key, int val) { - Json::Value obj1(Json::objectValue); - obj1[key] = val; - return obj1; - }; - - { - // All variants have element "One", none have element "Two" - MultivarJson<2> const s1{ - {makeJson("One", 12), makeJson("One", 42)}}; - BEAST_EXPECT(s1.isMember("One") == decltype(s1)::all); - BEAST_EXPECT(s1.isMember("Two") == decltype(s1)::none); - } - - { - // Some variants have element "One" and some have "Two" - MultivarJson<2> const s2{ - {makeJson("One", 12), makeJson("Two", 42)}}; - BEAST_EXPECT(s2.isMember("One") == decltype(s2)::some); - BEAST_EXPECT(s2.isMember("Two") == decltype(s2)::some); - } - - { - // Not all variants have element "One", because last one is null - MultivarJson<3> const s3{ - {makeJson("One", 12), makeJson("One", 42), {}}}; - BEAST_EXPECT(s3.isMember("One") == decltype(s3)::some); - BEAST_EXPECT(s3.isMember("Two") == decltype(s3)::none); - } - } - - { - // NOTE It's fine to change this test when we change API versions - testcase("apiVersionSelector"); - - static_assert(MultiApiJson::size == 2); - static MultiApiJson x{{obj1, str1}}; - - static_assert( - std::is_same_v); - static_assert([](auto&& v) { - return requires - { - v.select(apiVersionSelector(1)); - }; - }(x)); - - BEAST_EXPECT(x.select(apiVersionSelector(0)) == obj1); - BEAST_EXPECT(x.select(apiVersionSelector(2)) == str1); - - static_assert(apiVersionSelector(0)() == 0); - static_assert(apiVersionSelector(1)() == 0); - static_assert(apiVersionSelector(2)() == 1); - static_assert(apiVersionSelector(3)() == 1); - static_assert( - apiVersionSelector( - std::numeric_limits::max())() == 1); - } - - { - // There should be no reson to change this test - testcase("apiVersionSelector invariants"); - - static_assert( - apiVersionSelector(RPC::apiMinimumSupportedVersion)() == 0); - static_assert( - apiVersionSelector(RPC::apiBetaVersion)() + 1 // - == MultiApiJson::size); - - BEAST_EXPECT(MultiApiJson::size >= 1); - } - } -}; - -BEAST_DEFINE_TESTSUITE(MultivarJson, ripple_basics, ripple); - -} // namespace test -} // namespace ripple diff --git a/src/test/jtx/AMM.h b/src/test/jtx/AMM.h index c7c6f3b8477..4c6e8d78a4e 100644 --- a/src/test/jtx/AMM.h +++ b/src/test/jtx/AMM.h @@ -57,6 +57,67 @@ class LPToken } }; +struct CreateArg +{ + bool log = false; + std::uint16_t tfee = 0; + std::uint32_t fee = 0; + std::optional flags = std::nullopt; + std::optional seq = std::nullopt; + std::optional ms = std::nullopt; + std::optional err = std::nullopt; + bool close = true; +}; + +struct DepositArg +{ + std::optional account = std::nullopt; + std::optional tokens = std::nullopt; + std::optional asset1In = std::nullopt; + std::optional asset2In = std::nullopt; + std::optional maxEP = std::nullopt; + std::optional flags = std::nullopt; + std::optional> assets = std::nullopt; + std::optional seq = std::nullopt; + std::optional tfee = std::nullopt; + std::optional err = std::nullopt; +}; + +struct WithdrawArg +{ + std::optional account = std::nullopt; + std::optional tokens = std::nullopt; + std::optional asset1Out = std::nullopt; + std::optional asset2Out = std::nullopt; + std::optional maxEP = std::nullopt; + std::optional flags = std::nullopt; + std::optional> assets = std::nullopt; + std::optional seq = std::nullopt; + std::optional err = std::nullopt; +}; + +struct VoteArg +{ + std::optional account = std::nullopt; + std::uint32_t tfee = 0; + std::optional flags = std::nullopt; + std::optional seq = std::nullopt; + std::optional> assets = std::nullopt; + std::optional err = std::nullopt; +}; + +struct BidArg +{ + std::optional account = std::nullopt; + std::optional> bidMin = std::nullopt; + std::optional> bidMax = std::nullopt; + std::vector authAccounts = {}; + std::optional flags = std::nullopt; + std::optional seq = std::nullopt; + std::optional> assets = std::nullopt; + std::optional err = std::nullopt; +}; + /** Convenience class to test AMM functionality. */ class AMM @@ -91,13 +152,20 @@ class AMM std::optional flags = std::nullopt, std::optional seq = std::nullopt, std::optional ms = std::nullopt, - std::optional const& ter = std::nullopt); + std::optional const& ter = std::nullopt, + bool close = true); AMM(Env& env, Account const& account, STAmount const& asset1, STAmount const& asset2, ter const& ter, - bool log = false); + bool log = false, + bool close = true); + AMM(Env& env, + Account const& account, + STAmount const& asset1, + STAmount const& asset2, + CreateArg const& arg); /** Send amm_info RPC command */ @@ -189,6 +257,9 @@ class AMM std::optional const& tfee = std::nullopt, std::optional const& ter = std::nullopt); + IOUAmount + deposit(DepositArg const& arg); + IOUAmount withdraw( std::optional const& account, @@ -200,14 +271,15 @@ class AMM IOUAmount withdrawAll( std::optional const& account, - std::optional const& asset1OutDetails = std::nullopt) + std::optional const& asset1OutDetails = std::nullopt, + std::optional const& ter = std::nullopt) { return withdraw( account, std::nullopt, asset1OutDetails, asset1OutDetails ? tfOneAssetWithdrawAll : tfWithdrawAll, - std::nullopt); + ter); } IOUAmount @@ -230,6 +302,9 @@ class AMM std::optional const& seq, std::optional const& ter = std::nullopt); + IOUAmount + withdraw(WithdrawArg const& arg); + void vote( std::optional const& account, @@ -239,6 +314,9 @@ class AMM std::optional> const& assets = std::nullopt, std::optional const& ter = std::nullopt); + void + vote(VoteArg const& arg); + void bid(std::optional const& account, std::optional> const& bidMin = @@ -251,6 +329,9 @@ class AMM std::optional> const& assets = std::nullopt, std::optional const& ter = std::nullopt); + void + bid(BidArg const& arg); + AccountID const& ammAccount() const { diff --git a/src/test/jtx/Account.h b/src/test/jtx/Account.h index 1595d444354..cca92e76fe0 100644 --- a/src/test/jtx/Account.h +++ b/src/test/jtx/Account.h @@ -46,7 +46,7 @@ class Account /** The master account. */ static Account const master; - Account() = default; + Account() = delete; Account(Account&&) = default; Account(Account const&) = default; Account& diff --git a/src/test/jtx/Env.h b/src/test/jtx/Env.h index 6a55f2f9141..72cac29040a 100644 --- a/src/test/jtx/Env.h +++ b/src/test/jtx/Env.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -513,7 +512,11 @@ class Env of JTx submission. */ void - postconditions(JTx const& jt, TER ter, bool didApply); + postconditions( + JTx const& jt, + TER ter, + bool didApply, + Json::Value const& jr = Json::Value()); /** Apply funclets and submit. */ /** @{ */ @@ -755,40 +758,6 @@ Env::rpc(std::string const& cmd, Args&&... args) std::forward(args)...); } -/** - * The SingleVersionedTestCallable concept checks for a callable that takes - * an unsigned integer as its argument and returns void. - */ -template -concept SingleVersionedTestCallable = requires(T callable, unsigned int version) -{ - { - callable(version) - } - ->std::same_as; -}; - -/** - * The VersionedTestCallable concept checks if a set of callables all satisfy - * the SingleVersionedTestCallable concept. This allows forAllApiVersions to - * accept any number of functions. It executes a set of provided functions over - * a range of versions from RPC::apiMinimumSupportedVersion to - * RPC::apiBetaVersion. This is useful for running a series of tests or - * operations that need to be performed on multiple versions of an API. - */ -template -concept VersionedTestCallable = (... && SingleVersionedTestCallable); -void -forAllApiVersions(VersionedTestCallable auto... testCallable) -{ - for (auto testVersion = RPC::apiMinimumSupportedVersion; - testVersion <= RPC::apiMaximumValidVersion; - ++testVersion) - { - (..., testCallable(testVersion)); - } -} - } // namespace jtx } // namespace test } // namespace ripple diff --git a/src/test/jtx/Env_test.cpp b/src/test/jtx/Env_test.cpp index 6e26a40e25a..8a42b554b8e 100644 --- a/src/test/jtx/Env_test.cpp +++ b/src/test/jtx/Env_test.cpp @@ -29,7 +29,6 @@ #include #include -#include #include namespace ripple { @@ -51,7 +50,7 @@ class Env_test : public beast::unit_test::suite { using namespace jtx; { - Account a; + Account a("chenna"); Account b(a); a = b; a = std::move(b); @@ -748,8 +747,9 @@ class Env_test : public beast::unit_test::suite // Force the factor low enough to fail params[jss::fee_mult_max] = 1; params[jss::fee_div_max] = 2; - // RPC errors result in temINVALID - envs(noop(alice), fee(none), seq(none), ter(temINVALID))(params); + // RPC errors result in telENV_RPC_FAILED + envs(noop(alice), fee(none), seq(none), ter(telENV_RPC_FAILED))( + params); auto tx = env.tx(); BEAST_EXPECT(!tx); @@ -901,95 +901,6 @@ class Env_test : public beast::unit_test::suite pass(); } - void - testSyncSubmit() - { - using namespace jtx; - Env env(*this); - - auto const alice = Account{"alice"}; - auto const n = XRP(10000); - env.fund(n, alice); - BEAST_EXPECT(env.balance(alice) == n); - - // submit only - auto applyBlobTxn = [&env](char const* syncMode, auto&&... txnArgs) { - auto jt = env.jt(txnArgs...); - Serializer s; - jt.stx->add(s); - - Json::Value args{Json::objectValue}; - - args[jss::tx_blob] = strHex(s.slice()); - args[jss::fail_hard] = true; - args[jss::sync_mode] = syncMode; - - return env.rpc("json", "submit", args.toStyledString()); - }; - - auto jr = applyBlobTxn("sync", noop(alice)); - BEAST_EXPECT(jr[jss::result][jss::engine_result] == "tesSUCCESS"); - - jr = applyBlobTxn("async", noop(alice)); - BEAST_EXPECT(jr[jss::result][jss::engine_result] == "terSUBMITTED"); - // Make sure it gets processed before submitting and waiting. - env.app().getOPs().transactionBatch(true); - - auto applier = [&env]() { - while (!env.app().getOPs().transactionBatch(false)) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - }; - auto t = std::thread(applier); - - jr = applyBlobTxn("wait", noop(alice)); - BEAST_EXPECT(jr[jss::result][jss::engine_result] == "tesSUCCESS"); - t.join(); - - jr = applyBlobTxn("scott", noop(alice)); - BEAST_EXPECT(jr[jss::result][jss::error] == "invalidParams"); - - // sign and submit - auto applyJsonTxn = [&env]( - char const* syncMode, - std::string const secret, - Json::Value const& val) { - Json::Value args{Json::objectValue}; - args[jss::secret] = secret; - args[jss::tx_json] = val; - args[jss::fail_hard] = true; - args[jss::sync_mode] = syncMode; - - return env.rpc("json", "submit", args.toStyledString()); - }; - - Json::Value payment; - auto secret = toBase58(generateSeed("alice")); - payment = noop("alice"); - payment[sfSequence.fieldName] = env.seq("alice"); - payment[sfSetFlag.fieldName] = 0; - jr = applyJsonTxn("sync", secret, payment); - BEAST_EXPECT(jr[jss::result][jss::engine_result] == "tesSUCCESS"); - - payment[sfSequence.fieldName] = env.seq("alice"); - jr = applyJsonTxn("async", secret, payment); - BEAST_EXPECT(jr[jss::result][jss::engine_result] == "terSUBMITTED"); - - env.app().getOPs().transactionBatch(true); - payment[sfSequence.fieldName] = env.seq("alice"); - - auto aSeq = env.seq("alice"); - t = std::thread(applier); - jr = applyJsonTxn("wait", secret, payment); - BEAST_EXPECT(jr[jss::result][jss::engine_result] == "tesSUCCESS"); - t.join(); - // Ensure the last transaction was processed. - BEAST_EXPECT(env.seq("alice") == aSeq + 1); - - payment[sfSequence.fieldName] = env.seq("alice"); - jr = applyJsonTxn("scott", secret, payment); - BEAST_EXPECT(jr[jss::result][jss::error] == "invalidParams"); - } - void run() override { @@ -1015,7 +926,6 @@ class Env_test : public beast::unit_test::suite testSignAndSubmit(); testFeatures(); testExceptionalShutdown(); - testSyncSubmit(); } }; diff --git a/src/test/jtx/Oracle.h b/src/test/jtx/Oracle.h new file mode 100644 index 00000000000..f6fdbbff34a --- /dev/null +++ b/src/test/jtx/Oracle.h @@ -0,0 +1,186 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_TEST_JTX_ORACLE_H_INCLUDED +#define RIPPLE_TEST_JTX_ORACLE_H_INCLUDED + +#include +#include + +namespace ripple { +namespace test { +namespace jtx { +namespace oracle { + +// base asset, quote asset, price, scale +using DataSeries = std::vector, + std::optional>>; + +// Typical defaults for Create +struct CreateArg +{ + std::optional owner = std::nullopt; + std::optional documentID = 1; + DataSeries series = {{"XRP", "USD", 740, 1}}; + std::optional assetClass = "currency"; + std::optional provider = "provider"; + std::optional uri = "URI"; + std::optional lastUpdateTime = std::nullopt; + std::uint32_t flags = 0; + std::optional msig = std::nullopt; + std::optional seq = std::nullopt; + std::uint32_t fee = 10; + std::optional err = std::nullopt; + bool close = false; +}; + +// Typical defaults for Update +struct UpdateArg +{ + std::optional owner = std::nullopt; + std::optional documentID = std::nullopt; + DataSeries series = {}; + std::optional assetClass = std::nullopt; + std::optional provider = std::nullopt; + std::optional uri = "URI"; + std::optional lastUpdateTime = std::nullopt; + std::uint32_t flags = 0; + std::optional msig = std::nullopt; + std::optional seq = std::nullopt; + std::uint32_t fee = 10; + std::optional err = std::nullopt; +}; + +struct RemoveArg +{ + std::optional const& owner = std::nullopt; + std::optional const& documentID = std::nullopt; + std::optional const& msig = std::nullopt; + std::optional seq = std::nullopt; + std::uint32_t fee = 10; + std::optional const& err = std::nullopt; +}; + +// Simulate testStartTime as 10'000s from Ripple epoch time to make +// LastUpdateTime validation to work and to make unit-test consistent. +// The value doesn't matter much, it has to be greater +// than maxLastUpdateTimeDelta in order to pass LastUpdateTime +// validation {close-maxLastUpdateTimeDelta,close+maxLastUpdateTimeDelta}. +constexpr static std::chrono::seconds testStartTime = + epoch_offset + std::chrono::seconds(10'000); + +/** Oracle class facilitates unit-testing of the Price Oracle feature. + * It defines functions to create, update, and delete the Oracle object, + * to query for various states, and to call APIs. + */ +class Oracle +{ +private: + // Global fee if not 0 + static inline std::uint32_t fee = 0; + Env& env_; + AccountID owner_; + std::uint32_t documentID_; + +private: + void + submit( + Json::Value const& jv, + std::optional const& msig, + std::optional const& seq, + std::optional const& err); + +public: + Oracle(Env& env, CreateArg const& arg, bool submit = true); + + void + remove(RemoveArg const& arg); + + void + set(CreateArg const& arg); + void + set(UpdateArg const& arg); + + static Json::Value + aggregatePrice( + Env& env, + std::optional const& baseAsset, + std::optional const& quoteAsset, + std::optional>> const& + oracles = std::nullopt, + std::optional const& trim = std::nullopt, + std::optional const& timeTreshold = std::nullopt); + + std::uint32_t + documentID() const + { + return documentID_; + } + + [[nodiscard]] bool + exists() const + { + return exists(env_, owner_, documentID_); + } + + [[nodiscard]] static bool + exists(Env& env, AccountID const& account, std::uint32_t documentID); + + [[nodiscard]] bool + expectPrice(DataSeries const& pricess) const; + + [[nodiscard]] bool + expectLastUpdateTime(std::uint32_t lastUpdateTime) const; + + static Json::Value + ledgerEntry( + Env& env, + AccountID const& account, + std::variant const& documentID, + std::optional const& index = std::nullopt); + + Json::Value + ledgerEntry(std::optional const& index = std::nullopt) const + { + return Oracle::ledgerEntry(env_, owner_, documentID_, index); + } + + static void + setFee(std::uint32_t f) + { + fee = f; + } + + friend std::ostream& + operator<<(std::ostream& strm, Oracle const& oracle) + { + strm << oracle.ledgerEntry().toStyledString(); + return strm; + } +}; + +} // namespace oracle +} // namespace jtx +} // namespace test +} // namespace ripple + +#endif // RIPPLE_TEST_JTX_ORACLE_H_INCLUDED diff --git a/src/test/jtx/envconfig.h b/src/test/jtx/envconfig.h index 59b881539e5..02c99c52f09 100644 --- a/src/test/jtx/envconfig.h +++ b/src/test/jtx/envconfig.h @@ -25,6 +25,11 @@ namespace ripple { namespace test { +// frequently used macros defined here for convinience. +#define PORT_WS "port_ws" +#define PORT_RPC "port_rpc" +#define PORT_PEER "port_peer" + extern std::atomic envUseIPv4; inline const char* diff --git a/src/test/jtx/impl/AMM.cpp b/src/test/jtx/impl/AMM.cpp index dee1cb1bf5b..2d5ce90d306 100644 --- a/src/test/jtx/impl/AMM.cpp +++ b/src/test/jtx/impl/AMM.cpp @@ -57,7 +57,8 @@ AMM::AMM( std::optional flags, std::optional seq, std::optional ms, - std::optional const& ter) + std::optional const& ter, + bool close) : env_(env) , creatorAccount_(account) , asset1_(asset1) @@ -65,7 +66,7 @@ AMM::AMM( , ammID_(keylet::amm(asset1_.issue(), asset2_.issue()).key) , initialLPTokens_(initialTokens(asset1, asset2)) , log_(log) - , doClose_(true) + , doClose_(close) , lastPurchasePrice_(0) , bidMin_() , bidMax_() @@ -85,7 +86,8 @@ AMM::AMM( STAmount const& asset1, STAmount const& asset2, ter const& ter, - bool log) + bool log, + bool close) : AMM(env, account, asset1, @@ -96,7 +98,29 @@ AMM::AMM( std::nullopt, std::nullopt, std::nullopt, - ter) + ter, + close) +{ +} + +AMM::AMM( + Env& env, + Account const& account, + STAmount const& asset1, + STAmount const& asset2, + CreateArg const& arg) + : AMM(env, + account, + asset1, + asset2, + arg.log, + arg.tfee, + arg.fee, + arg.flags, + arg.seq, + arg.ms, + arg.err, + arg.close) { } @@ -470,6 +494,22 @@ AMM::deposit( return deposit(account, jv, assets, seq, ter); } +IOUAmount +AMM::deposit(DepositArg const& arg) +{ + return deposit( + arg.account, + arg.tokens, + arg.asset1In, + arg.asset2In, + arg.maxEP, + arg.flags, + arg.assets, + arg.seq, + arg.tfee, + arg.err); +} + IOUAmount AMM::withdraw( std::optional const& account, @@ -574,6 +614,21 @@ AMM::withdraw( return withdraw(account, jv, seq, assets, ter); } +IOUAmount +AMM::withdraw(WithdrawArg const& arg) +{ + return withdraw( + arg.account, + arg.tokens, + arg.asset1Out, + arg.asset2Out, + arg.maxEP, + arg.flags, + arg.assets, + arg.seq, + arg.err); +} + void AMM::vote( std::optional const& account, @@ -595,6 +650,12 @@ AMM::vote( submit(jv, seq, ter); } +void +AMM::vote(VoteArg const& arg) +{ + return vote(arg.account, arg.tfee, arg.flags, arg.seq, arg.assets, arg.err); +} + void AMM::bid( std::optional const& account, @@ -609,6 +670,9 @@ AMM::bid( if (auto const amm = env_.current()->read(keylet::amm(asset1_.issue(), asset2_.issue()))) { + assert( + !env_.current()->rules().enabled(fixInnerObjTemplate) || + amm->isFieldPresent(sfAuctionSlot)); if (amm->isFieldPresent(sfAuctionSlot)) { auto const& auctionSlot = @@ -663,6 +727,20 @@ AMM::bid( submit(jv, seq, ter); } +void +AMM::bid(BidArg const& arg) +{ + return bid( + arg.account, + arg.bidMin, + arg.bidMax, + arg.authAccounts, + arg.flags, + arg.seq, + arg.assets, + arg.err); +} + void AMM::submit( Json::Value const& jv, @@ -698,20 +776,30 @@ bool AMM::expectAuctionSlot(auto&& cb) const { if (auto const amm = - env_.current()->read(keylet::amm(asset1_.issue(), asset2_.issue())); - amm && amm->isFieldPresent(sfAuctionSlot)) + env_.current()->read(keylet::amm(asset1_.issue(), asset2_.issue()))) { - auto const& auctionSlot = - static_cast(amm->peekAtField(sfAuctionSlot)); - if (auctionSlot.isFieldPresent(sfAccount)) + assert( + !env_.current()->rules().enabled(fixInnerObjTemplate) || + amm->isFieldPresent(sfAuctionSlot)); + if (amm->isFieldPresent(sfAuctionSlot)) { - auto const slotFee = auctionSlot[sfDiscountedFee]; - auto const slotInterval = ammAuctionTimeSlot( - env_.app().timeKeeper().now().time_since_epoch().count(), - auctionSlot); - auto const slotPrice = auctionSlot[sfPrice].iou(); - auto const authAccounts = auctionSlot.getFieldArray(sfAuthAccounts); - return cb(slotFee, slotInterval, slotPrice, authAccounts); + auto const& auctionSlot = + static_cast(amm->peekAtField(sfAuctionSlot)); + if (auctionSlot.isFieldPresent(sfAccount)) + { + // This could fail in pre-fixInnerObjTemplate tests + // if the submitted transactions recreate one of + // the failure scenarios. Access as optional + // to avoid the failure. + auto const slotFee = auctionSlot[~sfDiscountedFee].value_or(0); + auto const slotInterval = ammAuctionTimeSlot( + env_.app().timeKeeper().now().time_since_epoch().count(), + auctionSlot); + auto const slotPrice = auctionSlot[sfPrice].iou(); + auto const authAccounts = + auctionSlot.getFieldArray(sfAuthAccounts); + return cb(slotFee, slotInterval, slotPrice, authAccounts); + } } } return false; diff --git a/src/test/jtx/impl/Env.cpp b/src/test/jtx/impl/Env.cpp index e82183c0001..6496f7df1d2 100644 --- a/src/test/jtx/impl/Env.cpp +++ b/src/test/jtx/impl/Env.cpp @@ -280,7 +280,9 @@ Env::parseResult(Json::Value const& jr) jr[jss::result].isMember(jss::engine_result_code)) ter = TER::fromInt(jr[jss::result][jss::engine_result_code].asInt()); else - ter = temINVALID; + // Use an error code that is not used anywhere in the transaction engine + // to distinguish this case. + ter = telENV_RPC_FAILED; return std::make_pair(ter, isTesSuccess(ter) || isTecClaim(ter)); } @@ -288,23 +290,29 @@ void Env::submit(JTx const& jt) { bool didApply; - if (jt.stx) - { - txid_ = jt.stx->getTransactionID(); - Serializer s; - jt.stx->add(s); - auto const jr = rpc("submit", strHex(s.slice())); + auto const jr = [&]() { + if (jt.stx) + { + txid_ = jt.stx->getTransactionID(); + Serializer s; + jt.stx->add(s); + auto const jr = rpc("submit", strHex(s.slice())); - std::tie(ter_, didApply) = parseResult(jr); - } - else - { - // Parsing failed or the JTx is - // otherwise missing the stx field. - ter_ = temMALFORMED; - didApply = false; - } - return postconditions(jt, ter_, didApply); + std::tie(ter_, didApply) = parseResult(jr); + + return jr; + } + else + { + // Parsing failed or the JTx is + // otherwise missing the stx field. + ter_ = temMALFORMED; + didApply = false; + + return Json::Value(); + } + }(); + return postconditions(jt, ter_, didApply, jr); } void @@ -342,11 +350,15 @@ Env::sign_and_submit(JTx const& jt, Json::Value params) std::tie(ter_, didApply) = parseResult(jr); - return postconditions(jt, ter_, didApply); + return postconditions(jt, ter_, didApply, jr); } void -Env::postconditions(JTx const& jt, TER ter, bool didApply) +Env::postconditions( + JTx const& jt, + TER ter, + bool didApply, + Json::Value const& jr) { if (jt.ter && !test.expect( @@ -356,6 +368,8 @@ Env::postconditions(JTx const& jt, TER ter, bool didApply) transHuman(*jt.ter) + ")")) { test.log << pretty(jt.jv) << std::endl; + if (jr) + test.log << pretty(jr) << std::endl; // Don't check postconditions if // we didn't get the expected result. return; diff --git a/src/test/jtx/impl/Oracle.cpp b/src/test/jtx/impl/Oracle.cpp new file mode 100644 index 00000000000..95da59952a0 --- /dev/null +++ b/src/test/jtx/impl/Oracle.cpp @@ -0,0 +1,292 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +#include + +#include + +namespace ripple { +namespace test { +namespace jtx { +namespace oracle { + +Oracle::Oracle(Env& env, CreateArg const& arg, bool submit) + : env_(env), owner_{}, documentID_{} +{ + // LastUpdateTime is checked to be in range + // {close-maxLastUpdateTimeDelta, close+maxLastUpdateTimeDelta}. + // To make the validation work and to make the clock consistent + // for tests running at different time, simulate Unix time starting + // on testStartTime since Ripple epoch. + auto const now = env_.timeKeeper().now(); + if (now.time_since_epoch().count() == 0 || arg.close) + env_.close(now + testStartTime - epoch_offset); + if (arg.owner) + owner_ = *arg.owner; + if (arg.documentID) + documentID_ = *arg.documentID; + if (submit) + set(arg); +} + +void +Oracle::remove(RemoveArg const& arg) +{ + Json::Value jv; + jv[jss::TransactionType] = jss::OracleDelete; + jv[jss::Account] = to_string(arg.owner.value_or(owner_)); + jv[jss::OracleDocumentID] = arg.documentID.value_or(documentID_); + if (Oracle::fee != 0) + jv[jss::Fee] = std::to_string(Oracle::fee); + else if (arg.fee != 0) + jv[jss::Fee] = std::to_string(arg.fee); + else + jv[jss::Fee] = std::to_string(env_.current()->fees().increment.drops()); + submit(jv, arg.msig, arg.seq, arg.err); +} + +void +Oracle::submit( + Json::Value const& jv, + std::optional const& msig, + std::optional const& seq, + std::optional const& err) +{ + if (msig) + { + if (seq && err) + env_(jv, *msig, *seq, *err); + else if (seq) + env_(jv, *msig, *seq); + else if (err) + env_(jv, *msig, *err); + else + env_(jv, *msig); + } + else if (seq && err) + env_(jv, *seq, *err); + else if (seq) + env_(jv, *seq); + else if (err) + env_(jv, *err); + else + env_(jv); + env_.close(); +} + +bool +Oracle::exists(Env& env, AccountID const& account, std::uint32_t documentID) +{ + assert(account.isNonZero()); + return env.le(keylet::oracle(account, documentID)) != nullptr; +} + +bool +Oracle::expectPrice(DataSeries const& series) const +{ + if (auto const sle = env_.le(keylet::oracle(owner_, documentID_))) + { + auto const& leSeries = sle->getFieldArray(sfPriceDataSeries); + if (leSeries.size() == 0 || leSeries.size() != series.size()) + return false; + for (auto const& data : series) + { + if (std::find_if( + leSeries.begin(), + leSeries.end(), + [&](STObject const& o) -> bool { + auto const& baseAsset = o.getFieldCurrency(sfBaseAsset); + auto const& quoteAsset = + o.getFieldCurrency(sfQuoteAsset); + auto const& price = o.getFieldU64(sfAssetPrice); + auto const& scale = o.getFieldU8(sfScale); + return baseAsset.getText() == std::get<0>(data) && + quoteAsset.getText() == std::get<1>(data) && + price == std::get<2>(data) && + scale == std::get<3>(data); + }) == leSeries.end()) + return false; + } + return true; + } + return false; +} + +bool +Oracle::expectLastUpdateTime(std::uint32_t lastUpdateTime) const +{ + auto const sle = env_.le(keylet::oracle(owner_, documentID_)); + return sle && (*sle)[sfLastUpdateTime] == lastUpdateTime; +} + +Json::Value +Oracle::aggregatePrice( + Env& env, + std::optional const& baseAsset, + std::optional const& quoteAsset, + std::optional>> const& + oracles, + std::optional const& trim, + std::optional const& timeThreshold) +{ + Json::Value jv; + Json::Value jvOracles(Json::arrayValue); + if (oracles) + { + for (auto const& id : *oracles) + { + Json::Value oracle; + oracle[jss::account] = to_string(id.first.id()); + oracle[jss::oracle_document_id] = id.second; + jvOracles.append(oracle); + } + jv[jss::oracles] = jvOracles; + } + if (trim) + jv[jss::trim] = *trim; + if (baseAsset) + jv[jss::base_asset] = *baseAsset; + if (quoteAsset) + jv[jss::quote_asset] = *quoteAsset; + if (timeThreshold) + jv[jss::time_threshold] = *timeThreshold; + + auto jr = env.rpc("json", "get_aggregate_price", to_string(jv)); + + if (jr.isObject() && jr.isMember(jss::result) && + jr[jss::result].isMember(jss::status)) + return jr[jss::result]; + return Json::nullValue; +} + +void +Oracle::set(UpdateArg const& arg) +{ + using namespace std::chrono; + Json::Value jv; + if (arg.owner) + owner_ = *arg.owner; + if (arg.documentID) + documentID_ = *arg.documentID; + jv[jss::TransactionType] = jss::OracleSet; + jv[jss::Account] = to_string(owner_); + jv[jss::OracleDocumentID] = documentID_; + if (arg.assetClass) + jv[jss::AssetClass] = strHex(*arg.assetClass); + if (arg.provider) + jv[jss::Provider] = strHex(*arg.provider); + if (arg.uri) + jv[jss::URI] = strHex(*arg.uri); + if (arg.flags != 0) + jv[jss::Flags] = arg.flags; + if (Oracle::fee != 0) + jv[jss::Fee] = std::to_string(Oracle::fee); + else if (arg.fee != 0) + jv[jss::Fee] = std::to_string(arg.fee); + else + jv[jss::Fee] = std::to_string(env_.current()->fees().increment.drops()); + // lastUpdateTime if provided is offset from testStartTime + if (arg.lastUpdateTime) + jv[jss::LastUpdateTime] = + to_string(testStartTime.count() + *arg.lastUpdateTime); + else + jv[jss::LastUpdateTime] = to_string( + duration_cast( + env_.current()->info().closeTime.time_since_epoch()) + .count() + + epoch_offset.count()); + Json::Value dataSeries(Json::arrayValue); + auto assetToStr = [](std::string const& s) { + // assume standard currency + if (s.size() == 3) + return s; + assert(s.size() <= 20); + // anything else must be 160-bit hex string + std::string h = strHex(s); + return strHex(s).append(40 - s.size() * 2, '0'); + }; + for (auto const& data : arg.series) + { + Json::Value priceData; + Json::Value price; + price[jss::BaseAsset] = assetToStr(std::get<0>(data)); + price[jss::QuoteAsset] = assetToStr(std::get<1>(data)); + if (std::get<2>(data)) + price[jss::AssetPrice] = *std::get<2>(data); + if (std::get<3>(data)) + price[jss::Scale] = *std::get<3>(data); + priceData[jss::PriceData] = price; + dataSeries.append(priceData); + } + jv[jss::PriceDataSeries] = dataSeries; + submit(jv, arg.msig, arg.seq, arg.err); +} + +void +Oracle::set(CreateArg const& arg) +{ + set(UpdateArg{ + .owner = arg.owner, + .documentID = arg.documentID, + .series = arg.series, + .assetClass = arg.assetClass, + .provider = arg.provider, + .uri = arg.uri, + .lastUpdateTime = arg.lastUpdateTime, + .flags = arg.flags, + .msig = arg.msig, + .seq = arg.seq, + .fee = arg.fee, + .err = arg.err}); +} + +Json::Value +Oracle::ledgerEntry( + Env& env, + AccountID const& account, + std::variant const& documentID, + std::optional const& index) +{ + Json::Value jvParams; + jvParams[jss::oracle][jss::account] = to_string(account); + if (std::holds_alternative(documentID)) + jvParams[jss::oracle][jss::oracle_document_id] = + std::get(documentID); + else + jvParams[jss::oracle][jss::oracle_document_id] = + std::get(documentID); + if (index) + { + std::uint32_t i; + if (boost::conversion::try_lexical_convert(*index, i)) + jvParams[jss::oracle][jss::ledger_index] = i; + else + jvParams[jss::oracle][jss::ledger_index] = *index; + } + return env.rpc("json", "ledger_entry", to_string(jvParams))[jss::result]; +} + +} // namespace oracle +} // namespace jtx +} // namespace test +} // namespace ripple \ No newline at end of file diff --git a/src/test/jtx/impl/envconfig.cpp b/src/test/jtx/impl/envconfig.cpp index 7f8163f5ee7..cbca244be6d 100644 --- a/src/test/jtx/impl/envconfig.cpp +++ b/src/test/jtx/impl/envconfig.cpp @@ -20,7 +20,6 @@ #include #include -#include #include namespace ripple { @@ -57,22 +56,22 @@ setupConfigForUnitTests(Config& cfg) cfg.deprecatedClearSection(ConfigSection::importNodeDatabase()); cfg.legacy("database_path", ""); cfg.setupControl(true, true, true); - cfg["server"].append("port_peer"); - cfg["port_peer"].set("ip", getEnvLocalhostAddr()); - cfg["port_peer"].set("port", port_peer); - cfg["port_peer"].set("protocol", "peer"); - - cfg["server"].append("port_rpc"); - cfg["port_rpc"].set("ip", getEnvLocalhostAddr()); - cfg["port_rpc"].set("admin", getEnvLocalhostAddr()); - cfg["port_rpc"].set("port", port_rpc); - cfg["port_rpc"].set("protocol", "http,ws2"); - - cfg["server"].append("port_ws"); - cfg["port_ws"].set("ip", getEnvLocalhostAddr()); - cfg["port_ws"].set("admin", getEnvLocalhostAddr()); - cfg["port_ws"].set("port", port_ws); - cfg["port_ws"].set("protocol", "ws"); + cfg["server"].append(PORT_PEER); + cfg[PORT_PEER].set("ip", getEnvLocalhostAddr()); + cfg[PORT_PEER].set("port", port_peer); + cfg[PORT_PEER].set("protocol", "peer"); + + cfg["server"].append(PORT_RPC); + cfg[PORT_RPC].set("ip", getEnvLocalhostAddr()); + cfg[PORT_RPC].set("admin", getEnvLocalhostAddr()); + cfg[PORT_RPC].set("port", port_rpc); + cfg[PORT_RPC].set("protocol", "http,ws2"); + + cfg["server"].append(PORT_WS); + cfg[PORT_WS].set("ip", getEnvLocalhostAddr()); + cfg[PORT_WS].set("admin", getEnvLocalhostAddr()); + cfg[PORT_WS].set("port", port_ws); + cfg[PORT_WS].set("protocol", "ws"); cfg.SSL_VERIFY = false; } @@ -81,35 +80,35 @@ namespace jtx { std::unique_ptr no_admin(std::unique_ptr cfg) { - (*cfg)["port_rpc"].set("admin", ""); - (*cfg)["port_ws"].set("admin", ""); + (*cfg)[PORT_RPC].set("admin", ""); + (*cfg)[PORT_WS].set("admin", ""); return cfg; } std::unique_ptr secure_gateway(std::unique_ptr cfg) { - (*cfg)["port_rpc"].set("admin", ""); - (*cfg)["port_ws"].set("admin", ""); - (*cfg)["port_rpc"].set("secure_gateway", getEnvLocalhostAddr()); + (*cfg)[PORT_RPC].set("admin", ""); + (*cfg)[PORT_WS].set("admin", ""); + (*cfg)[PORT_RPC].set("secure_gateway", getEnvLocalhostAddr()); return cfg; } std::unique_ptr admin_localnet(std::unique_ptr cfg) { - (*cfg)["port_rpc"].set("admin", "127.0.0.0/8"); - (*cfg)["port_ws"].set("admin", "127.0.0.0/8"); + (*cfg)[PORT_RPC].set("admin", "127.0.0.0/8"); + (*cfg)[PORT_WS].set("admin", "127.0.0.0/8"); return cfg; } std::unique_ptr secure_gateway_localnet(std::unique_ptr cfg) { - (*cfg)["port_rpc"].set("admin", ""); - (*cfg)["port_ws"].set("admin", ""); - (*cfg)["port_rpc"].set("secure_gateway", "127.0.0.0/8"); - (*cfg)["port_ws"].set("secure_gateway", "127.0.0.0/8"); + (*cfg)[PORT_RPC].set("admin", ""); + (*cfg)[PORT_WS].set("admin", ""); + (*cfg)[PORT_RPC].set("secure_gateway", "127.0.0.0/8"); + (*cfg)[PORT_WS].set("secure_gateway", "127.0.0.0/8"); return cfg; } @@ -127,7 +126,7 @@ validator(std::unique_ptr cfg, std::string const& seed) std::unique_ptr port_increment(std::unique_ptr cfg, int increment) { - for (auto const sectionName : {"port_peer", "port_rpc", "port_ws"}) + for (auto const sectionName : {PORT_PEER, PORT_RPC, PORT_WS}) { Section& s = (*cfg)[sectionName]; auto const port = s.get("port"); @@ -143,8 +142,8 @@ std::unique_ptr addGrpcConfig(std::unique_ptr cfg) { std::string port_grpc = std::to_string(port_base + 3); - (*cfg)["port_grpc"].set("ip", getEnvLocalhostAddr()); - (*cfg)["port_grpc"].set("port", port_grpc); + (*cfg)[SECTION_PORT_GRPC].set("ip", getEnvLocalhostAddr()); + (*cfg)[SECTION_PORT_GRPC].set("port", port_grpc); return cfg; } @@ -154,9 +153,9 @@ addGrpcConfigWithSecureGateway( std::string const& secureGateway) { std::string port_grpc = std::to_string(port_base + 3); - (*cfg)["port_grpc"].set("ip", getEnvLocalhostAddr()); - (*cfg)["port_grpc"].set("port", port_grpc); - (*cfg)["port_grpc"].set("secure_gateway", secureGateway); + (*cfg)[SECTION_PORT_GRPC].set("ip", getEnvLocalhostAddr()); + (*cfg)[SECTION_PORT_GRPC].set("port", port_grpc); + (*cfg)[SECTION_PORT_GRPC].set("secure_gateway", secureGateway); return cfg; } diff --git a/src/test/net/DatabaseDownloader_test.cpp b/src/test/net/DatabaseDownloader_test.cpp index d4ed2ebcedf..31c8abfd12c 100644 --- a/src/test/net/DatabaseDownloader_test.cpp +++ b/src/test/net/DatabaseDownloader_test.cpp @@ -147,6 +147,7 @@ class DatabaseDownloader_test : public beast::unit_test::suite // server to request from. Use the /textfile endpoint // to get a simple text file sent as response. auto server = createServer(env); + log << "Downloading DB from " << server->local_endpoint() << std::endl; ripple::test::detail::FileDirGuard const data{ *this, "downloads", "data", "", false, false}; @@ -225,6 +226,8 @@ class DatabaseDownloader_test : public beast::unit_test::suite auto server = createServer(env); auto host = server->local_endpoint().address().to_string(); auto port = std::to_string(server->local_endpoint().port()); + log << "Downloading DB from " << server->local_endpoint() + << std::endl; server->stop(); BEAST_EXPECT(dl->download( host, @@ -249,6 +252,8 @@ class DatabaseDownloader_test : public beast::unit_test::suite ripple::test::detail::FileDirGuard const datafile{ *this, "downloads", "data", "", false, false}; auto server = createServer(env, false); + log << "Downloading DB from " << server->local_endpoint() + << std::endl; BEAST_EXPECT(dl->download( server->local_endpoint().address().to_string(), std::to_string(server->local_endpoint().port()), @@ -272,6 +277,8 @@ class DatabaseDownloader_test : public beast::unit_test::suite ripple::test::detail::FileDirGuard const datafile{ *this, "downloads", "data", "", false, false}; auto server = createServer(env); + log << "Downloading DB from " << server->local_endpoint() + << std::endl; BEAST_EXPECT(dl->download( server->local_endpoint().address().to_string(), std::to_string(server->local_endpoint().port()), diff --git a/src/test/overlay/ProtocolVersion_test.cpp b/src/test/overlay/ProtocolVersion_test.cpp index 3bfba5099f4..a5a26fe74ec 100644 --- a/src/test/overlay/ProtocolVersion_test.cpp +++ b/src/test/overlay/ProtocolVersion_test.cpp @@ -88,7 +88,7 @@ class ProtocolVersion_test : public beast::unit_test::suite BEAST_EXPECT( negotiateProtocolVersion( "RTXP/1.2, XRPL/2.2, XRPL/2.3, XRPL/999.999") == - make_protocol(2, 3)); + make_protocol(2, 2)); BEAST_EXPECT( negotiateProtocolVersion("XRPL/999.999, WebSocket/1.0") == std::nullopt); diff --git a/src/test/overlay/reduce_relay_test.cpp b/src/test/overlay/reduce_relay_test.cpp index c722476dbaa..025e3295f2a 100644 --- a/src/test/overlay/reduce_relay_test.cpp +++ b/src/test/overlay/reduce_relay_test.cpp @@ -58,6 +58,12 @@ static constexpr std::uint32_t MAX_MESSAGES = 200000; class PeerPartial : public Peer { public: + PeerPartial() + : nodePublicKey_(derivePublicKey(KeyType::ed25519, randomSecretKey())) + { + } + + PublicKey nodePublicKey_; virtual ~PeerPartial() { } @@ -103,8 +109,7 @@ class PeerPartial : public Peer PublicKey const& getNodePublic() const override { - static PublicKey key{}; - return key; + return nodePublicKey_; } Json::Value json() override @@ -312,9 +317,8 @@ class Validator using Links = std::unordered_map; public: - Validator() + Validator() : pkey_(std::get<0>(randomKeyPair(KeyType::ed25519))) { - pkey_ = std::get<0>(randomKeyPair(KeyType::ed25519)); protocol::TMValidation v; v.set_validation("validation"); message_ = std::make_shared(v, protocol::mtVALIDATION, pkey_); @@ -439,7 +443,7 @@ class Validator private: Links links_; - PublicKey pkey_{}; + PublicKey pkey_; MessageSPtr message_ = nullptr; inline static std::uint16_t sid_ = 0; std::uint16_t id_ = 0; @@ -926,7 +930,7 @@ class reduce_relay_test : public beast::unit_test::suite bool isSelected_ = false; Peer::id_t peer_; std::uint16_t validator_; - PublicKey key_; + std::optional key_; time_point time_; bool handled_ = false; }; @@ -1052,17 +1056,17 @@ class reduce_relay_test : public beast::unit_test::suite // 4) peer is in Slot's peers_ (if not then it is deleted // by Slots::deleteIdlePeers()) bool mustHandle = false; - if (event.state_ == State::On) + if (event.state_ == State::On && BEAST_EXPECT(event.key_)) { event.isSelected_ = - network_.overlay().isSelected(event.key_, event.peer_); - auto peers = network_.overlay().getPeers(event.key_); + network_.overlay().isSelected(*event.key_, event.peer_); + auto peers = network_.overlay().getPeers(*event.key_); auto d = reduce_relay::epoch(now).count() - std::get<3>(peers[event.peer_]); mustHandle = event.isSelected_ && d > milliseconds(reduce_relay::IDLED).count() && network_.overlay().inState( - event.key_, reduce_relay::PeerState::Squelched) > + *event.key_, reduce_relay::PeerState::Squelched) > 0 && peers.find(event.peer_) != peers.end(); } diff --git a/src/test/peerfinder/PeerFinder_test.cpp b/src/test/peerfinder/PeerFinder_test.cpp index b0750e689f3..daa316e566c 100644 --- a/src/test/peerfinder/PeerFinder_test.cpp +++ b/src/test/peerfinder/PeerFinder_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -157,6 +158,86 @@ class PeerFinder_test : public beast::unit_test::suite BEAST_EXPECT(n <= (seconds + 59) / 60); } + void + test_duplicateOutIn() + { + testcase("duplicate out/in"); + TestStore store; + TestChecker checker; + TestStopwatch clock; + Logic logic(clock, store, checker, journal_); + logic.addFixedPeer( + "test", beast::IP::Endpoint::from_string("65.0.0.1:5")); + { + Config c; + c.autoConnect = false; + c.listeningPort = 1024; + c.ipLimit = 2; + logic.config(c); + } + + auto const list = logic.autoconnect(); + if (BEAST_EXPECT(!list.empty())) + { + BEAST_EXPECT(list.size() == 1); + auto const remote = list.front(); + auto const slot1 = logic.new_outbound_slot(remote); + if (BEAST_EXPECT(slot1 != nullptr)) + { + BEAST_EXPECT( + logic.connectedAddresses_.count(remote.address()) == 1); + auto const local = + beast::IP::Endpoint::from_string("65.0.0.2:1024"); + auto const slot2 = logic.new_inbound_slot(local, remote); + BEAST_EXPECT( + logic.connectedAddresses_.count(remote.address()) == 1); + if (!BEAST_EXPECT(slot2 == nullptr)) + logic.on_closed(slot2); + logic.on_closed(slot1); + } + } + } + + void + test_duplicateInOut() + { + testcase("duplicate in/out"); + TestStore store; + TestChecker checker; + TestStopwatch clock; + Logic logic(clock, store, checker, journal_); + logic.addFixedPeer( + "test", beast::IP::Endpoint::from_string("65.0.0.1:5")); + { + Config c; + c.autoConnect = false; + c.listeningPort = 1024; + c.ipLimit = 2; + logic.config(c); + } + + auto const list = logic.autoconnect(); + if (BEAST_EXPECT(!list.empty())) + { + BEAST_EXPECT(list.size() == 1); + auto const remote = list.front(); + auto const local = + beast::IP::Endpoint::from_string("65.0.0.2:1024"); + auto const slot1 = logic.new_inbound_slot(local, remote); + if (BEAST_EXPECT(slot1 != nullptr)) + { + BEAST_EXPECT( + logic.connectedAddresses_.count(remote.address()) == 1); + auto const slot2 = logic.new_outbound_slot(remote); + BEAST_EXPECT( + logic.connectedAddresses_.count(remote.address()) == 1); + if (!BEAST_EXPECT(slot2 == nullptr)) + logic.on_closed(slot2); + logic.on_closed(slot1); + } + } + } + void test_config() { @@ -279,6 +360,8 @@ class PeerFinder_test : public beast::unit_test::suite { test_backoff1(); test_backoff2(); + test_duplicateOutIn(); + test_duplicateInOut(); test_config(); test_invalid_config(); } diff --git a/src/test/protocol/ApiVersion_test.cpp b/src/test/protocol/ApiVersion_test.cpp new file mode 100644 index 00000000000..f4aba9e9966 --- /dev/null +++ b/src/test/protocol/ApiVersion_test.cpp @@ -0,0 +1,75 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/XRPLF/rippled/ + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { +struct ApiVersion_test : beast::unit_test::suite +{ + void + run() override + { + { + testcase("API versions invariants"); + + static_assert( + RPC::apiMinimumSupportedVersion <= + RPC::apiMaximumSupportedVersion); + static_assert( + RPC::apiMinimumSupportedVersion <= RPC::apiMaximumValidVersion); + static_assert( + RPC::apiMaximumSupportedVersion <= RPC::apiMaximumValidVersion); + static_assert(RPC::apiBetaVersion <= RPC::apiMaximumValidVersion); + + BEAST_EXPECT(true); + } + + { + // Update when we change versions + testcase("API versions"); + + static_assert(RPC::apiMinimumSupportedVersion >= 1); + static_assert(RPC::apiMinimumSupportedVersion < 2); + static_assert(RPC::apiMaximumSupportedVersion >= 2); + static_assert(RPC::apiMaximumSupportedVersion < 3); + static_assert(RPC::apiMaximumValidVersion >= 3); + static_assert(RPC::apiMaximumValidVersion < 4); + static_assert(RPC::apiBetaVersion >= 3); + static_assert(RPC::apiBetaVersion < 4); + + BEAST_EXPECT(true); + } + } +}; + +BEAST_DEFINE_TESTSUITE(ApiVersion, protocol, ripple); + +} // namespace test +} // namespace ripple diff --git a/src/test/protocol/Memo_test.cpp b/src/test/protocol/Memo_test.cpp index b39482e42d0..89ae6dfe18a 100644 --- a/src/test/protocol/Memo_test.cpp +++ b/src/test/protocol/Memo_test.cpp @@ -56,7 +56,7 @@ class Memo_test : public beast::unit_test::suite JTx memoSize = makeJtxWithMemo(); memoSize.jv[sfMemos.jsonName][0u][sfMemo.jsonName] [sfMemoData.jsonName] = std::string(2020, '0'); - env(memoSize, ter(temINVALID)); + env(memoSize, ter(telENV_RPC_FAILED)); // This memo is just barely small enough. memoSize.jv[sfMemos.jsonName][0u][sfMemo.jsonName] @@ -72,7 +72,7 @@ class Memo_test : public beast::unit_test::suite auto& m = mi[sfCreatedNode.jsonName]; // CreatedNode in Memos m[sfMemoData.jsonName] = "3030303030"; - env(memoNonMemo, ter(temINVALID)); + env(memoNonMemo, ter(telENV_RPC_FAILED)); } { // Put an invalid field in a Memo object. @@ -80,7 +80,7 @@ class Memo_test : public beast::unit_test::suite memoExtra .jv[sfMemos.jsonName][0u][sfMemo.jsonName][sfFlags.jsonName] = 13; - env(memoExtra, ter(temINVALID)); + env(memoExtra, ter(telENV_RPC_FAILED)); } { // Put a character that is not allowed in a URL in a MemoType field. @@ -88,7 +88,7 @@ class Memo_test : public beast::unit_test::suite memoBadChar.jv[sfMemos.jsonName][0u][sfMemo.jsonName] [sfMemoType.jsonName] = strHex(std::string_view("ONE +#include + +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +namespace { + +// This needs to be in a namespace because of deduction guide +template +struct Overload : Ts... +{ + using Ts::operator()...; +}; +template +Overload(Ts...) -> Overload; + +} // namespace + +struct MultiApiJson_test : beast::unit_test::suite +{ + static auto + makeJson(const char* key, int val) + { + Json::Value obj1(Json::objectValue); + obj1[key] = val; + return obj1; + } + + constexpr static auto index = + [](unsigned int v) constexpr noexcept -> std::size_t + { + return v - 1; + }; + + template + constexpr static auto valid = [](unsigned int v) constexpr noexcept -> bool + { + return v > 0 && v <= size; + }; + + void + run() override + { + using ripple::detail::MultiApiJson; + + Json::Value const obj1 = makeJson("value", 1); + Json::Value const obj2 = makeJson("value", 2); + Json::Value const obj3 = makeJson("value", 3); + Json::Value const jsonNull{}; + + MultiApiJson<1, 3> subject{}; + static_assert(sizeof(subject) == sizeof(subject.val)); + static_assert(subject.size == subject.val.size()); + static_assert( + std::is_same_v>); + + BEAST_EXPECT(subject.val.size() == 3); + BEAST_EXPECT( + (subject.val == + std::array{jsonNull, jsonNull, jsonNull})); + + subject.val[0] = obj1; + subject.val[1] = obj2; + + { + testcase("forApiVersions, forAllApiVersions"); + + // Some static data for test inputs + static const int primes[] = {2, 3, 5, 7, 11, 13, 17, 19, 23, + 29, 31, 37, 41, 43, 47, 53, 59, 61, + 67, 71, 73, 79, 83, 89, 97}; + static_assert(std::size(primes) > RPC::apiMaximumValidVersion); + + MultiApiJson<1, 3> s1{}; + static_assert( + s1.size == + RPC::apiMaximumValidVersion + 1 - + RPC::apiMinimumSupportedVersion); + + int productAllVersions = 1; + for (unsigned i = RPC::apiMinimumSupportedVersion; + i <= RPC::apiMaximumValidVersion; + ++i) + { + auto const index = i - RPC::apiMinimumSupportedVersion; + BEAST_EXPECT(index == s1.index(i)); + BEAST_EXPECT(s1.valid(i)); + s1.val[index] = makeJson("value", primes[i]); + productAllVersions *= primes[i]; + } + BEAST_EXPECT(!s1.valid(0)); + BEAST_EXPECT(!s1.valid(RPC::apiMaximumValidVersion + 1)); + BEAST_EXPECT( + !s1.valid(std::numeric_limits::max())); + + int result = 1; + static_assert( + RPC::apiMinimumSupportedVersion + 1 <= + RPC::apiMaximumValidVersion); + forApiVersions< + RPC::apiMinimumSupportedVersion, + RPC::apiMinimumSupportedVersion + 1>( + std::as_const(s1).visit(), + [this]( + Json::Value const& json, + unsigned int version, + int* result) { + BEAST_EXPECT( + version >= RPC::apiMinimumSupportedVersion && + version <= RPC::apiMinimumSupportedVersion + 1); + if (BEAST_EXPECT(json.isMember("value"))) + { + *result *= json["value"].asInt(); + } + }, + &result); + BEAST_EXPECT( + result == + primes[RPC::apiMinimumSupportedVersion] * + primes[RPC::apiMinimumSupportedVersion + 1]); + + // Check all the values with mutable data + forAllApiVersions( + s1.visit(), [&s1, this](Json::Value& json, auto version) { + BEAST_EXPECT(s1.val[s1.index(version)] == json); + if (BEAST_EXPECT(json.isMember("value"))) + { + BEAST_EXPECT(json["value"].asInt() == primes[version]); + } + }); + + result = 1; + forAllApiVersions( + std::as_const(s1).visit(), + [this]( + Json::Value const& json, + unsigned int version, + int* result) { + BEAST_EXPECT( + version >= RPC::apiMinimumSupportedVersion && + version <= RPC::apiMaximumValidVersion); + if (BEAST_EXPECT(json.isMember("value"))) + { + *result *= json["value"].asInt(); + } + }, + &result); + + BEAST_EXPECT(result == productAllVersions); + + // Several overloads we want to fail + static_assert([](auto&& v) { + return !requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](Json::Value&, auto) {}); // missing const + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return !requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](Json::Value&) {}); // missing const + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return !requires + { + forAllApiVersions( + std::forward(v).visit(), // + []() {}); // missing parameters + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return !requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](auto) {}, + 1); // missing parameters + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return !requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](auto, auto) {}, + 1); // missing parameters + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return !requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](auto, auto, const char*) {}, + 1); // parameter type mismatch + }; + }(std::as_const(s1))); + + // Sanity checks + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](auto) {}); + }; + }(s1)); + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](Json::Value const&) {}); + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](auto...) {}); + }; + }(s1)); + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](Json::Value const&, auto...) {}); + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](Json::Value&, auto, auto, auto...) {}, + 0, + ""); + }; + }(s1)); + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + []( + Json::Value const&, + std::integral_constant, + int, + const char*) {}, + 0, + ""); + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](auto...) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return requires + { + forAllApiVersions( + std::forward(v).visit(), // + [](auto...) {}); + }; + }(std::move(std::as_const(s1)))); + } + + { + testcase("default copy construction / assignment"); + + MultiApiJson<1, 3> x{subject}; + + BEAST_EXPECT(x.val.size() == subject.val.size()); + BEAST_EXPECT(x.val[0] == subject.val[0]); + BEAST_EXPECT(x.val[1] == subject.val[1]); + BEAST_EXPECT(x.val[2] == subject.val[2]); + BEAST_EXPECT(x.val == subject.val); + BEAST_EXPECT(&x.val[0] != &subject.val[0]); + BEAST_EXPECT(&x.val[1] != &subject.val[1]); + BEAST_EXPECT(&x.val[2] != &subject.val[2]); + + MultiApiJson<1, 3> y; + BEAST_EXPECT((y.val == std::array{})); + y = subject; + BEAST_EXPECT(y.val == subject.val); + BEAST_EXPECT(&y.val[0] != &subject.val[0]); + BEAST_EXPECT(&y.val[1] != &subject.val[1]); + BEAST_EXPECT(&y.val[2] != &subject.val[2]); + + y = std::move(x); + BEAST_EXPECT(y.val == subject.val); + BEAST_EXPECT(&y.val[0] != &subject.val[0]); + BEAST_EXPECT(&y.val[1] != &subject.val[1]); + BEAST_EXPECT(&y.val[2] != &subject.val[2]); + } + + { + testcase("set"); + + auto x = MultiApiJson<1, 2>{Json::objectValue}; + x.set("name1", 42); + BEAST_EXPECT(x.val[0].isMember("name1")); + BEAST_EXPECT(x.val[1].isMember("name1")); + BEAST_EXPECT(x.val[0]["name1"].isInt()); + BEAST_EXPECT(x.val[1]["name1"].isInt()); + BEAST_EXPECT(x.val[0]["name1"].asInt() == 42); + BEAST_EXPECT(x.val[1]["name1"].asInt() == 42); + + x.set("name2", "bar"); + BEAST_EXPECT(x.val[0].isMember("name2")); + BEAST_EXPECT(x.val[1].isMember("name2")); + BEAST_EXPECT(x.val[0]["name2"].isString()); + BEAST_EXPECT(x.val[1]["name2"].isString()); + BEAST_EXPECT(x.val[0]["name2"].asString() == "bar"); + BEAST_EXPECT(x.val[1]["name2"].asString() == "bar"); + + // Tests of requires clause - these are expected to match + static_assert([](auto&& v) { + return requires + { + v.set("name", Json::nullValue); + }; + }(x)); + static_assert([](auto&& v) { + return requires + { + v.set("name", "value"); + }; + }(x)); + static_assert([](auto&& v) { + return requires + { + v.set("name", true); + }; + }(x)); + static_assert([](auto&& v) { + return requires + { + v.set("name", 42); + }; + }(x)); + + // Tests of requires clause - these are expected NOT to match + struct foo_t final + { + }; + static_assert([](auto&& v) { + return !requires + { + v.set("name", foo_t{}); + }; + }(x)); + static_assert([](auto&& v) { + return !requires + { + v.set("name", std::nullopt); + }; + }(x)); + } + + { + testcase("isMember"); + + // Well defined behaviour even if we have different types of members + BEAST_EXPECT(subject.isMember("foo") == decltype(subject)::none); + + { + // All variants have element "One", none have element "Two" + MultiApiJson<1, 2> s1{}; + s1.val[0] = makeJson("One", 12); + s1.val[1] = makeJson("One", 42); + BEAST_EXPECT(s1.isMember("One") == decltype(s1)::all); + BEAST_EXPECT(s1.isMember("Two") == decltype(s1)::none); + } + + { + // Some variants have element "One" and some have "Two" + MultiApiJson<1, 2> s2{}; + s2.val[0] = makeJson("One", 12); + s2.val[1] = makeJson("Two", 42); + BEAST_EXPECT(s2.isMember("One") == decltype(s2)::some); + BEAST_EXPECT(s2.isMember("Two") == decltype(s2)::some); + } + + { + // Not all variants have element "One", because last one is null + MultiApiJson<1, 3> s3{}; + s3.val[0] = makeJson("One", 12); + s3.val[1] = makeJson("One", 42); + BEAST_EXPECT(s3.isMember("One") == decltype(s3)::some); + BEAST_EXPECT(s3.isMember("Two") == decltype(s3)::none); + } + } + + { + testcase("visitor"); + + MultiApiJson<1, 3> s1{}; + s1.val[0] = makeJson("value", 2); + s1.val[1] = makeJson("value", 3); + s1.val[2] = makeJson("value", 5); + + BEAST_EXPECT(not s1.valid(0)); + BEAST_EXPECT(s1.index(0) == 0); + + BEAST_EXPECT(s1.valid(1)); + BEAST_EXPECT(s1.index(1) == 0); + + BEAST_EXPECT(not s1.valid(4)); + + // Test different overloads + static_assert([](auto&& v) { + return requires + { + v.visitor( + v, + std::integral_constant{}, + [](Json::Value&, std::integral_constant) { + }); + }; + }(s1)); + BEAST_EXPECT( + s1.visitor( + s1, + std::integral_constant{}, + Overload{ + [](Json::Value& v, + std::integral_constant) { + return v["value"].asInt(); + }, + [](Json::Value const&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 2); + + static_assert([](auto&& v) { + return requires + { + v.visitor( + v, + std::integral_constant{}, + [](Json::Value&) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visitor( + s1, + std::integral_constant{}, + Overload{ + [](Json::Value& v) { return v["value"].asInt(); }, + [](Json::Value const&) { return 0; }, + [](auto...) { return 0; }}) == 2); + + static_assert([](auto&& v) { + return requires + { + v.visitor( + v, + std::integral_constant{}, + [](Json::Value const&, + std::integral_constant) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + s1.visitor( + std::as_const(s1), + std::integral_constant{}, + Overload{ + [](Json::Value const& v, + std::integral_constant) { + return v["value"].asInt(); + }, + [](Json::Value&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 3); + + static_assert([](auto&& v) { + return requires + { + v.visitor( + v, + std::integral_constant{}, + [](Json::Value const&) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + s1.visitor( + std::as_const(s1), + std::integral_constant{}, + Overload{ + [](Json::Value const& v) { return v["value"].asInt(); }, + [](Json::Value&) { return 0; }, + [](auto...) { return 0; }}) == 3); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value&, unsigned) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visitor( + s1, // + 3u, + Overload{ + [](Json::Value& v, unsigned) { + return v["value"].asInt(); + }, + [](Json::Value const&, unsigned) { return 0; }, + [](auto, auto) { return 0; }}) == 5); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value&) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visitor( + s1, // + 3, + Overload{ + [](Json::Value& v) { return v["value"].asInt(); }, + [](Json::Value const&) { return 0; }, + [](auto...) { return 0; }}) == 5); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value const&, unsigned) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + s1.visitor( + std::as_const(s1), // + 2u, + Overload{ + [](Json::Value const& v, unsigned) { + return v["value"].asInt(); + }, + [](Json::Value const&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 3); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value const&) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + s1.visitor( + std::as_const(s1), // + 2, + Overload{ + [](Json::Value const& v) { return v["value"].asInt(); }, + [](Json::Value&) { return 0; }, + [](auto...) { return 0; }}) == 3); + + // Test type conversions + BEAST_EXPECT( + s1.visitor( + s1, + std::integral_constant{}, // to unsigned + [](Json::Value& v, unsigned) { + return v["value"].asInt(); + }) == 2); + BEAST_EXPECT( + s1.visitor( + std::as_const(s1), + std::integral_constant{}, // to unsigned + [](Json::Value const& v, unsigned) { + return v["value"].asInt(); + }) == 3); + BEAST_EXPECT( + s1.visitor( + s1, // to const + std::integral_constant{}, + [](Json::Value const& v, auto) { + return v["value"].asInt(); + }) == 5); + BEAST_EXPECT( + s1.visitor( + s1, // to const + std::integral_constant{}, + [](Json::Value const& v) { return v["value"].asInt(); }) == + 5); + BEAST_EXPECT( + s1.visitor( + s1, + 3, // to long + [](Json::Value& v, long) { return v["value"].asInt(); }) == + 5); + BEAST_EXPECT( + s1.visitor( + std::as_const(s1), + 1, // to long + [](Json::Value const& v, long) { + return v["value"].asInt(); + }) == 2); + BEAST_EXPECT( + s1.visitor( + s1, // to const + 2, + [](Json::Value const& v, auto) { + return v["value"].asInt(); + }) == 3); + BEAST_EXPECT( + s1.visitor( + s1, // type deduction + 2, + [](auto& v, auto) { return v["value"].asInt(); }) == 3); + BEAST_EXPECT( + s1.visitor( + s1, // to const, type deduction + 2, + [](auto const& v, auto) { return v["value"].asInt(); }) == + 3); + BEAST_EXPECT( + s1.visitor( + s1, // type deduction + 2, + [](auto& v) { return v["value"].asInt(); }) == 3); + BEAST_EXPECT( + s1.visitor( + s1, // to const, type deduction + 2, + [](auto const& v) { return v["value"].asInt(); }) == 3); + + // Test passing of additional arguments + BEAST_EXPECT( + s1.visitor( + s1, + std::integral_constant{}, + [](Json::Value& v, auto ver, auto a1, auto a2) { + return ver * a1 * a2 * v["value"].asInt(); + }, + 5, + 7) == 2 * 5 * 7 * 3); + BEAST_EXPECT( + s1.visitor( + s1, + std::integral_constant{}, + [](Json::Value& v, auto ver, auto... args) { + return ver * (1 * ... * args) * v["value"].asInt(); + }, + 5, + 7) == 2 * 5 * 7 * 3); + + // Several overloads we want to fail + static_assert([](auto&& v) { + return !requires + { + v.visitor( + v, + 1, // + [](Json::Value&, auto) {}); // missing const + }; + }(std::as_const(s1))); + + static_assert([](auto&& v) { + return !requires + { + v.visitor( + std::move(v), // cannot bind rvalue + 1, + [](Json::Value&, auto) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return !requires + { + v.visitor( + v, + 1, // + []() {}); // missing parameter + }; + }(s1)); + + static_assert([](auto&& v) { + return !requires + { + v.visitor( + v, + 1, // + [](Json::Value&, int, int) {}); // too many parameters + }; + }(s1)); + + // Want these to be unambiguous + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](auto) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value&) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value&, auto...) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value const&) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](Json::Value const&, auto...) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](auto...) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](auto, auto...) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor(v, 1, [](auto, auto, auto...) {}); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor( + v, 1, [](auto, auto, auto...) {}, ""); + }; + }(s1)); + + static_assert([](auto&& v) { + return requires + { + v.visitor( + v, 1, [](auto, auto, auto, auto...) {}, ""); + }; + }(s1)); + } + + { + testcase("visit"); + + MultiApiJson<1, 3> s1{}; + s1.val[0] = makeJson("value", 2); + s1.val[1] = makeJson("value", 3); + s1.val[2] = makeJson("value", 5); + + // Test different overloads + static_assert([](auto&& v) { + return requires + { + v.visit( + std::integral_constant{}, + [](Json::Value&, std::integral_constant) { + }); + }; + }(s1)); + BEAST_EXPECT( + s1.visit( + std::integral_constant{}, + Overload{ + [](Json::Value& v, + std::integral_constant) { + return v["value"].asInt(); + }, + [](Json::Value const&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 2); + static_assert([](auto&& v) { + return requires + { + v.visit()( + std::integral_constant{}, + [](Json::Value&, std::integral_constant) { + }); + }; + }(s1)); + BEAST_EXPECT( + s1.visit()( + std::integral_constant{}, + Overload{ + [](Json::Value& v, + std::integral_constant) { + return v["value"].asInt(); + }, + [](Json::Value const&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 2); + + static_assert([](auto&& v) { + return requires + { + v.visit( + std::integral_constant{}, + [](Json::Value&) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visit( + std::integral_constant{}, + Overload{ + [](Json::Value& v) { return v["value"].asInt(); }, + [](Json::Value const&) { return 0; }, + [](auto...) { return 0; }}) == 2); + static_assert([](auto&& v) { + return requires + { + v.visit()( + std::integral_constant{}, + [](Json::Value&) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visit()( + std::integral_constant{}, + Overload{ + [](Json::Value& v) { return v["value"].asInt(); }, + [](Json::Value const&) { return 0; }, + [](auto...) { return 0; }}) == 2); + + static_assert([](auto&& v) { + return requires + { + v.visit( + std::integral_constant{}, + [](Json::Value const&, + std::integral_constant) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit( + std::integral_constant{}, + Overload{ + [](Json::Value const& v, + std::integral_constant) { + return v["value"].asInt(); + }, + [](Json::Value&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 3); + static_assert([](auto&& v) { + return requires + { + v.visit()( + std::integral_constant{}, + [](Json::Value const&, + std::integral_constant) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit()( + std::integral_constant{}, + Overload{ + [](Json::Value const& v, + std::integral_constant) { + return v["value"].asInt(); + }, + [](Json::Value&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 3); + + static_assert([](auto&& v) { + return requires + { + v.visit( + std::integral_constant{}, + [](Json::Value const&) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit( + std::integral_constant{}, + Overload{ + [](Json::Value const& v) { return v["value"].asInt(); }, + [](Json::Value&) { return 0; }, + [](auto...) { return 0; }}) == 3); + static_assert([](auto&& v) { + return requires + { + v.visit()( + std::integral_constant{}, + [](Json::Value const&) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit()( + std::integral_constant{}, + Overload{ + [](Json::Value const& v) { return v["value"].asInt(); }, + [](Json::Value&) { return 0; }, + [](auto...) { return 0; }}) == 3); + + static_assert([](auto&& v) { + return requires + { + v.visit(1, [](Json::Value&, unsigned) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visit( + 3u, + Overload{ + [](Json::Value& v, unsigned) { + return v["value"].asInt(); + }, + [](Json::Value const&, unsigned) { return 0; }, + [](Json::Value&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 5); + static_assert([](auto&& v) { + return requires + { + v.visit()(1, [](Json::Value&, unsigned) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visit()( + 3u, + Overload{ + [](Json::Value& v, unsigned) { + return v["value"].asInt(); + }, + [](Json::Value const&, unsigned) { return 0; }, + [](Json::Value&, auto) { return 0; }, + [](auto, auto) { return 0; }}) == 5); + + static_assert([](auto&& v) { + return requires + { + v.visit(1, [](Json::Value&) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visit( + 3, + Overload{ + [](Json::Value& v) { return v["value"].asInt(); }, + [](Json::Value const&) { return 0; }, + [](auto...) { return 0; }}) == 5); + static_assert([](auto&& v) { + return requires + { + v.visit()(1, [](Json::Value&) {}); + }; + }(s1)); + BEAST_EXPECT( + s1.visit()( + 3, + Overload{ + [](Json::Value& v) { return v["value"].asInt(); }, + [](Json::Value const&) { return 0; }, + [](auto...) { return 0; }}) == 5); + + static_assert([](auto&& v) { + return requires + { + v.visit(1, [](Json::Value const&, unsigned) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit( + 2u, + Overload{ + [](Json::Value const& v, unsigned) { + return v["value"].asInt(); + }, + [](Json::Value const&, auto) { return 0; }, + [](Json::Value&, unsigned) { return 0; }, + [](auto, auto) { return 0; }}) == 3); + static_assert([](auto&& v) { + return requires + { + v.visit()(1, [](Json::Value const&, unsigned) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit()( + 2u, + Overload{ + [](Json::Value const& v, unsigned) { + return v["value"].asInt(); + }, + [](Json::Value const&, auto) { return 0; }, + [](Json::Value&, unsigned) { return 0; }, + [](auto, auto) { return 0; }}) == 3); + + static_assert([](auto&& v) { + return requires + { + v.visit(1, [](Json::Value const&) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit( + 2, + Overload{ + [](Json::Value const& v) { return v["value"].asInt(); }, + [](Json::Value&) { return 0; }, + [](auto...) { return 0; }}) == 3); + static_assert([](auto&& v) { + return requires + { + v.visit()(1, [](Json::Value const&) {}); + }; + }(std::as_const(s1))); + BEAST_EXPECT( + std::as_const(s1).visit()( + 2, + Overload{ + [](Json::Value const& v) { return v["value"].asInt(); }, + [](Json::Value&) { return 0; }, + [](auto...) { return 0; }}) == 3); + + // Rvalue MultivarJson visitor only binds to regular reference + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit(1, [](Json::Value&&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit( + 1, [](Json::Value const&&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit(1, [](Json::Value&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit( + 1, [](Json::Value const&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit()( + 1, [](Json::Value&&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit()( + 1, [](Json::Value const&&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit()( + 1, [](Json::Value&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit()( + 1, [](Json::Value const&) {}); + }; + }(std::move(s1))); + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit( + 1, [](Json::Value const&&) {}); + }; + }(std::move(std::as_const(s1)))); + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit( + 1, [](Json::Value const&) {}); + }; + }(std::move(std::as_const(s1)))); + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit()( + 1, [](Json::Value const&&) {}); + }; + }(std::move(std::as_const(s1)))); + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit()( + 1, [](Json::Value const&) {}); + }; + }(std::move(std::as_const(s1)))); + + // Missing const + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit( + 1, [](Json::Value&, auto) {}); + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit()( + 1, [](Json::Value&, auto) {}); + }; + }(std::as_const(s1))); + + // Missing parameter + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit(1, []() {}); + }; + }(s1)); + static_assert([](auto&& v) { + return !requires + { + std::forward(v).visit()(1, []() {}); + }; + }(s1)); + + // Sanity checks + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit(1, [](auto...) {}); + }; + }(std::as_const(s1))); + static_assert([](auto&& v) { + return requires + { + std::forward(v).visit()(1, [](auto...) {}); + }; + }(std::as_const(s1))); + } + } +}; + +BEAST_DEFINE_TESTSUITE(MultiApiJson, protocol, ripple); + +} // namespace test +} // namespace ripple diff --git a/src/test/protocol/PublicKey_test.cpp b/src/test/protocol/PublicKey_test.cpp index c7974021877..040d752f481 100644 --- a/src/test/protocol/PublicKey_test.cpp +++ b/src/test/protocol/PublicKey_test.cpp @@ -363,10 +363,12 @@ class PublicKey_test : public beast::unit_test::suite } // Try some random secret keys - std::array keys; + std::vector keys; + keys.reserve(32); - for (std::size_t i = 0; i != keys.size(); ++i) - keys[i] = derivePublicKey(keyType, randomSecretKey()); + for (std::size_t i = 0; i != keys.capacity(); ++i) + keys.emplace_back(derivePublicKey(keyType, randomSecretKey())); + BEAST_EXPECT(keys.size() == 32); for (std::size_t i = 0; i != keys.size(); ++i) { @@ -447,7 +449,11 @@ class PublicKey_test : public beast::unit_test::suite BEAST_EXPECT(pk1 == pk2); BEAST_EXPECT(pk2 == pk1); - PublicKey pk3; + PublicKey pk3 = derivePublicKey( + KeyType::secp256k1, + generateSecretKey( + KeyType::secp256k1, generateSeed("arbitraryPassPhrase"))); + // Testing the copy assignment operation of PublicKey class pk3 = pk2; BEAST_EXPECT(pk3 == pk2); BEAST_EXPECT(pk1 == pk3); diff --git a/src/test/protocol/STObject_test.cpp b/src/test/protocol/STObject_test.cpp index d89916eddf7..34d1cc82fac 100644 --- a/src/test/protocol/STObject_test.cpp +++ b/src/test/protocol/STObject_test.cpp @@ -17,7 +17,6 @@ */ //============================================================================== -#include #include #include #include @@ -609,12 +608,7 @@ class STObject_test : public beast::unit_test::suite auto const kp = generateKeyPair(KeyType::secp256k1, generateSeed("masterpassphrase")); st[sf5] = kp.first; - BEAST_EXPECT(st[sf5] != PublicKey{}); st[~sf5] = std::nullopt; -#if 0 - pk = st[sf5]; - BEAST_EXPECT(pk.size() == 0); -#endif } // By reference fields diff --git a/src/test/protocol/STTx_test.cpp b/src/test/protocol/STTx_test.cpp index 4ef30fb7a74..f41c283e3ab 100644 --- a/src/test/protocol/STTx_test.cpp +++ b/src/test/protocol/STTx_test.cpp @@ -1591,7 +1591,11 @@ class STTx_test : public beast::unit_test::suite }); j.sign(keypair.first, keypair.second); - Rules defaultRules{{}}; + // Rules store a reference to the presets. Create a local to guarantee + // proper lifetime. + std::unordered_set> const presets; + Rules const defaultRules{presets}; + BEAST_EXPECT(!defaultRules.enabled(featureExpandedSignerList)); unexpected( !j.checkSign(STTx::RequireFullyCanonicalSig::yes, defaultRules), diff --git a/src/test/protocol/SecretKey_test.cpp b/src/test/protocol/SecretKey_test.cpp index 25b5511dd74..08a19124508 100644 --- a/src/test/protocol/SecretKey_test.cpp +++ b/src/test/protocol/SecretKey_test.cpp @@ -275,10 +275,12 @@ class SecretKey_test : public beast::unit_test::suite } // Try some random secret keys - std::array keys; + std::vector keys; + keys.reserve(32); - for (std::size_t i = 0; i != keys.size(); ++i) - keys[i] = randomSecretKey(); + for (std::size_t i = 0; i != keys.capacity(); ++i) + keys.emplace_back(randomSecretKey()); + BEAST_EXPECT(keys.size() == 32); for (std::size_t i = 0; i != keys.size(); ++i) { diff --git a/src/test/rpc/AccountTx_test.cpp b/src/test/rpc/AccountTx_test.cpp index 640b774f525..6601996925e 100644 --- a/src/test/rpc/AccountTx_test.cpp +++ b/src/test/rpc/AccountTx_test.cpp @@ -734,7 +734,7 @@ class AccountTx_test : public beast::unit_test::suite void run() override { - test::jtx::forAllApiVersions( + forAllApiVersions( std::bind_front(&AccountTx_test::testParameters, this)); testContents(); testAccountDelete(); diff --git a/src/test/rpc/Feature_test.cpp b/src/test/rpc/Feature_test.cpp index dcd95c8a968..fd63aee98e2 100644 --- a/src/test/rpc/Feature_test.cpp +++ b/src/test/rpc/Feature_test.cpp @@ -205,11 +205,94 @@ class Feature_test : public beast::unit_test::suite return cfg; })}; - auto jrr = env.rpc("feature")[jss::result]; - // The current HTTP/S ServerHandler returns an HTTP 403 error code here - // rather than a noPermission JSON error. The JSONRPCClient just eats - // that error and returns an null result. - BEAST_EXPECT(jrr.isNull()); + { + auto result = env.rpc("feature")[jss::result]; + BEAST_EXPECT(result.isMember(jss::features)); + // There should be at least 50 amendments. Don't do exact + // comparison to avoid maintenance as more amendments are added in + // the future. + BEAST_EXPECT(result[jss::features].size() >= 50); + for (auto it = result[jss::features].begin(); + it != result[jss::features].end(); + ++it) + { + uint256 id; + (void)id.parseHex(it.key().asString().c_str()); + if (!BEAST_EXPECT((*it).isMember(jss::name))) + return; + bool expectEnabled = + env.app().getAmendmentTable().isEnabled(id); + bool expectSupported = + env.app().getAmendmentTable().isSupported(id); + BEAST_EXPECTS( + (*it).isMember(jss::enabled) && + (*it)[jss::enabled].asBool() == expectEnabled, + (*it)[jss::name].asString() + " enabled"); + BEAST_EXPECTS( + (*it).isMember(jss::supported) && + (*it)[jss::supported].asBool() == expectSupported, + (*it)[jss::name].asString() + " supported"); + BEAST_EXPECT(!(*it).isMember(jss::vetoed)); + BEAST_EXPECT(!(*it).isMember(jss::majority)); + BEAST_EXPECT(!(*it).isMember(jss::count)); + BEAST_EXPECT(!(*it).isMember(jss::validations)); + BEAST_EXPECT(!(*it).isMember(jss::threshold)); + } + } + + { + Json::Value params; + // invalid feature + params[jss::feature] = + "1234567890ABCDEF1234567890ABCDEF1234567890ABCDEF1234567890ABCD" + "EF"; + auto const result = env.rpc( + "json", + "feature", + boost::lexical_cast(params))[jss::result]; + BEAST_EXPECTS( + result[jss::error] == "badFeature", result.toStyledString()); + BEAST_EXPECT( + result[jss::error_message] == "Feature unknown or invalid."); + } + + { + Json::Value params; + params[jss::feature] = + "93E516234E35E08CA689FA33A6D38E103881F8DCB53023F728C307AA89D515" + "A7"; + // invalid param + params[jss::vetoed] = true; + auto const result = env.rpc( + "json", + "feature", + boost::lexical_cast(params))[jss::result]; + BEAST_EXPECTS( + result[jss::error] == "noPermission", + result[jss::error].asString()); + BEAST_EXPECT( + result[jss::error_message] == + "You don't have permission for this command."); + } + + { + std::string const feature = + "C4483A1896170C66C098DEA5B0E024309C60DC960DE5F01CD7AF986AA3D9AD" + "37"; + Json::Value params; + params[jss::feature] = feature; + auto const result = env.rpc( + "json", + "feature", + boost::lexical_cast(params))[jss::result]; + BEAST_EXPECT(result.isMember(feature)); + auto const amendmentResult = result[feature]; + BEAST_EXPECT(amendmentResult[jss::enabled].asBool() == false); + BEAST_EXPECT(amendmentResult[jss::supported].asBool() == true); + BEAST_EXPECT( + amendmentResult[jss::name].asString() == + "fixMasterKeyAsRegularKey"); + } } void diff --git a/src/test/rpc/GatewayBalances_test.cpp b/src/test/rpc/GatewayBalances_test.cpp index 091b9f51686..4c6ec9aaf1b 100644 --- a/src/test/rpc/GatewayBalances_test.cpp +++ b/src/test/rpc/GatewayBalances_test.cpp @@ -172,10 +172,7 @@ class GatewayBalances_test : public beast::unit_test::suite qry2[jss::account] = alice.human(); qry2[jss::hotwallet] = "asdf"; - for (auto apiVersion = RPC::apiMinimumSupportedVersion; - apiVersion <= RPC::apiBetaVersion; - ++apiVersion) - { + forAllApiVersions([&, this](unsigned apiVersion) { qry2[jss::api_version] = apiVersion; auto jv = wsc->invoke("gateway_balances", qry2); expect(jv[jss::status] == "error"); @@ -184,7 +181,7 @@ class GatewayBalances_test : public beast::unit_test::suite auto const error = apiVersion < 2u ? "invalidHotWallet" : "invalidParams"; BEAST_EXPECT(response[jss::error] == error); - } + }); } void diff --git a/src/test/rpc/GetAggregatePrice_test.cpp b/src/test/rpc/GetAggregatePrice_test.cpp new file mode 100644 index 00000000000..4c45b7b9d2b --- /dev/null +++ b/src/test/rpc/GetAggregatePrice_test.cpp @@ -0,0 +1,260 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2023 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include + +namespace ripple { +namespace test { +namespace jtx { +namespace oracle { + +class GetAggregatePrice_test : public beast::unit_test::suite +{ +public: + void + testErrors() + { + testcase("Errors"); + using namespace jtx; + using Oracles = std::vector>; + Account const owner{"owner"}; + Account const some{"some"}; + static Oracles oracles = {{owner, 1}}; + + { + Env env(*this); + // missing base_asset + auto ret = + Oracle::aggregatePrice(env, std::nullopt, "USD", oracles); + BEAST_EXPECT( + ret[jss::error_message].asString() == + "Missing field 'base_asset'."); + + // missing quote_asset + ret = Oracle::aggregatePrice(env, "XRP", std::nullopt, oracles); + BEAST_EXPECT( + ret[jss::error_message].asString() == + "Missing field 'quote_asset'."); + + // missing oracles array + ret = Oracle::aggregatePrice(env, "XRP", "USD"); + BEAST_EXPECT( + ret[jss::error_message].asString() == + "Missing field 'oracles'."); + + // empty oracles array + ret = Oracle::aggregatePrice(env, "XRP", "USD", Oracles{}); + BEAST_EXPECT(ret[jss::error].asString() == "oracleMalformed"); + + // invalid oracle document id + ret = Oracle::aggregatePrice(env, "XRP", "USD", {{{owner, 2}}}); + BEAST_EXPECT(ret[jss::error].asString() == "objectNotFound"); + + // invalid owner + ret = Oracle::aggregatePrice(env, "XRP", "USD", {{{some, 1}}}); + BEAST_EXPECT(ret[jss::error].asString() == "objectNotFound"); + + // oracles have wrong asset pair + env.fund(XRP(1'000), owner); + Oracle oracle( + env, {.owner = owner, .series = {{"XRP", "EUR", 740, 1}}}); + ret = Oracle::aggregatePrice( + env, "XRP", "USD", {{{owner, oracle.documentID()}}}); + BEAST_EXPECT(ret[jss::error].asString() == "objectNotFound"); + + // invalid trim value + ret = Oracle::aggregatePrice( + env, "XRP", "USD", {{{owner, oracle.documentID()}}}, 0); + BEAST_EXPECT(ret[jss::error].asString() == "invalidParams"); + ret = Oracle::aggregatePrice( + env, "XRP", "USD", {{{owner, oracle.documentID()}}}, 26); + BEAST_EXPECT(ret[jss::error].asString() == "invalidParams"); + } + + // too many oracles + { + Env env(*this); + std::vector> oracles; + for (int i = 0; i < 201; ++i) + { + Account const owner(std::to_string(i)); + env.fund(XRP(1'000), owner); + Oracle oracle(env, {.owner = owner, .documentID = i}); + oracles.emplace_back(owner, oracle.documentID()); + } + auto const ret = Oracle::aggregatePrice(env, "XRP", "USD", oracles); + BEAST_EXPECT(ret[jss::error].asString() == "oracleMalformed"); + } + } + + void + testRpc() + { + testcase("RPC"); + using namespace jtx; + + auto prep = [&](Env& env, auto& oracles) { + oracles.reserve(10); + for (int i = 0; i < 10; ++i) + { + Account const owner{std::to_string(i)}; + env.fund(XRP(1'000), owner); + Oracle oracle( + env, + {.owner = owner, + .documentID = rand(), + .series = { + {"XRP", "USD", 740 + i, 1}, {"XRP", "EUR", 740, 1}}}); + oracles.emplace_back(owner, oracle.documentID()); + } + }; + + // Aggregate data set includes all price oracle instances, no trimming + // or time threshold + { + Env env(*this); + std::vector> oracles; + prep(env, oracles); + // entire and trimmed stats + auto ret = Oracle::aggregatePrice(env, "XRP", "USD", oracles); + BEAST_EXPECT(ret[jss::entire_set][jss::mean] == "74.45"); + BEAST_EXPECT(ret[jss::entire_set][jss::size].asUInt() == 10); + BEAST_EXPECT( + ret[jss::entire_set][jss::standard_deviation] == + "0.3027650354097492"); + BEAST_EXPECT(ret[jss::median] == "74.45"); + BEAST_EXPECT(ret[jss::time] == 946694900); + } + + // Aggregate data set includes all price oracle instances + { + Env env(*this); + std::vector> oracles; + prep(env, oracles); + // entire and trimmed stats + auto ret = + Oracle::aggregatePrice(env, "XRP", "USD", oracles, 20, 100); + BEAST_EXPECT(ret[jss::entire_set][jss::mean] == "74.45"); + BEAST_EXPECT(ret[jss::entire_set][jss::size].asUInt() == 10); + BEAST_EXPECT( + ret[jss::entire_set][jss::standard_deviation] == + "0.3027650354097492"); + BEAST_EXPECT(ret[jss::median] == "74.45"); + BEAST_EXPECT(ret[jss::trimmed_set][jss::mean] == "74.45"); + BEAST_EXPECT(ret[jss::trimmed_set][jss::size].asUInt() == 6); + BEAST_EXPECT( + ret[jss::trimmed_set][jss::standard_deviation] == + "0.187082869338697"); + BEAST_EXPECT(ret[jss::time] == 946694900); + } + + // A reduced dataset, as some price oracles have data beyond three + // updated ledgers + { + Env env(*this); + std::vector> oracles; + prep(env, oracles); + for (int i = 0; i < 3; ++i) + { + Oracle oracle( + env, + {.owner = oracles[i].first, + .documentID = oracles[i].second}, + false); + // push XRP/USD by more than three ledgers, so this price + // oracle is not included in the dataset + oracle.set(UpdateArg{.series = {{"XRP", "EUR", 740, 1}}}); + oracle.set(UpdateArg{.series = {{"XRP", "EUR", 740, 1}}}); + oracle.set(UpdateArg{.series = {{"XRP", "EUR", 740, 1}}}); + } + for (int i = 3; i < 6; ++i) + { + Oracle oracle( + env, + {.owner = oracles[i].first, + .documentID = oracles[i].second}, + false); + // push XRP/USD by two ledgers, so this price + // is included in the dataset + oracle.set(UpdateArg{.series = {{"XRP", "EUR", 740, 1}}}); + oracle.set(UpdateArg{.series = {{"XRP", "EUR", 740, 1}}}); + } + + // entire and trimmed stats + auto ret = + Oracle::aggregatePrice(env, "XRP", "USD", oracles, 20, 200); + BEAST_EXPECT(ret[jss::entire_set][jss::mean] == "74.6"); + BEAST_EXPECT(ret[jss::entire_set][jss::size].asUInt() == 7); + BEAST_EXPECT( + ret[jss::entire_set][jss::standard_deviation] == + "0.2160246899469287"); + BEAST_EXPECT(ret[jss::median] == "74.6"); + BEAST_EXPECT(ret[jss::trimmed_set][jss::mean] == "74.6"); + BEAST_EXPECT(ret[jss::trimmed_set][jss::size].asUInt() == 5); + BEAST_EXPECT( + ret[jss::trimmed_set][jss::standard_deviation] == + "0.158113883008419"); + BEAST_EXPECT(ret[jss::time] == 946694900); + } + + // Reduced data set because of the time threshold + { + Env env(*this); + std::vector> oracles; + prep(env, oracles); + for (int i = 0; i < oracles.size(); ++i) + { + Oracle oracle( + env, + {.owner = oracles[i].first, + .documentID = oracles[i].second}, + false); + // push XRP/USD by two ledgers, so this price + // is included in the dataset + oracle.set(UpdateArg{.series = {{"XRP", "USD", 740, 1}}}); + } + + // entire stats only, limit lastUpdateTime to {200, 125} + auto ret = Oracle::aggregatePrice( + env, "XRP", "USD", oracles, std::nullopt, 75); + BEAST_EXPECT(ret[jss::entire_set][jss::mean] == "74"); + BEAST_EXPECT(ret[jss::entire_set][jss::size].asUInt() == 8); + BEAST_EXPECT(ret[jss::entire_set][jss::standard_deviation] == "0"); + BEAST_EXPECT(ret[jss::median] == "74"); + BEAST_EXPECT(ret[jss::time] == 946695000); + } + } + + void + run() override + { + testErrors(); + testRpc(); + } +}; + +BEAST_DEFINE_TESTSUITE(GetAggregatePrice, app, ripple); + +} // namespace oracle +} // namespace jtx +} // namespace test +} // namespace ripple diff --git a/src/test/rpc/JSONRPC_test.cpp b/src/test/rpc/JSONRPC_test.cpp index 1e8ce554be3..8fd453ee9f6 100644 --- a/src/test/rpc/JSONRPC_test.cpp +++ b/src/test/rpc/JSONRPC_test.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -70,14 +69,14 @@ struct TxnTestData static constexpr TxnTestData txnTestArray[] = { - {"Minimal payment.", + {"Minimal payment, no Amount only DeliverMax", __LINE__, R"({ "command": "doesnt_matter", "secret": "masterpassphrase", "tx_json": { "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", - "Amount": "1000000000", + "DeliverMax": "1000000000", "Destination": "rnUy2SHTrB9DubsPmkJZUXTf5FcNDGrYEA", "TransactionType": "Payment" } @@ -87,7 +86,7 @@ static constexpr TxnTestData txnTestArray[] = { "Missing field 'account'.", "Missing field 'tx_json.Sequence'."}}}, - {"Pass in Fee with minimal payment.", + {"Pass in Fee with minimal payment, both Amount and DeliverMax.", __LINE__, R"({ "command": "doesnt_matter", @@ -97,6 +96,7 @@ static constexpr TxnTestData txnTestArray[] = { "Fee": 10, "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", "Amount": "1000000000", + "DeliverMax": "1000000000", "Destination": "rnUy2SHTrB9DubsPmkJZUXTf5FcNDGrYEA", "TransactionType": "Payment" } @@ -106,7 +106,7 @@ static constexpr TxnTestData txnTestArray[] = { "Missing field 'tx_json.Sequence'.", "Missing field 'tx_json.Sequence'."}}}, - {"Pass in Sequence.", + {"Pass in Sequence, no Amount only DeliverMax", __LINE__, R"({ "command": "doesnt_matter", @@ -115,7 +115,7 @@ static constexpr TxnTestData txnTestArray[] = { "tx_json": { "Sequence": 0, "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", - "Amount": "1000000000", + "DeliverMax": "1000000000", "Destination": "rnUy2SHTrB9DubsPmkJZUXTf5FcNDGrYEA", "TransactionType": "Payment" } @@ -125,7 +125,8 @@ static constexpr TxnTestData txnTestArray[] = { "Missing field 'tx_json.Fee'.", "Missing field 'tx_json.SigningPubKey'."}}}, - {"Pass in Sequence and Fee with minimal payment.", + {"Pass in Sequence and Fee with minimal payment, both Amount and " + "DeliverMax.", __LINE__, R"({ "command": "doesnt_matter", @@ -136,6 +137,7 @@ static constexpr TxnTestData txnTestArray[] = { "Fee": 10, "Account": "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", "Amount": "1000000000", + "DeliverMax": "1000000000", "Destination": "rnUy2SHTrB9DubsPmkJZUXTf5FcNDGrYEA", "TransactionType": "Payment" } @@ -2499,7 +2501,7 @@ class JSONRPC_test : public beast::unit_test::suite fakeProcessTransaction( std::shared_ptr&, bool, - SubmitSync, + bool, NetworkOPs::FailHard) { ; @@ -2549,8 +2551,7 @@ class JSONRPC_test : public beast::unit_test::suite Role role, std::chrono::seconds validatedLedgerAge, Application& app, - ProcessTransactionFn const& processTransaction, - RPC::SubmitSync sync); + ProcessTransactionFn const& processTransaction); using TestStuff = std::tuple; @@ -2605,8 +2606,7 @@ class JSONRPC_test : public beast::unit_test::suite testRole, 1s, env.app(), - processTxn, - RPC::SubmitSync::sync); + processTxn); } std::string errStr; diff --git a/src/test/rpc/KeyGeneration_test.cpp b/src/test/rpc/KeyGeneration_test.cpp index 735bae40bd8..28f9afd3b7f 100644 --- a/src/test/rpc/KeyGeneration_test.cpp +++ b/src/test/rpc/KeyGeneration_test.cpp @@ -16,8 +16,6 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== - -#include #include #include #include @@ -337,8 +335,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(ret.first == publicKey); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(ret->first == publicKey); + } } { @@ -348,8 +349,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(ret.first == publicKey); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(ret->first == publicKey); + } } { @@ -359,8 +363,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(ret.first == publicKey); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(ret->first == publicKey); + } } keyType.emplace("secp256k1"); @@ -375,8 +382,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(ret.first == publicKey); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(ret->first == publicKey); + } } { @@ -388,8 +398,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(ret.first == publicKey); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(ret->first == publicKey); + } } { @@ -401,8 +414,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(ret.first == publicKey); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(ret->first == publicKey); + } } } @@ -416,10 +432,10 @@ class WalletPropose_test : public ripple::TestSuite params[jss::secret] = 314159265; auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'secret', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { @@ -430,10 +446,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'secret', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { @@ -445,7 +461,7 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); - BEAST_EXPECT(ret.first.size() == 0); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'secret', not string."); @@ -460,10 +476,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "The secret field is not allowed if key_type is used."); - BEAST_EXPECT(ret.first.size() == 0); } // Specify unknown or bad "key_type" @@ -475,9 +491,9 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'key_type'."); - BEAST_EXPECT(ret.first.size() == 0); } { @@ -488,10 +504,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'key_type', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { @@ -502,10 +518,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'key_type', not string."); - BEAST_EXPECT(ret.first.size() == 0); } // Specify non-string passphrase @@ -517,10 +533,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'passphrase', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a passphrase: object @@ -531,10 +547,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'passphrase', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a passphrase: array @@ -545,10 +561,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'passphrase', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a passphrase: empty string @@ -559,8 +575,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } // Specify non-string or invalid seed @@ -572,10 +588,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'seed', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a string: object @@ -586,10 +602,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'seed', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a string: array @@ -600,10 +616,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'seed', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a seed: empty @@ -614,8 +630,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a seed: invalid characters @@ -626,8 +642,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a seed: random string @@ -638,8 +654,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } // Specify non-string or invalid seed_hex @@ -651,10 +667,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'seed_hex', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a string: object @@ -665,10 +681,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'seed_hex', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // not a string: array @@ -679,10 +695,10 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT( error[jss::error_message] == "Invalid field 'seed_hex', not string."); - BEAST_EXPECT(ret.first.size() == 0); } { // empty @@ -693,8 +709,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } { // short @@ -705,8 +721,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } { // not hex @@ -717,8 +733,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } { // overlong @@ -730,8 +746,8 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(contains_error(error)); + BEAST_EXPECT(!ret); BEAST_EXPECT(error[jss::error_message] == "Disallowed seed."); - BEAST_EXPECT(ret.first.size() == 0); } } @@ -750,8 +766,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(toBase58(calcAccountID(ret.first)) == addr); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(toBase58(calcAccountID(ret->first)) == addr); + } } { @@ -779,8 +798,11 @@ class WalletPropose_test : public ripple::TestSuite auto ret = keypairForSignature(params, error); BEAST_EXPECT(!contains_error(error)); - BEAST_EXPECT(ret.first.size() != 0); - BEAST_EXPECT(toBase58(calcAccountID(ret.first)) == addr); + if (BEAST_EXPECT(ret)) + { + BEAST_EXPECT(ret->first.size() != 0); + BEAST_EXPECT(toBase58(calcAccountID(ret->first)) == addr); + } } { diff --git a/src/test/rpc/LedgerRPC_test.cpp b/src/test/rpc/LedgerRPC_test.cpp index 2b4d8527a64..877b8ef2a02 100644 --- a/src/test/rpc/LedgerRPC_test.cpp +++ b/src/test/rpc/LedgerRPC_test.cpp @@ -2305,7 +2305,7 @@ class LedgerRPC_test : public beast::unit_test::suite testLedgerAccountsOption(); testLedgerEntryDID(); - test::jtx::forAllApiVersions(std::bind_front( + forAllApiVersions(std::bind_front( &LedgerRPC_test::testLedgerEntryInvalidParams, this)); } }; diff --git a/src/test/rpc/LedgerRequestRPC_test.cpp b/src/test/rpc/LedgerRequestRPC_test.cpp index a1f3daafb30..6c59e72c4b8 100644 --- a/src/test/rpc/LedgerRequestRPC_test.cpp +++ b/src/test/rpc/LedgerRequestRPC_test.cpp @@ -359,7 +359,7 @@ class LedgerRequestRPC_test : public beast::unit_test::suite { testLedgerRequest(); testEvolution(); - test::jtx::forAllApiVersions( + forAllApiVersions( std::bind_front(&LedgerRequestRPC_test::testBadInput, this)); testMoreThan256Closed(); testNonAdmin(); diff --git a/src/test/rpc/NoRipple_test.cpp b/src/test/rpc/NoRipple_test.cpp index 3077b06f8a3..8da80e6483c 100644 --- a/src/test/rpc/NoRipple_test.cpp +++ b/src/test/rpc/NoRipple_test.cpp @@ -284,12 +284,9 @@ class NoRipple_test : public beast::unit_test::suite testSetAndClear(); auto withFeatsTests = [this](FeatureBitset features) { - for (auto testVersion = RPC::apiMinimumSupportedVersion; - testVersion <= RPC::apiBetaVersion; - ++testVersion) - { + forAllApiVersions([&, this](unsigned testVersion) { testDefaultRipple(features, testVersion); - } + }); testNegativeBalance(features); testPairwise(features); }; diff --git a/src/test/rpc/RPCCall_test.cpp b/src/test/rpc/RPCCall_test.cpp index 5f66250b103..09096ba76af 100644 --- a/src/test/rpc/RPCCall_test.cpp +++ b/src/test/rpc/RPCCall_test.cpp @@ -6219,8 +6219,7 @@ class RPCCall_test : public beast::unit_test::suite void run() override { - test::jtx::forAllApiVersions( - std::bind_front(&RPCCall_test::testRPCCall, this)); + forAllApiVersions(std::bind_front(&RPCCall_test::testRPCCall, this)); } }; diff --git a/src/test/rpc/ReportingETL_test.cpp b/src/test/rpc/ReportingETL_test.cpp index ed055d0fd93..078a51d7bcd 100644 --- a/src/test/rpc/ReportingETL_test.cpp +++ b/src/test/rpc/ReportingETL_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -56,7 +57,8 @@ class ReportingETL_test : public beast::unit_test::suite testcase("GetLedger"); using namespace test::jtx; std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); + std::string grpcPort = + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); env.close(); @@ -498,7 +500,8 @@ class ReportingETL_test : public beast::unit_test::suite testcase("GetLedgerData"); using namespace test::jtx; std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); + std::string grpcPort = + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); auto grpcLedgerData = [&grpcPort]( auto sequence, std::string marker = "") { @@ -620,7 +623,8 @@ class ReportingETL_test : public beast::unit_test::suite testcase("GetLedgerDiff"); using namespace test::jtx; std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); + std::string grpcPort = + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); auto grpcLedgerDiff = [&grpcPort]( @@ -735,7 +739,8 @@ class ReportingETL_test : public beast::unit_test::suite testcase("GetLedgerDiff"); using namespace test::jtx; std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); + std::string grpcPort = + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); auto grpcLedgerEntry = [&grpcPort](auto sequence, auto key) { @@ -895,7 +900,7 @@ class ReportingETL_test : public beast::unit_test::suite std::unique_ptr config = envconfig( addGrpcConfigWithSecureGateway, getEnvLocalhostAddr()); std::string grpcPort = - *(*config)["port_grpc"].get("port"); + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); env.close(); @@ -955,7 +960,7 @@ class ReportingETL_test : public beast::unit_test::suite std::unique_ptr config = envconfig(addGrpcConfigWithSecureGateway, secureGatewayIp); std::string grpcPort = - *(*config)["port_grpc"].get("port"); + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); env.close(); @@ -1008,7 +1013,7 @@ class ReportingETL_test : public beast::unit_test::suite std::unique_ptr config = envconfig( addGrpcConfigWithSecureGateway, getEnvLocalhostAddr()); std::string grpcPort = - *(*config)["port_grpc"].get("port"); + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); env.close(); @@ -1065,7 +1070,7 @@ class ReportingETL_test : public beast::unit_test::suite std::unique_ptr config = envconfig(addGrpcConfigWithSecureGateway, secureGatewayIp); std::string grpcPort = - *(*config)["port_grpc"].get("port"); + *(*config)[SECTION_PORT_GRPC].get("port"); Env env(*this, std::move(config)); env.close(); diff --git a/src/test/rpc/RobustTransaction_test.cpp b/src/test/rpc/RobustTransaction_test.cpp index 01ac71e272a..37b16c58d7f 100644 --- a/src/test/rpc/RobustTransaction_test.cpp +++ b/src/test/rpc/RobustTransaction_test.cpp @@ -17,7 +17,6 @@ */ //============================================================================== -#include #include #include #include @@ -89,8 +88,7 @@ class RobustTransaction_test : public beast::unit_test::suite } BEAST_EXPECT(jv[jss::result][jss::engine_result] == "tefPAST_SEQ"); - // Submit future sequence transaction -- this transaction should be - // held until the sequence gap is closed. + // Submit future sequence transaction payment[jss::tx_json][sfSequence.fieldName] = env.seq("alice") + 1; jv = wsc->invoke("submit", payment); if (wsc->version() == 2) @@ -116,8 +114,6 @@ class RobustTransaction_test : public beast::unit_test::suite } BEAST_EXPECT(jv[jss::result][jss::engine_result] == "tesSUCCESS"); - // Apply held transactions. - env.app().getOPs().transactionBatch(true); // Wait for the jobqueue to process everything env.app().getJobQueue().rendezvous(); diff --git a/src/test/rpc/ServerInfo_test.cpp b/src/test/rpc/ServerInfo_test.cpp index 1d78f1cc36b..ece98e99a7e 100644 --- a/src/test/rpc/ServerInfo_test.cpp +++ b/src/test/rpc/ServerInfo_test.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -107,7 +108,7 @@ admin = 127.0.0.1 auto const rpc_port = (*config)["port_rpc"].get("port"); auto const grpc_port = - (*config)["port_grpc"].get("port"); + (*config)[SECTION_PORT_GRPC].get("port"); auto const ws_port = (*config)["port_ws"].get("port"); BEAST_EXPECT(grpc_port); BEAST_EXPECT(rpc_port); diff --git a/src/test/rpc/TransactionEntry_test.cpp b/src/test/rpc/TransactionEntry_test.cpp index 90c643700c4..5eddb640cbf 100644 --- a/src/test/rpc/TransactionEntry_test.cpp +++ b/src/test/rpc/TransactionEntry_test.cpp @@ -388,7 +388,7 @@ class TransactionEntry_test : public beast::unit_test::suite run() override { testBadInput(); - test::jtx::forAllApiVersions( + forAllApiVersions( std::bind_front(&TransactionEntry_test::testRequest, this)); } }; diff --git a/src/test/rpc/Transaction_test.cpp b/src/test/rpc/Transaction_test.cpp index 9fbda07c789..ac02dd11cda 100644 --- a/src/test/rpc/Transaction_test.cpp +++ b/src/test/rpc/Transaction_test.cpp @@ -849,7 +849,7 @@ class Transaction_test : public beast::unit_test::suite run() override { using namespace test::jtx; - test::jtx::forAllApiVersions( + forAllApiVersions( std::bind_front(&Transaction_test::testBinaryRequest, this)); FeatureBitset const all{supported_amendments()}; @@ -863,7 +863,7 @@ class Transaction_test : public beast::unit_test::suite testRangeCTIDRequest(features); testCTIDValidation(features); testCTIDRPC(features); - test::jtx::forAllApiVersions( + forAllApiVersions( std::bind_front(&Transaction_test::testRequest, this, features)); } }; diff --git a/src/test/rpc/Version_test.cpp b/src/test/rpc/Version_test.cpp index 60ffd30fcf6..34e55b2be93 100644 --- a/src/test/rpc/Version_test.cpp +++ b/src/test/rpc/Version_test.cpp @@ -83,7 +83,8 @@ class Version_test : public beast::unit_test::suite "{\"api_version\": " + std::to_string( std::max( - RPC::apiMaximumSupportedVersion, RPC::apiBetaVersion) + + RPC::apiMaximumSupportedVersion.value, + RPC::apiBetaVersion.value) + 1) + "}"); BEAST_EXPECT(badVersion(re)); @@ -112,15 +113,15 @@ class Version_test : public beast::unit_test::suite Json::Value j_object = Json::Value(Json::objectValue); BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, false) == versionIfUnspecified); - j_object[jss::api_version] = RPC::apiVersionIfUnspecified; + j_object[jss::api_version] = RPC::apiVersionIfUnspecified.value; BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, false) == versionIfUnspecified); - j_object[jss::api_version] = RPC::apiMinimumSupportedVersion; + j_object[jss::api_version] = RPC::apiMinimumSupportedVersion.value; BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, false) == RPC::apiMinimumSupportedVersion); - j_object[jss::api_version] = RPC::apiMaximumSupportedVersion; + j_object[jss::api_version] = RPC::apiMaximumSupportedVersion.value; BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, false) == RPC::apiMaximumSupportedVersion); @@ -133,14 +134,14 @@ class Version_test : public beast::unit_test::suite BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, false) == RPC::apiInvalidVersion); - j_object[jss::api_version] = RPC::apiBetaVersion; + j_object[jss::api_version] = RPC::apiBetaVersion.value; BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, true) == RPC::apiBetaVersion); j_object[jss::api_version] = RPC::apiBetaVersion + 1; BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, true) == RPC::apiInvalidVersion); - j_object[jss::api_version] = RPC::apiInvalidVersion; + j_object[jss::api_version] = RPC::apiInvalidVersion.value; BEAST_EXPECT( RPC::getAPIVersionNumber(j_object, false) == RPC::apiInvalidVersion); @@ -202,17 +203,17 @@ class Version_test : public beast::unit_test::suite "\"id\": 5, " "\"method\": \"version\", " "\"params\": {}}"; - auto const with_wrong_api_verion = - std::string("{ ") + + auto const with_wrong_api_verion = std::string("{ ") + "\"jsonrpc\": \"2.0\", " "\"ripplerpc\": \"2.0\", " "\"id\": 6, " "\"method\": \"version\", " "\"params\": { " "\"api_version\": " + - std::to_string( - std::max(RPC::apiMaximumSupportedVersion, RPC::apiBetaVersion) + - 1) + + std::to_string(std::max( + RPC::apiMaximumSupportedVersion.value, + RPC::apiBetaVersion.value) + + 1) + "}}"; auto re = env.rpc( "json2", @@ -275,8 +276,9 @@ class Version_test : public beast::unit_test::suite jrr[jss::version].isMember(jss::last)) return; BEAST_EXPECT( - jrr[jss::version][jss::first] == RPC::apiMinimumSupportedVersion); - BEAST_EXPECT(jrr[jss::version][jss::last] == RPC::apiBetaVersion); + jrr[jss::version][jss::first] == + RPC::apiMinimumSupportedVersion.value); + BEAST_EXPECT(jrr[jss::version][jss::last] == RPC::apiBetaVersion.value); } public: