Merge branch 'develop' into noacquire

.codecov.yml

@@ -1,5 +1,6 @@
-
-codecov:
-  ci:
-    - !appveyor
-    - travis
+coverage:
+  status:
+    project:
+      default:
+        target: 60%
+        threshold: 2%

.github/actions/build/action.yml

@@ -6,6 +6,8 @@ inputs:
     required: true
   cmake-args:
     default: null
+  cmake-target:
+    default: all
 # An implicit input is the environment variable `build_dir`.
 runs:
   using: composite
@@ -26,4 +28,5 @@ runs:
         cmake \
           --build ${build_dir} \
           --config ${{ inputs.configuration }} \
-          --parallel ${NUM_PROCESSORS:-$(nproc)}
+          --parallel ${NUM_PROCESSORS:-$(nproc)} \
+          --target ${{ inputs.cmake-target }}

.github/actions/dependencies/action.yml

@@ -12,8 +12,36 @@ runs:
     - name: export custom recipes
       shell: bash
       run: |
+        conan config set general.revisions_enabled=1
         conan export external/snappy snappy/1.1.10@
+        conan export external/rocksdb rocksdb/6.29.5@
         conan export external/soci soci/4.0.3@
+    - name: add Ripple Conan remote
+      shell: bash
+      run: |
+        conan remote list
+        conan remote remove ripple || true
+        # Do not quote the URL. An empty string will be accepted (with
+        # a non-fatal warning), but a missing argument will not.
+        conan remote add ripple ${{ env.CONAN_URL }} --insert 0
+    - name: try to authenticate to Ripple Conan remote
+      id: remote
+      shell: bash
+      run: |
+        # `conan user` implicitly uses the environment variables
+        # CONAN_LOGIN_USERNAME_<REMOTE> and CONAN_PASSWORD_<REMOTE>.
+        # https://docs.conan.io/1/reference/commands/misc/user.html#using-environment-variables
+        # https://docs.conan.io/1/reference/env_vars.html#conan-login-username-conan-login-username-remote-name
+        # https://docs.conan.io/1/reference/env_vars.html#conan-password-conan-password-remote-name
+        echo outcome=$(conan user --remote ripple --password >&2 \
+          && echo success || echo failure) | tee ${GITHUB_OUTPUT}
+    - name: list missing binaries
+      id: binaries
+      shell: bash
+      # Print the list of dependencies that would need to be built locally.
+      # A non-empty list means we have "failed" to cache binaries remotely.
+      run: |
+        echo missing=$(conan info . --build missing --settings build_type=${{ inputs.configuration }} --json 2>/dev/null  | grep '^\[') | tee ${GITHUB_OUTPUT}
     - name: install dependencies
       shell: bash
       run: |
@@ -24,3 +52,7 @@ runs:
           --build missing \
           --settings build_type=${{ inputs.configuration }} \
           ..
+    - name: upload dependencies to remote
+      if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success')
+      shell: bash
+      run: conan upload --remote ripple '*' --all --parallel --confirm

.github/pull_request_template.md

@@ -1,6 +1,12 @@
 <!--
 This PR template helps you to write a good pull request description.
 Please feel free to include additional useful information even beyond what is requested below.
+
+If your branch is on a personal fork and has a name that allows it to
+run CI build/test jobs (e.g. "ci/foo"), remember to rename it BEFORE
+opening the PR.  This avoids unnecessary redundant test runs. Renaming
+the branch after opening the PR will close the PR.
+https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-branches-in-your-repository/renaming-a-branch
 -->
 
 ## High Level Overview of Change
@@ -33,6 +39,7 @@ Please check [x] relevant options, delete irrelevant ones.
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] Refactor (non-breaking change that only restructures code)
+- [ ] Performance (increase or change in throughput and/or latency)
 - [ ] Tests (you added tests for code that already exists, or your new feature included in this PR)
 - [ ] Documentation update
 - [ ] Chore (no impact to binary, e.g. `.gitignore`, formatting, dropping support for older tooling)
@@ -58,6 +65,12 @@ Please check [x] relevant options, delete irrelevant ones.
 ## Before / After
 If relevant, use this section for an English description of the change at a technical level.
 If this change affects an API, examples should be included here.
+
+For performance-impacting changes, please provide these details:
+1. Is this a new feature, bug fix, or improvement to existing functionality?
+2. What behavior/functionality does the change impact?
+3. In what processing can the impact be measured? Be as specific as possible - e.g. RPC client call, payment transaction that involves LOB, AMM, caching, DB operations, etc.
+4. Does this change affect concurrent processing - e.g. does it involve acquiring locks, multi-threaded processing, or async processing?
 -->
 
 <!--

.github/workflows/clang-format.yml

@@ -8,7 +8,7 @@ jobs:
     env:
       CLANG_VERSION: 10
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install clang-format
       run: |
         codename=$( lsb_release --codename --short )
@@ -30,7 +30,7 @@ jobs:
         git diff --exit-code | tee "clang-format.patch"
     - name: Upload patch
       if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       continue-on-error: true
       with:
         name: clang-format.patch

.github/workflows/doxygen.yml

@@ -13,11 +13,10 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-    container:
-      image: docker://rippleci/rippled-ci-builder:2944b78d22db
+    container: rippleci/rippled-build-ubuntu:aaf5e3e
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: check environment
         run: |
           echo ${PATH} | tr ':' '\n'

.github/workflows/levelization.yml

@@ -8,7 +8,7 @@ jobs:
     env:
       CLANG_VERSION: 10
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Check levelization
       run: Builds/levelization/levelization.sh
     - name: Check for differences
@@ -18,7 +18,7 @@ jobs:
         git diff --exit-code | tee "levelization.patch"
     - name: Upload patch
       if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       continue-on-error: true
       with:
         name: levelization.patch

.github/workflows/macos.yml

@@ -1,5 +1,16 @@
 name: macos
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+    # If the branches list is ever changed, be sure to change it on all
+    # build/test jobs (nix, macos, windows)
+    branches:
+      # Always build the package branches
+      - develop
+      - release
+      - master
+      # Branches that opt-in to running
+      - 'ci/**'
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -22,23 +33,32 @@ jobs:
       NUM_PROCESSORS: 12
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      - name: install Conan
+        run: |
+          brew install conan@1
+          echo '/opt/homebrew/opt/conan@1/bin' >> $GITHUB_PATH
       - name: install Ninja
         if: matrix.generator == 'Ninja'
         run: brew install ninja
       - name: check environment
         run: |
+          env | sort
           echo ${PATH} | tr ':' '\n'
           python --version
           conan --version
           cmake --version
-          env | sort
       - name: configure Conan
         run : |
-          conan profile get env.CXXFLAGS default || true
+          conan profile new default --detect || true
+          conan profile update settings.compiler.cppstd=20 default
           conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]' default
-      - name: dependencies
+      - name: build dependencies
         uses: ./.github/actions/dependencies
+        env:
+          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+          CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
+          CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
         with:
           configuration: ${{ matrix.configuration }}
       - name: build

.github/workflows/nix.yml

@@ -1,5 +1,16 @@
 name: nix
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+    # If the branches list is ever changed, be sure to change it on all
+    # build/test jobs (nix, macos, windows)
+    branches:
+      # Always build the package branches
+      - develop
+      - release
+      - master
+      # Branches that opt-in to running
+      - 'ci/**'
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -49,21 +60,19 @@ jobs:
               cc: /usr/bin/clang-14
               cxx: /usr/bin/clang++-14
     runs-on: [self-hosted, heavy]
-    container: thejohnfreeman/rippled-build-ubuntu:12e19cd9034b
+    container: rippleci/rippled-build-ubuntu:aaf5e3e
     env:
       build_dir: .build
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: check environment
         run: |
           echo ${PATH} | tr ':' '\n'
           conan --version
           cmake --version
           env | sort
       - name: configure Conan
-        env:
-          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
         run: |
           conan profile new default --detect
           conan profile update settings.compiler.cppstd=20 default
@@ -73,33 +82,17 @@ jobs:
           conan profile update env.CC=${{ matrix.profile.cc }} default
           conan profile update env.CXX=${{ matrix.profile.cxx }} default
           conan profile update conf.tools.build:compiler_executables='{"c": "${{ matrix.profile.cc }}", "cpp": "${{ matrix.profile.cxx }}"}' default
-          # Do not quote the URL. An empty string will be accepted (with
-          # a non-fatal warning), but a missing argument will not.
-          conan remote add ripple ${{ env.CONAN_URL }} --insert 0
-      - name: try to authenticate to ripple Conan remote
-        id: remote
-        run: |
-          echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} --password ${{ secrets.CONAN_TOKEN }} >&2 && echo success || echo failure) | tee ${GITHUB_OUTPUT}
       - name: archive profile
         # Create this archive before dependencies are added to the local cache.
         run: tar -czf conan.tar -C ~/.conan .
-      - name: list missing binaries
-        id: binaries
-        # Print the list of dependencies that would need to be built locally.
-        # A non-empty list means we have "failed" to cache binaries remotely.
-        run: |
-          echo missing=$(conan info . --build missing --settings build_type=${{ matrix.configuration }} --json 2>/dev/null  | grep '^\[') | tee ${GITHUB_OUTPUT}
       - name: build dependencies
-        if: (steps.binaries.outputs.missing != '[]')
         uses: ./.github/actions/dependencies
+        env:
+          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+          CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
+          CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
         with:
           configuration: ${{ matrix.configuration }}
-      - name: upload dependencies to remote
-        if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success')
-        run: conan upload --remote ripple '*' --all --parallel --confirm
-      - name: recreate archive with dependencies
-        if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'failure')
-        run: tar -czf conan.tar -C ~/.conan .
       - name: upload archive
         uses: actions/upload-artifact@v3
         with:
@@ -125,7 +118,7 @@ jobs:
           - "-Dunity=ON"
     needs: dependencies
     runs-on: [self-hosted, heavy]
-    container: thejohnfreeman/rippled-build-ubuntu:12e19cd9034b
+    container: rippleci/rippled-build-ubuntu:aaf5e3e
     env:
       build_dir: .build
     steps:
@@ -139,15 +132,16 @@ jobs:
           tar -xzf conan.tar -C ~/.conan
       - name: check environment
         run: |
+          env | sort
           echo ${PATH} | tr ':' '\n'
           conan --version
           cmake --version
-          env | sort
-          ls ~/.conan
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: dependencies
         uses: ./.github/actions/dependencies
+        env:
+          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
         with:
           configuration: ${{ matrix.configuration }}
       - name: build
@@ -159,3 +153,78 @@ jobs:
       - name: test
         run: |
           ${build_dir}/rippled --unittest --unittest-jobs $(nproc)
+
+
+  coverage:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux
+        compiler:
+          - gcc
+        configuration:
+          - Debug
+    needs: dependencies
+    runs-on: [self-hosted, heavy]
+    container: rippleci/rippled-build-ubuntu:aaf5e3e
+    env:
+      build_dir: .build
+    steps:
+      - name: download cache
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.platform }}-${{ matrix.compiler }}-${{ matrix.configuration }}
+      - name: extract cache
+        run: |
+          mkdir -p ~/.conan
+          tar -xzf conan.tar -C ~/.conan
+      - name: check environment
+        run: |
+          echo ${PATH} | tr ':' '\n'
+          conan --version
+          cmake --version
+          gcovr --version
+          env | sort
+          ls ~/.conan
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: dependencies
+        uses: ./.github/actions/dependencies
+        env:
+          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+        with:
+          configuration: ${{ matrix.configuration }}
+      - name: build
+        uses: ./.github/actions/build
+        with:
+          generator: Ninja
+          configuration: ${{ matrix.configuration }}
+          cmake-args: >-
+            -Dcoverage=ON
+            -Dcoverage_format=xml
+            -DCODE_COVERAGE_VERBOSE=ON
+            -DCMAKE_CXX_FLAGS="-O0"
+            -DCMAKE_C_FLAGS="-O0"
+          cmake-target: coverage
+      - name: build
+        shell: bash
+        run: |
+          mv "${build_dir}/coverage.xml" ./
+      - name: archive coverage report
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage.xml
+          path: coverage.xml
+          retention-days: 30
+      - name: upload coverage report
+        uses: wandalen/wretry.action@v1.3.0
+        with:
+          action: codecov/codecov-action@v4
+          with: |
+            files: coverage.xml
+            fail_ci_if_error: true
+            verbose: true
+            token: ${{ secrets.CODECOV_TOKEN }}
+          attempt_limit: 5
+          attempt_delay: 35000 # in milliseconds