feature: Add script to download ledger snapshot and options

.github/workflows/ci.yml

@@ -83,7 +83,7 @@
       - name: Setup python
         uses: RDXWorks-actions/setup-python@main
         with:
-          python-version: 3.10.6
+          python-version: 3.10.13
       - name: Install pipenv
         run: python -m pip install --upgrade pipenv wheel
       - name: Install dependencies
@@ -122,7 +122,7 @@
       - name: Setup python
         uses: RDXWorks-actions/setup-python@main
         with:
-          python-version: 3.10.6
+          python-version: 3.10.13
       - name: Install pipenv
         run: python -m pip install --upgrade pipenv wheel
       - name: Install dependencies
@@ -165,7 +165,7 @@
       - name: setup python
         uses: RDXWorks-actions/setup-python@main
         with:
-          python-version: 3.10.6
+          python-version: 3.10.13
       - name: Install pipenv
         run: |
           pip install pipenv==2023.7.23
@@ -215,7 +215,7 @@
       - name: setup python
         uses: RDXWorks-actions/setup-python@main
         with:
-          python-version: 3.10.6
+          python-version: 3.10.13
       - name: Install pipenv
         run: |
           pip install pipenv==2023.7.23
@@ -346,7 +346,7 @@
           KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: corenode-01
         run: |
           ls -a
           chmod +x ./babylonnode
           export PROMPT_FEEDS="node-runner-cli/test-prompts/corenode-01.yml"
@@ -475,19 +475,25 @@
         with:
           name: ubuntu 22.04
       - name: Run configure command
         run: |
+          set -x
+          pwd
           chmod +x ./babylonnode
-          sudo apt-get update
-          sudo apt-get install containerd runc
+          mv ./babylonnode $HOME/babylonnode && cd $HOME
+          sudo apt-get update -y
+          sudo apt-get install containerd runc jq -y
           ./babylonnode docker dependencies
       - name: Setup config
         run: |
-          chmod +x ./babylonnode
+          set -x
+          pwd && ls -la
+          CHECKOUT_PATH=$(pwd)
+          cd $HOME
+          ls -la work/babylon-nodecli/
           mkdir -p "$HOME/babylon-node-config"
           export DISABLE_VERSION_CHECK=true
-          export RADIXDLT_APP_VERSION_OVERRIDE="rcnet-v2-phase2-r4"
           export DOCKER_COMPOSE_LOCATION="/usr/local/bin/docker-compose"
-          export PROMPT_FEEDS="node-runner-cli/test-prompts/core-gateway-all-local.yml"
+          export PROMPT_FEEDS="$CHECKOUT_PATH/node-runner-cli/test-prompts/core-gateway-all-local.yml"
           ./babylonnode docker config -m DETAILED \
             -d "$HOME/babylon-node-config" \
             -k "$KEYSTORE_PASSWORD" -nk -a
@@ -495,9 +501,12 @@
           KEYSTORE_PASSWORD: ${{secrets.KEYSTORE_PASSWORD}}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Run CLI setup
         run: |
+          set -x
           export DISABLE_VERSION_CHECK=true
           export DOCKER_COMPOSE_LOCATION="/usr/local/bin/docker-compose"
+          pwd && ls -la
+          cd $HOME
           ./babylonnode docker install -f "$HOME/babylon-node-config/config.yaml" -a
           sleep 60
           ./babylonnode auth set-admin-password -m DOCKER -p "$NGINX_ADMIN_PASSWORD"
@@ -512,13 +521,19 @@
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run Monitoring setup
         run: |
+          set -x
+          pwd && ls -la
           export DOCKER_COMPOSE_LOCATION="/usr/local/bin/docker-compose"
+          cd $HOME
           ./babylonnode monitoring config \
                         -m MONITOR_CORE  \
                         -cm "$NGINX_METRICS_PASSWORD"  \
                         -gm "$NGINX_METRICS_PASSWORD"  \
                         -am "$NGINX_METRICS_PASSWORD"
+          ls -la
+          docker compose ps
+
           ./babylonnode monitoring install -a
         env:
           NGINX_ADMIN_PASSWORD: ${{secrets.NGINX_ADMIN_PASSWORD}}

docs/command_reference.adoc

@@ -54,8 +54,8 @@ optional arguments:
                         in DETAILED mode Use this for automation purpose only
   -d CONFIGDIR, --configdir CONFIGDIR
                         Path to node-config directory where config file will
-                        stored. Default value is /Users/kim.fehrs/babylon-
-                        node-config
+                        stored. Default value is /Users/shambu/babylon-node-
+                        config
   -k KEYSTOREPASSWORD, --keystorepassword KEYSTOREPASSWORD
                         Core Node requires a keystore. This is the password
                         for the keystore file the CLI will create new key with
@@ -122,13 +122,13 @@ optional arguments:
                         configure the core node. It is templated into
                         default.config and acts as custom configuration that
                         is not overwritten on install. The default value is
-                        `/Users/kim.fehrs/babylon-node-config/advanced-
-                        user.env-file` if not provided
+                        `/Users/shambu/babylon-node-config/advanced-user.env-
+                        file` if not provided
   -f CONFIGFILE, --configfile CONFIGFILE
                         Path to config file. This file is generated by running
                         'babylonnode docker config'The default value is
-                        `/Users/kim.fehrs/babylon-node-config/config.yaml` if
-                        not provided
+                        `/Users/shambu/babylon-node-config/config.yaml` if not
+                        provided
   -u, --update          Pass this option to update the deployed softwares to
                         latest version. CLI prompts to confirm the versions if
                         '-a' is not passed
@@ -150,8 +150,8 @@ optional arguments:
   -f CONFIGFILE, --configfile CONFIGFILE
                         Path to config file. This file is generated by running
                         'babylonnode docker config'The default value is
-                        `/Users/kim.fehrs/babylon-node-config/config.yaml` if
-                        not provided
+                        `/Users/shambu/babylon-node-config/config.yaml` if not
+                        provided
 ----
 
 ==== babylonnode docker stop
@@ -168,8 +168,7 @@ optional arguments:
   -f COMPOSEFILE, --composefile COMPOSEFILE
                         Path to docker-compose file. This file is generated by
                         running 'babylonnode docker install'The default value
-                        is `/Users/kim.fehrs/docker-compose.yml` if not
-                        provided
+                        is `/Users/shambu/docker-compose.yml` if not provided
   -v, --removevolumes   Remove the volumes
 ----
 === Radix node CLI command reference
@@ -232,8 +231,8 @@ optional arguments:
                         mode Use this for automation purpose only
   -d CONFIGDIR, --configdir CONFIGDIR
                         Path to node-config directory where config file will
-                        stored. Default value is /Users/kim.fehrs/babylon-
-                        node-config
+                        stored. Default value is /Users/shambu/babylon-node-
+                        config
   -dd DATA_DIRECTORY, --data_directory DATA_DIRECTORY
                         Folder for data generated by the node
   -i HOSTIP, --hostip HOSTIP
@@ -294,14 +293,14 @@ optional arguments:
   -f CONFIGFILE, --configfile CONFIGFILE
                         Path to config file. This file is generated by running
                         'babylonnode systemd config'The default value is
-                        `/Users/kim.fehrs/babylon-node-config/config.yaml` if
-                        not provided
+                        `/Users/shambu/babylon-node-config/config.yaml` if not
+                        provided
   -auc ADVANCEDUSERCONFIG, --advanceduserconfig ADVANCEDUSERCONFIG
                         Path to advanced config file. This file can directly
                         configure the core node. It is templated into
                         default.config and acts as custom configuration that
                         is not overwritten on install. The default value is
-                        `/Users/kim.fehrs/babylon-node-config/advanced-
+                        `/Users/shambu/babylon-node-config/advanced-
                         user.default.config` if not provided
   -m, --manual          Only generate systemd file but not put it into systemd
                         folder.This is mainly used for automation in
@@ -342,14 +341,19 @@ optional arguments:
 === Ledger sync using an external S3 bucket
 Below are the list of commands that can be used with cli to sync the ledger with a S3 bucket.
 
-==== babylonnode ledger sync
+==== babylonnode ledger s3-download
 [source, bash,subs="+quotes, +attributes" ]
 ----
-usage: babylonnode ledger sync [-h] -d DEST -bn BUCKETNAME [-bf BUCKETFOLDER]
+usage: babylonnode ledger s3-download [-h] -d DEST -bn BUCKETNAME
+                                      [-bf BUCKETFOLDER]
 
-This commands allows to download the content of an external S3 bucket to the
-ledger folder. The ledger folder and the name and folder of an external S3
-bucket should be indicated
+Downloads a backuped ledger from an S3 bucket. Args: args: An object
+containing the following attributes: - bucketname (str): The name of the S3
+bucket. - bucketfolder (str): The folder within the S3 bucket. - dest (str):
+The destination path where the backup ledger will be downloaded. Raises:
+ValueError: If the bucket name is not provided. Example: args =
+Namespace(bucketname='my-bucket', bucketfolder='backups', dest='/local/path')
+s3_download(args)
 
 options:
   -h, --help            show this help message and exit
@@ -366,6 +370,29 @@ optional arguments:
                         S3 bucket folder to download the backup of the ledger
                         from
 ----
+
+==== babylonnode ledger fetch-community-snapshot
+[source, bash,subs="+quotes, +attributes" ]
+----
+usage: babylonnode ledger fetch-community-snapshot [-h] [-d DEST]
+                                                   [-s {radix.live}]
+
+Downloads the latest community snapshot of the ledger. Args: args: An object
+containing the following attributes: - dest (str): The destination path where
+the backup ledger will be downloaded. Raises: ValueError: If the source is not
+provided. Example: args = Namespace(dest='/local/path')
+download_and_extract_snapshot(args)
+
+options:
+  -h, --help            show this help message and exit
+
+optional arguments:
+  -d DEST, --dest DEST  Destination path where the backup of the ledger will
+                        be downloaded
+  -s {radix.live}, --source {radix.live}
+                        Source to download the ledger from. Radix.live is the
+                        only supported source at the moment.
+----
 === Core node setup using systemd
 Below are the list of commands supported in cli to setup a core node process as a systemd process
 
@@ -402,14 +429,14 @@ optional arguments:
   -f CONFIGFILE, --configfile CONFIGFILE
                         Path to config file. This file is generated by running
                         'babylonnode systemd config'The default value is
-                        `/Users/kim.fehrs/babylon-node-config/config.yaml` if
-                        not provided
+                        `/Users/shambu/babylon-node-config/config.yaml` if not
+                        provided
   -auc ADVANCEDUSERCONFIG, --advanceduserconfig ADVANCEDUSERCONFIG
                         Path to advanced config file. This file can directly
                         configure the core node. It is templated into
                         default.config and acts as custom configuration that
                         is not overwritten on install. The default value is
-                        `/Users/kim.fehrs/babylon-node-config/advanced-
+                        `/Users/shambu/babylon-node-config/advanced-
                         user.default.config` if not provided
   -m, --manual          Only generate systemd file but not put it into systemd
                         folder.This is mainly used for automation in
@@ -686,7 +713,7 @@ options:
 optional arguments:
   -f MONITORINGCONFIGFILE, --monitoringconfigfile MONITORINGCONFIGFILE
                         Path to config file. Default is
-                        '/Users/kim.fehrs/monitoring/monitoring_config.yaml'
+                        '/Users/shambu/monitoring/monitoring_config.yaml'
   -a, --autoapprove     Set this to true to run without any prompts
 ----
 
@@ -705,7 +732,7 @@ options:
 optional arguments:
   -f MONITORINGCONFIGFILE, --monitoringconfigfile MONITORINGCONFIGFILE
                         Path to config file. Default is
-                        '/Users/kim.fehrs/monitoring/monitoring_config.yaml'
+                        '/Users/shambu/monitoring/monitoring_config.yaml'
   -a, --autoapprove     Set this to true to run without any prompts
 ----
 
@@ -722,7 +749,7 @@ options:
 optional arguments:
   -f MONITORINGCONFIGFILE, --monitoringconfigfile MONITORINGCONFIGFILE
                         Path to config file. Default is
-                        '/Users/kim.fehrs/monitoring/monitoring_config.yaml'
+                        '/Users/shambu/monitoring/monitoring_config.yaml'
   -v, --removevolumes   Remove the volumes
 ----
 

node-runner-cli/Dockerfile.ubuntufocal

@@ -16,16 +16,16 @@ RUN apt-get update \
 RUN set -ex \
     && curl https://pyenv.run | bash
 
-RUN /root/.pyenv/bin/pyenv install -v 3.10.6
+RUN /root/.pyenv/bin/pyenv install -v 3.10.13
 RUN pip install pipenv==2023.7.23
 
 WORKDIR /app
 COPY Pipfile Pipfile.lock /app/
-RUN PIPENV_VENV_IN_PROJECT=1 pipenv --python=/root/.pyenv/versions/3.10.6/bin/python run pipenv install
-RUN pipenv --python=/root/.pyenv/versions/3.10.6/bin/python run pip install pyinstaller
+RUN PIPENV_VENV_IN_PROJECT=1 pipenv --python=/root/.pyenv/versions/3.10.13/bin/python run pipenv install
+RUN pipenv --python=/root/.pyenv/versions/3.10.13/bin/python run pip install pyinstaller
 
 COPY . /app
-RUN pipenv run --python=/root/.pyenv/versions/3.10.6/bin/python pyinstaller babylonnode.spec
+RUN pipenv run --python=/root/.pyenv/versions/3.10.13/bin/python pyinstaller babylonnode.spec
 
 RUN DISABLE_VERSION_CHECK=true /app/dist/babylonnode version
 

node-runner-cli/Dockerfile.ubuntujammy

@@ -23,7 +23,7 @@ RUN pipenv run pip install pyinstaller
 
 COPY . /app
 RUN pipenv run pyinstaller babylonnode.spec
-
+RUN pipenv run python --version
 RUN DISABLE_VERSION_CHECK=true /app/dist/babylonnode version
 
 FROM scratch AS export-stage

node-runner-cli/Pipfile

@@ -19,6 +19,7 @@ pytest = "==8.1.1"
 radix-engine-toolkit = "==0.12.1.dev3"
 cryptography = "==41.0.4"
 pyinstaller-hooks-contrib = "==2023.12"
+zstandard= "==0.15.2"
 
 [requires]
-python_version = "3.10.6"
+python_version = "3.10.13"