Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Small refactor to merge setup script to use baked in configs for ropsten #4053

Merged
merged 1 commit into from
May 24, 2022
Merged

Small refactor to merge setup script to use baked in configs for ropsten #4053

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions kiln/devnets/fixed.vars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Data dir will be mounted in /data/lodestar
LODESTAR_FIXED_VARS="--rootDir /data/lodestar --eth1.providerUrls http://127.0.0.1:8545 --execution.urls http://127.0.0.1:8551 --api.rest.enabled --api.rest.host 0.0.0.0 --api.rest.api '*' --network.connectToDiscv5Bootnodes --network.discv5.enabled true --eth1.enabled true --jwt-secret /data/jwtsecret"

# Data dir will be mounted in /data/lodestar
LODESTAR_VAL_FIXED_VARS="--rootDir /data/lodestar"

# Data dir will be mounted in /data/nethermind
NETHERMIND_FIXED_VARS="--datadir /data/nethermind --Network.DiscoveryPort=30303 --Network.P2PPort=30303 --Merge.Enabled=true --Init.DiagnosticMode=None --JsonRpc.Enabled=true --JsonRpc.Host=0.0.0.0 --JsonRpc.AdditionalRpcUrls \"http://localhost:8545|http;ws|net;eth;subscribe;engine;web3;client|no-auth,http://localhost:8551|http;ws|net;eth;subscribe;engine;web3;client\" --JsonRpc.JwtSecretFile /data/jwtsecret"

# Data dir will be mounted in /data/geth
GETH_FIXED_VARS="--datadir /data/geth --authrpc.jwtsecret /data/jwtsecret --http --http.api engine,net,eth,web3 --http.port 8545 --allow-insecure-unlock --http.addr 0.0.0.0 --http.corsdomain \"*\" --http.vhosts \"*\" --authrpc.port=8551"

# Data dir will be mounted in /data/ethereumjs
ETHEREUMJS_FIXED_VARS="--saveReceipts --rpc --rpcport 8545 --ws --rpcEngine --rpcEnginePort=8551 --rpcDebug --loglevel=debug"

# Data dir will be mounted in /data/besu
BESU_FIXED_VARS="--rpc-http-enabled=true --rpc-http-api=ADMIN,CLIQUE,MINER,ETH,NET,DEBUG,TXPOOL,TRACE --rpc-http-host=0.0.0.0 --rpc-http-port=8545 --Xmerge-support=true --engine-rpc-http-port=8551 --rpc-http-cors-origins=\"*\" --host-allowlist=\"*\" --engine-host-allowlist=\"*\" --p2p-enabled=true --engine-jwt-enabled=true"
12 changes: 6 additions & 6 deletions kiln/devnets/kiln.vars
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,16 @@ BESU_IMAGE=hyperledger/besu:develop

LODESTAR_IMAGE=chainsafe/lodestar:next

LODESTAR_EXTRA_ARGS="--execution.urls http://127.0.0.1:8551 --api.rest.enabled --api.rest.host 0.0.0.0 --api.rest.api '*'"
LODESTAR_EXTRA_ARGS="--terminal-total-difficulty-override $MERGE_TTD $LODESTAR_FIXED_VARS"

LODESTAR_VALIDATOR_ARGS='--network kiln --fromMnemonic "lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow" --mnemonicIndexes 0..5'
LODESTAR_VALIDATOR_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD --fromMnemonic \"lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow\" --mnemonicIndexes 0..5 $LODESTAR_VAL_FIXED_VARS"

NETHERMIND_EXTRA_ARGS="--config kiln --Network.DiscoveryPort=30303 --Network.P2PPort=30303 --Merge.Enabled=true --Merge.TerminalTotalDifficulty=$MERGE_TTD --Init.DiagnosticMode=None --JsonRpc.Enabled=true --JsonRpc.Host=0.0.0.0 --JsonRpc.AdditionalRpcUrls \"http://localhost:8545|http;ws|net;eth;subscribe;engine;web3;client|no-auth,http://localhost:8551|http;ws|net;eth;subscribe;engine;web3;client\""
NETHERMIND_EXTRA_ARGS="--config kiln --Merge.TerminalTotalDifficulty=$MERGE_TTD $NETHERMIND_FIXED_VARS"

GETH_EXTRA_ARGS="--http --http.api engine,net,eth,web3 --http.port 8545 --allow-insecure-unlock --http.addr 0.0.0.0 --http.corsdomain \"*\" --http.vhosts \"*\" --authrpc.port=8551 --override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID"
GETH_EXTRA_ARGS="--override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID $GETH_FIXED_VARS"

ETHEREUMJS_EXTRA_ARGS="--saveReceipts --rpc --rpcport 8545 --ws --rpcEngine --rpcEnginePort=8551 --rpcDebug --loglevel=debug"
ETHEREUMJS_EXTRA_ARGS="$ETHEREUMJS_FIXED_VARS"

BESU_EXTRA_ARGS="--rpc-http-enabled=true --rpc-http-api=ADMIN,CLIQUE,MINER,ETH,NET,DEBUG,TXPOOL,TRACE --rpc-http-host=0.0.0.0 --rpc-http-port=8545 --Xmerge-support=true --engine-rpc-http-port=8551 --rpc-http-cors-origins=\"*\" --host-allowlist=\"*\" --engine-host-allowlist=\"*\" --p2p-enabled=true --engine-jwt-enabled=true --network-id=$NETWORK_ID"
BESU_EXTRA_ARGS="--network-id=$NETWORK_ID $BESU_FIXED_VARS"

EXTRA_BOOTNODES=""
12 changes: 6 additions & 6 deletions kiln/devnets/mainnetshadow-5.vars
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,16 @@ BESU_IMAGE=hyperledger/besu:develop

LODESTAR_IMAGE=chainsafe/lodestar:next

LODESTAR_EXTRA_ARGS="--eth1.providerUrls http://127.0.0.1:8545 --execution.urls http://127.0.0.1:8551 --api.rest.enabled --api.rest.host 0.0.0.0 --api.rest.api '*'"
LODESTAR_EXTRA_ARGS="--terminal-total-difficulty-override $MERGE_TTD $LODESTAR_FIXED_VARS"

LODESTAR_VALIDATOR_ARGS='--network kiln --fromMnemonic "lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow" --mnemonicIndexes 0..5'
LODESTAR_VALIDATOR_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD --fromMnemonic \"lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow\" --mnemonicIndexes 0..5 $LODESTAR_VAL_FIXED_VARS"

NETHERMIND_EXTRA_ARGS="--config mainnet_shadowfork --Network.DiscoveryPort=30303 --Network.P2PPort=30303 --Merge.Enabled=true --Merge.TerminalTotalDifficulty=$MERGE_TTD --Init.DiagnosticMode=None --JsonRpc.Enabled=true --JsonRpc.Host=0.0.0.0 --JsonRpc.AdditionalRpcUrls \"http://localhost:8545|http;ws|net;eth;subscribe;engine;web3;client|no-auth,http://localhost:8551|http;ws|net;eth;subscribe;engine;web3;client\""
NETHERMIND_EXTRA_ARGS="--config mainnet_shadowfork --Merge.TerminalTotalDifficulty=$MERGE_TTD $NETHERMIND_FIXED_VARS"

GETH_EXTRA_ARGS="--http --http.api engine,net,eth,web3 --http.port 8545 --allow-insecure-unlock --http.addr 0.0.0.0 --http.corsdomain \"*\" --http.vhosts \"*\" --authrpc.port=8551 --override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID"
GETH_EXTRA_ARGS="--override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID $GETH_FIXED_VARS"

ETHEREUMJS_EXTRA_ARGS="--saveReceipts --rpc --rpcport 8545 --ws --rpcEngine --rpcEnginePort=8551 --rpcDebug --loglevel=debug"
ETHEREUMJS_EXTRA_ARGS="$ETHEREUMJS_FIXED_VARS"

BESU_EXTRA_ARGS="--rpc-http-enabled=true --rpc-http-api=ADMIN,CLIQUE,MINER,ETH,NET,DEBUG,TXPOOL,TRACE --rpc-http-host=0.0.0.0 --rpc-http-port=8545 --Xmerge-support=true --engine-rpc-http-port=8551 --rpc-http-cors-origins=\"*\" --host-allowlist=\"*\" --engine-host-allowlist=\"*\" --p2p-enabled=true --engine-jwt-enabled=true --network-id=$NETWORK_ID"
BESU_EXTRA_ARGS="--network-id=$NETWORK_ID $BESU_FIXED_VARS"

EXTRA_BOOTNODES=""
29 changes: 29 additions & 0 deletions kiln/devnets/ropsten.vars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
DEVNET_NAME=ropsten
# Empty config git dir will be assumed to be clients having bakedin configs
CONFIG_GIT_DIR=
NETWORK_ID=3
MERGE_TTD=43531756765713534

# This will be available in /data/jwtsecret
JWT_SECRET="0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"

GETH_IMAGE=parithoshj/geth:master
NETHERMIND_IMAGE=nethermindeth/nethermind:kiln_shadowfork
ETHEREUMJS_IMAGE=g11tech/ethereumjs:kiln
BESU_IMAGE=hyperledger/besu:develop

LODESTAR_IMAGE=chainsafe/lodestar:next

LODESTAR_EXTRA_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD $LODESTAR_FIXED_VARS"

LODESTAR_VALIDATOR_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD --fromMnemonic \"lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow\" --mnemonicIndexes 0..5 $LODESTAR_VAL_FIXED_VARS"

NETHERMIND_EXTRA_ARGS="--config ropsten --Merge.TerminalTotalDifficulty=$MERGE_TTD $NETHERMIND_FIXED_VARS"

GETH_EXTRA_ARGS="--ropsten --override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID $GETH_FIXED_VARS"

ETHEREUMJS_EXTRA_ARGS="--network ropsten $ETHEREUMJS_FIXED_VARS"

BESU_EXTRA_ARGS="--network-id=$NETWORK_ID $BESU_FIXED_VARS"

EXTRA_BOOTNODES=""
107 changes: 81 additions & 26 deletions kiln/devnets/setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# set -e

source parse-args.sh
source "./fixed.vars"
source $devnetVars

currentDir=$(pwd)
Expand All @@ -22,17 +23,41 @@ then
fi


mkdir $dataDir && mkdir $dataDir/lodestar && mkdir $dataDir/geth && mkdir $dataDir/nethermind && mkdir $dataDir/ethereumjs && mkdir $dataDir/besu && cd $dataDir && git init && git remote add -f origin $setupConfigUrl && git config core.sparseCheckout true && echo "$configGitDir/*" >> .git/info/sparse-checkout && git pull --depth=1 origin main && cd $currentDir
mkdir $dataDir && mkdir $dataDir/lodestar && mkdir $dataDir/geth && mkdir $dataDir/nethermind && mkdir $dataDir/ethereumjs && mkdir $dataDir/besu

if [ ! -n "$(ls -A $dataDir/$configGitDir)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/genesis.json)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/genesis.ssz)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/nethermind_genesis.json)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/el_bootnode.txt)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/bootstrap_nodes.txt)" ]
if [ -n "$configGitDir" ]
then
echo "Configuration directory not setup properly, remove the data directory and run again."
echo "exiting ..."
exit;
else
echo "Configuration discovered!"
if [ ! -n "$(ls -A $dataDir/$configGitDir)" ]
then
cd $dataDir && git init && git remote add -f origin $setupConfigUrl && git config core.sparseCheckout true && echo "$configGitDir/*" >> .git/info/sparse-checkout && git pull --depth=1 origin main && cd $currentDir
fi;

if [ ! -n "$(ls -A $dataDir/$configGitDir)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/genesis.json)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/genesis.ssz)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/nethermind_genesis.json)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/el_bootnode.txt)" ] || [ ! -n "$(ls -A $dataDir/$configGitDir/bootstrap_nodes.txt)" ]
then
echo "Configuration directory not setup properly, remove the data directory and run again."
echo "exiting ..."
exit;
else
echo "Configuration discovered!"
fi;

# Load the required variables from the config dir
bootNode=$(cat $dataDir/$configGitDir/el_bootnode.txt)
bootNode=($bootNode)
bootNodeWithSpace=$(IFS=" " ; echo "${bootNode[*]}")
bootNode=$(IFS=, ; echo "${bootNode[*]}")

bootEnr=$(cat $dataDir/$configGitDir/bootstrap_nodes.txt)
bootEnr=($bootEnr)
bootEnr=$(IFS=" " ; echo "${bootEnr[*]}")

depositContractDeployBlock=$(cat $dataDir/$configGitDir/deposit_contract_block.txt)

else
echo "No configuration specified, assuming the configuration baked in the images and args appropriately set to use it!"
fi;


run_cmd(){
execCmd=$1;
if [ -n "$detached" ]
Expand Down Expand Up @@ -72,11 +97,6 @@ then
fi;

platform=$(uname)
bootNode=$(cat $dataDir/$configGitDir/el_bootnode.txt)
bootNode=($bootNode)
bootNodeWithSpace=$(IFS=" " ; echo "${bootNode[*]}")
bootNode=$(IFS=, ; echo "${bootNode[*]}")


if [ $platform == 'Darwin' ]
then
Expand All @@ -91,48 +111,67 @@ then
$dockerExec pull $GETH_IMAGE

elName="$DEVNET_NAME-geth"
if [ ! -n "$(ls -A $dataDir/geth)" ]
if [ ! -n "$(ls -A $dataDir/geth)" ] && [ -n "$configGitDir" ]
then
echo "setting up geth directory"
$dockerExec run --rm -v $currentDir/$dataDir/$configGitDir:/config -v $currentDir/$dataDir/geth:/data $GETH_IMAGE --datadir /data init /config/genesis.json
fi;
elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir:/data $GETH_IMAGE --datadir /data/geth --authrpc.jwtsecret /data/jwtsecret $GETH_EXTRA_ARGS --bootnodes $EXTRA_BOOTNODES$bootNode"

elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir:/data $GETH_IMAGE $GETH_EXTRA_ARGS"
if [ -n "$configGitDir" ]
then
elCmd="$elCmd --bootnodes $EXTRA_BOOTNODES$bootNode"
fi;

elif [ "$elClient" == "nethermind" ]
then
echo "nethermindImage: $NETHERMIND_IMAGE"
$dockerExec pull $NETHERMIND_IMAGE

elName="$DEVNET_NAME-nethermind"
elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir/$configGitDir:/config -v $currentDir/$dataDir:/data $NETHERMIND_IMAGE --datadir /data/nethermind --Init.ChainSpecPath=/config/nethermind_genesis.json --JsonRpc.JwtSecretFile /data/jwtsecret $NETHERMIND_EXTRA_ARGS --Discovery.Bootnodes $EXTRA_BOOTNODES$bootNode"
elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir:/data"
if [ -n "$configGitDir" ]
then
elCmd="$elCmd -v $currentDir/$dataDir/$configGitDir:/config $NETHERMIND_IMAGE --Init.ChainSpecPath=/config/nethermind_genesis.json --Discovery.Bootnodes $EXTRA_BOOTNODES$bootNode"
else
elCmd="$elCmd $NETHERMIND_IMAGE"
fi;
elCmd="$elCmd $NETHERMIND_EXTRA_ARGS"

elif [ "$elClient" == "ethereumjs" ]
then
echo "ethereumjsImage: $ETHEREUMJS_IMAGE"
$dockerExec pull $ETHEREUMJS_IMAGE

elName="$DEVNET_NAME-ethereumjs"
elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir/$configGitDir:/config -v $currentDir/$dataDir:/data $ETHEREUMJS_IMAGE --datadir /data/ethereumjs --gethGenesis /config/genesis.json --jwt-secret /data/jwtsecret $ETHEREUMJS_EXTRA_ARGS --bootnodes=$EXTRA_BOOTNODES$bootNode"
elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir:/data"
if [ -n "$configGitDir" ]
then
elCmd="$elCmd -v $currentDir/$dataDir/$configGitDir:/config $ETHEREUMJS_IMAGE --bootnodes=$EXTRA_BOOTNODES$bootNode"
else
elCmd="$elCmd $ETHEREUMJS_IMAGE"
fi;
elCmd="$elCmd --datadir /data/ethereumjs --gethGenesis /config/genesis.json --jwt-secret /data/jwtsecret $ETHEREUMJS_EXTRA_ARGS "

elif [ "$elClient" == "besu" ]
then
echo "besuImage: $BESU_IMAGE"
$dockerExec pull $BESU_IMAGE

elName="$DEVNET_NAME-besu"
elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir/$configGitDir:/config -v $currentDir/$dataDir:/data $BESU_IMAGE --data-path=/data --genesis-file=/config/besu_genesis.json --engine-jwt-secret=/data/jwtsecret $BESU_EXTRA_ARGS --bootnodes=$EXTRA_BOOTNODES$bootNode"

elCmd="$dockerCmd --name $elName $elDockerNetwork -v $currentDir/$dataDir:/data"
if [ -n "$configGitDir" ]
then
elCmd="$elCmd -v $currentDir/$dataDir/$configGitDir:/config $BESU_IMAGE --genesis-file=/config/besu_genesis.json --bootnodes=$EXTRA_BOOTNODES$bootNode"
else
elCmd="$elCmd $BESU_IMAGE"
fi;
elCmd="$elCmd $BESU_IMAGE --data-path=/data --engine-jwt-secret=/data/jwtsecret $BESU_EXTRA_ARGS"
fi

echo "lodestarImage: $LODESTAR_IMAGE"
$dockerExec pull $LODESTAR_IMAGE

bootEnr=$(cat $dataDir/$configGitDir/bootstrap_nodes.txt)
bootEnr=($bootEnr)
bootEnr=$(IFS=" " ; echo "${bootEnr[*]}")

depositContractDeployBlock=$(cat $dataDir/$configGitDir/deposit_contract_block.txt)

if [ $platform == 'Darwin' ]
then
clDockerNetwork="--net=container:$elName"
Expand All @@ -141,10 +180,26 @@ else
fi

clName="$DEVNET_NAME-lodestar"
clCmd="$dockerCmd --name $clName $clDockerNetwork -v $currentDir/$dataDir/$configGitDir:/config -v $currentDir/$dataDir:/data $LODESTAR_IMAGE beacon --rootDir /data/lodestar --paramsFile /config/config.yaml --genesisStateFile /config/genesis.ssz --network.connectToDiscv5Bootnodes --network.discv5.enabled true --eth1.enabled true --eth1.depositContractDeployBlock $depositContractDeployBlock $LODESTAR_EXTRA_ARGS --bootnodesFile /config/boot_enr.yaml --jwt-secret /data/jwtsecret"
clCmd="$dockerCmd --name $clName $clDockerNetwork -v $currentDir/$dataDir:/data"
# mount and use config
if [ -n "$configGitDir" ]
then
clCmd="$clCmd -v $currentDir/$dataDir/$configGitDir:/config $LODESTAR_IMAGE beacon --paramsFile /config/config.yaml --genesisStateFile /config/genesis.ssz --eth1.depositContractDeployBlock $depositContractDeployBlock --bootnodesFile /config/boot_enr.yaml"
else
clCmd="$clCmd $LODESTAR_IMAGE beacon"
fi;
clCmd="$clCmd $LODESTAR_EXTRA_ARGS"

valName="$DEVNET_NAME-validator"
valCmd="$dockerCmd --name $valName $clDockerNetwork -v $currentDir/$dataDir/$configGitDir:/config -v $currentDir/$dataDir:/data $LODESTAR_IMAGE validator --rootDir /data/lodestar --paramsFile /config/config.yaml $LODESTAR_VALIDATOR_ARGS"
valCmd="$dockerCmd --name $valName $clDockerNetwork -v $currentDir/$dataDir:/data"
# mount and use config
if [ -n "$configGitDir" ]
then
valCmd="$valCmd -v $currentDir/$dataDir/$configGitDir:/config $LODESTAR_IMAGE validator --paramsFile /config/config.yaml"
else
valCmd="$valCmd $LODESTAR_IMAGE validator"
fi;
valCmd="$valCmd $LODESTAR_VALIDATOR_ARGS"

echo -n $JWT_SECRET > $dataDir/jwtsecret
run_cmd "$elCmd"
Expand Down