diff --git a/.github/workflows/conventional-pr-title-checker.yml b/.github/workflows/conventional-pr-title-checker.yml deleted file mode 100644 index 845b07e..0000000 --- a/.github/workflows/conventional-pr-title-checker.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Check PR title for conventional commits -name: Check PR title -on: - pull_request_target: - types: - - opened - - reopened - - edited - - synchronize - -# cancel redundant builds -concurrency: - group: "title-checker-${{ github.head_ref }}" - cancel-in-progress: true - -jobs: - title_check: - runs-on: ubuntu-latest - steps: - - uses: aslafy-z/conventional-pr-title-action@v3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 2655377..71568da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,8 @@ RUN foundryup RUN git clone https://github.com/ethereum-optimism/optimism.git && \ cd optimism && \ - git checkout tutorials/chain && \ + git checkout develop && \ + git pull origin develop && \ pnpm install && \ pnpm build @@ -55,6 +56,7 @@ WORKDIR /workspace RUN apt-get update && apt-get install -y --no-install-recommends \ jq \ direnv \ + git \ bash \ curl \ ca-certificates \ diff --git a/README.md b/README.md index e5167c2..10bbd48 100644 --- a/README.md +++ b/README.md @@ -1 +1,15 @@ ## Welcome to Optimism Package +default package for Optimism +```yaml +optimism_package: + participants: + - el_type: op-geth + el_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth:latest + cl_type: op-node + cl_image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:develop + count: 1 + network_params: + network: kurtosis + network_id: "2151908" + seconds_per_slot: 2 +``` diff --git a/main.star b/main.star index 6bb1130..e414c35 100644 --- a/main.star +++ b/main.star @@ -1,6 +1,18 @@ input_parser = import_module("./src/package_io/input_parser.star") ethereum_package = import_module("github.com/kurtosis-tech/ethereum-package/main.star") contract_deployer = import_module("./src/contracts/contract_deployer.star") +static_files = import_module( + "github.com/kurtosis-tech/ethereum-package/src/static_files/static_files.star" +) +participant_network = import_module("./src/participant_network.star") + + +def get_l1_stuff(all_l1_participants, l1_network_params): + first_l1_el_node = all_l1_participants[0].el_context.rpc_http_url + first_l1_cl_node = all_l1_participants[0].cl_context.beacon_http_url + l1_chain_id = l1_network_params.network_id + l1_block_time = l1_network_params.seconds_per_slot + return first_l1_el_node, first_l1_cl_node, l1_chain_id, l1_block_time def run(plan, args={}): @@ -21,22 +33,50 @@ def run(plan, args={}): plan.print("Deploying a local L1") l1 = ethereum_package.run(plan, ethereum_args) all_l1_participants = l1.all_participants + l1_network_params = l1.network_params priv_key = l1.pre_funded_accounts[ 12 ].private_key # reserved for L2 contract deployer # Deploy L2 smart contracts - plan.print("Deploying the L2 smart contracts") - first_l1_el_node = all_l1_participants[0].el_context.rpc_http_url - first_l1_cl_node = all_l1_participants[0].cl_context.beacon_http_url - contract_deployer.launch_contract_deployer( - plan, first_l1_el_node, first_l1_cl_node, priv_key - ) - # Parse the values for the args plan.print("Parsing the L2 input args") optimism_args = args["optimism_package"] + first_l1_el_node, first_l1_cl_node, l1_chain_id, l1_block_time = get_l1_stuff( + all_l1_participants, l1_network_params + ) + + args_with_right_defaults = input_parser.input_parser(plan, optimism_args) + network_params = args_with_right_defaults.network_params + + l2_block_time = network_params.seconds_per_slot + l2_chain_id = network_params.network_id + + el_cl_data = contract_deployer.launch_contract_deployer( + plan, + first_l1_el_node, + first_l1_cl_node, + priv_key, + l1_chain_id, + l2_chain_id, + l1_block_time, + l2_block_time, + ) + # Deploy the L2 plan.print("Deploying a local L2") - args_with_right_defaults = input_parser.input_parser(plan, optimism_args) - plan.print(args_with_right_defaults) + + jwt_file = plan.upload_files( + src=static_files.JWT_PATH_FILEPATH, + name="op_jwt_file", + ) + + all_participants = participant_network.launch_participant_network( + plan, + args_with_right_defaults.participants, + jwt_file, + network_params, + el_cl_data, + ) + + plan.print(all_participants) diff --git a/network_params.yaml b/network_params.yaml index 5ab54e1..71d2bb3 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -8,9 +8,9 @@ optimism_package: ethereum_package: participants: - el_type: geth - - el_type: reth + #- el_type: reth network_params: preset: minimal additional_services: - dora - - blockscout + #- blockscout diff --git a/src/contracts/contract_deployer.star b/src/contracts/contract_deployer.star index 8afc3ba..4456144 100644 --- a/src/contracts/contract_deployer.star +++ b/src/contracts/contract_deployer.star @@ -1,4 +1,4 @@ -IMAGE = "parithoshj/op-test:v3" +IMAGE = "bbusa/op:latest" ENVRC_PATH = "/workspace/optimism/.envrc" @@ -11,8 +11,12 @@ def launch_contract_deployer( el_rpc_http_url, cl_rpc_http_url, priv_key, + l1_chain_id, + l2_chain_id, + l1_block_time, + l2_block_time, ): - plan.run_sh( + op_genesis = plan.run_sh( description="Deploying L2 contracts (takes a few minutes (30 mins for mainnet preset - 4 mins for minimal preset) -- L1 has to be finalized first)", image=IMAGE, env_vars={ @@ -20,6 +24,20 @@ def launch_contract_deployer( "WEB3_PRIVATE_KEY": str(priv_key), "CL_RPC_URL": str(cl_rpc_http_url), "FUND_VALUE": "10", + "DEPLOYMENT_OUTFILE": "/workspace/optimism/packages/contracts-bedrock/deployments/" + + str(l1_chain_id) + + "/kurtosis.json", + "DEPLOY_CONFIG_PATH": "/workspace/optimism/packages/contracts-bedrock/deploy-config/getting-started.json", + "STATE_DUMP_PATH": "/workspace/optimism/packages/contracts-bedrock/deployments/" + + str(l1_chain_id) + + "/state-dump.json", + "L1_RPC_KIND": "any", + "L1_RPC_URL": str(el_rpc_http_url), + "L1_CHAIN_ID": str(l1_chain_id), + "L2_CHAIN_ID": str(l2_chain_id), + "L1_BLOCK_TIME": str(l1_block_time), + "L2_BLOCK_TIME": str(l2_block_time), + "DEPLOYMENT_CONTEXT": "getting-started", }, store=[ StoreSpec(src="/network-configs", name="op-genesis-configs"), @@ -32,17 +50,11 @@ def launch_contract_deployer( "sed -i '1d' {0}".format( ENVRC_PATH ), # Remove the first line (not commented out) - "echo 'export L1_RPC_KIND=any' >> {0}".format(ENVRC_PATH), - "echo 'export L1_RPC_URL={0}' >> {1}".format( - el_rpc_http_url, ENVRC_PATH - ), "echo 'export IMPL_SALT=$(openssl rand -hex 32)' >> {0}".format( ENVRC_PATH ), - "echo 'export DEPLOYMENT_CONTEXT=getting-started' >> {0}".format( - ENVRC_PATH - ), ". {0}".format(ENVRC_PATH), + "mkdir -p /network-configs", "web3 transfer $FUND_VALUE to $GS_ADMIN_ADDRESS", # Fund Admin "sleep 3", "web3 transfer $FUND_VALUE to $GS_BATCHER_ADDRESS", # Fund Batcher @@ -63,13 +75,20 @@ def launch_contract_deployer( "sleep 12", "forge script scripts/Deploy.s.sol:Deploy --private-key $GS_ADMIN_PRIVATE_KEY --broadcast --rpc-url $L1_RPC_URL", "sleep 3", + "CONTRACT_ADDRESSES_PATH=$DEPLOYMENT_OUTFILE forge script scripts/L2Genesis.s.sol:L2Genesis --sig 'runWithStateDump()' --chain-id $L2_CHAIN_ID", "cd /workspace/optimism/op-node", - "go run cmd/main.go genesis l2 --deploy-config ../packages/contracts-bedrock/deploy-config/getting-started.json --l1-deployments ../packages/contracts-bedrock/deployments/getting-started/.deploy --outfile.l2 genesis.json --outfile.rollup rollup.json --l1-rpc $L1_RPC_URL", - "mkdir -p /network-configs", - "mv /workspace/optimism/op-node/genesis.json /network-configs/genesis.json", - "mv /workspace/optimism/op-node/rollup.json /network-configs/rollup.json", - "mv /workspace/optimism/packages/contracts-bedrock/deployments/getting-started/.deploy /network-configs/.deploy", + "go run cmd/main.go genesis l2 \ + --l1-rpc $L1_RPC_URL \ + --deploy-config $DEPLOY_CONFIG_PATH \ + --l2-allocs $STATE_DUMP_PATH \ + --l1-deployments $DEPLOYMENT_OUTFILE \ + --outfile.l2 /network-configs/genesis.json \ + --outfile.rollup /network-configs/rollup.json", + "mv $DEPLOY_CONFIG_PATH /network-configs/getting-started.json", + "mv $DEPLOYMENT_OUTFILE /network-configs/kurtosis.json", + "mv $STATE_DUMP_PATH /network-configs/state-dump.json", ] ), wait="2000s", ) + return op_genesis.files_artifacts[0] diff --git a/src/el/el_launcher.star b/src/el/el_launcher.star new file mode 100644 index 0000000..10c04f0 --- /dev/null +++ b/src/el/el_launcher.star @@ -0,0 +1,69 @@ +constants = import_module( + "github.com/kurtosis-tech/ethereum-package/src/package_io/constants.star" +) +shared_utils = import_module( + "github.com/kurtosis-tech/ethereum-package/src/shared_utils/shared_utils.star" +) + +op_geth = import_module("./op-geth/op_geth_launcher.star") + + +def launch( + plan, + jwt_file, + network_params, + el_cl_data, + participants, + num_participants, +): + el_launchers = { + "op-geth": { + "launcher": op_geth.new_op_geth_launcher( + el_cl_data, + jwt_file, + network_params.network, + network_params.network_id, + ), + "launch_method": op_geth.launch, + }, + } + + all_el_contexts = [] + + for index, participant in enumerate(participants): + cl_type = participant.cl_type + el_type = participant.el_type + + if el_type not in el_launchers: + fail( + "Unsupported launcher '{0}', need one of '{1}'".format( + el_type, ",".join(el_launchers.keys()) + ) + ) + + el_launcher, launch_method = ( + el_launchers[el_type]["launcher"], + el_launchers[el_type]["launch_method"], + ) + + # Zero-pad the index using the calculated zfill value + index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants)))) + + el_service_name = "op-el-{0}-{1}-{2}".format(index_str, el_type, cl_type) + + el_context = launch_method( + plan, + el_launcher, + el_service_name, + participant.el_image, + all_el_contexts, + ) + # # Add participant el additional prometheus metrics + # for metrics_info in el_context.el_metrics_info: + # if metrics_info != None: + # metrics_info["config"] = participant.prometheus_config + + all_el_contexts.append(el_context) + + plan.print("Successfully added {0} EL participants".format(num_participants)) + return all_el_contexts diff --git a/src/el/op-geth/op_geth_launcher.star b/src/el/op-geth/op_geth_launcher.star index e69de29..dc7026c 100644 --- a/src/el/op-geth/op_geth_launcher.star +++ b/src/el/op-geth/op_geth_launcher.star @@ -0,0 +1,267 @@ +shared_utils = import_module( + "github.com/kurtosis-tech/ethereum-package/src/shared_utils/shared_utils.star" +) +# input_parser = import_module("../../package_io/input_parser.star") +el_context = import_module( + "github.com/kurtosis-tech/ethereum-package/src/el/el_context.star" +) +el_admin_node_info = import_module( + "github.com/kurtosis-tech/ethereum-package/src/el/el_admin_node_info.star" +) + + +node_metrics = import_module( + "github.com/kurtosis-tech/ethereum-package/src//node_metrics_info.star" +) +constants = import_module( + "github.com/kurtosis-tech/ethereum-package/src/package_io/constants.star" +) + +RPC_PORT_NUM = 8545 +WS_PORT_NUM = 8546 +DISCOVERY_PORT_NUM = 30303 +ENGINE_RPC_PORT_NUM = 8551 +METRICS_PORT_NUM = 9001 + +# The min/max CPU/memory that the execution node can use +EXECUTION_MIN_CPU = 300 +EXECUTION_MIN_MEMORY = 512 + +# Port IDs +RPC_PORT_ID = "rpc" +WS_PORT_ID = "ws" +TCP_DISCOVERY_PORT_ID = "tcp-discovery" +UDP_DISCOVERY_PORT_ID = "udp-discovery" +ENGINE_RPC_PORT_ID = "engine-rpc" +ENGINE_WS_PORT_ID = "engineWs" +METRICS_PORT_ID = "metrics" + +# TODO(old) Scale this dynamically based on CPUs available and Geth nodes mining +NUM_MINING_THREADS = 1 + +METRICS_PATH = "/debug/metrics/prometheus" + +# The dirpath of the execution data directory on the client container +EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/geth/execution-data" + + +def get_used_ports(discovery_port=DISCOVERY_PORT_NUM): + used_ports = { + RPC_PORT_ID: shared_utils.new_port_spec( + RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL), + TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.TCP_PROTOCOL + ), + UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec( + discovery_port, shared_utils.UDP_PROTOCOL + ), + ENGINE_RPC_PORT_ID: shared_utils.new_port_spec( + ENGINE_RPC_PORT_NUM, + shared_utils.TCP_PROTOCOL, + ), + METRICS_PORT_ID: shared_utils.new_port_spec( + METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL + ), + } + return used_ports + + +ENTRYPOINT_ARGS = ["sh", "-c"] + +VERBOSITY_LEVELS = { + constants.GLOBAL_LOG_LEVEL.error: "1", + constants.GLOBAL_LOG_LEVEL.warn: "2", + constants.GLOBAL_LOG_LEVEL.info: "3", + constants.GLOBAL_LOG_LEVEL.debug: "4", + constants.GLOBAL_LOG_LEVEL.trace: "5", +} + +BUILDER_IMAGE_STR = "builder" +SUAVE_ENABLED_GETH_IMAGE_STR = "suave" + + +def launch( + plan, + launcher, + service_name, + image, + existing_el_clients, +): + # log_level = input_parser.get_client_log_level_or_default( + # participant_log_level, global_log_level, VERBOSITY_LEVELS + # ) + + network_name = shared_utils.get_network_name(launcher.network) + + # el_min_cpu = int(el_min_cpu) if int(el_min_cpu) > 0 else EXECUTION_MIN_CPU + # el_max_cpu = ( + # int(el_max_cpu) + # if int(el_max_cpu) > 0 + # else constants.RAM_CPU_OVERRIDES[network_name]["geth_max_cpu"] + # ) + # el_min_mem = int(el_min_mem) if int(el_min_mem) > 0 else EXECUTION_MIN_MEMORY + # el_max_mem = ( + # int(el_max_mem) + # if int(el_max_mem) > 0 + # else constants.RAM_CPU_OVERRIDES[network_name]["geth_max_mem"] + # ) + + # el_volume_size = ( + # el_volume_size + # if int(el_volume_size) > 0 + # else constants.VOLUME_SIZE[network_name]["geth_volume_size"] + # ) + + # cl_client_name = service_name.split("-")[3] + + config = get_config( + plan, + launcher.el_cl_genesis_data, + launcher.jwt_file, + launcher.network, + launcher.network_id, + image, + service_name, + existing_el_clients, + ) + + service = plan.add_service(service_name, config) + + enode, enr = el_admin_node_info.get_enode_enr_for_node( + plan, service_name, RPC_PORT_ID + ) + + metrics_url = "{0}:{1}".format(service.ip_address, METRICS_PORT_NUM) + geth_metrics_info = node_metrics.new_node_metrics_info( + service_name, METRICS_PATH, metrics_url + ) + + http_url = "http://{0}:{1}".format(service.ip_address, RPC_PORT_NUM) + + return el_context.new_el_context( + "geth", + enr, + enode, + service.ip_address, + RPC_PORT_NUM, + WS_PORT_NUM, + ENGINE_RPC_PORT_NUM, + http_url, + service_name, + [geth_metrics_info], + ) + + +def get_config( + plan, + el_cl_genesis_data, + jwt_file, + network, + network_id, + image, + service_name, + existing_el_clients, +): + init_datadir_cmd_str = "geth init --datadir={0} {1}".format( + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json", + ) + + public_ports = {} + discovery_port = DISCOVERY_PORT_NUM + used_ports = get_used_ports(discovery_port) + + cmd = [ + "geth", + # Disable path based storage scheme for electra fork and verkle + # TODO: REMOVE Once geth default db is path based, and builder rebased + "--networkid={0}".format(network_id), + # "--verbosity=" + verbosity_level, + "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, + "--gcmode=archive", + "--http", + "--http.addr=0.0.0.0", + "--http.vhosts=*", + "--http.corsdomain=*", + # WARNING: The admin info endpoint is enabled so that we can easily get ENR/enode, which means + # that users should NOT store private information in these Kurtosis nodes! + "--http.api=admin,engine,net,eth,web3,debug", + "--ws", + "--ws.addr=0.0.0.0", + "--ws.port={0}".format(WS_PORT_NUM), + "--ws.api=admin,engine,net,eth,web3,debug", + "--ws.origins=*", + "--allow-insecure-unlock", + "--authrpc.port={0}".format(ENGINE_RPC_PORT_NUM), + "--authrpc.addr=0.0.0.0", + "--authrpc.vhosts=*", + "--authrpc.jwtsecret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, + "--syncmode=full", + "--rpc.allow-unprotected-txs", + "--metrics", + "--metrics.addr=0.0.0.0", + "--metrics.port={0}".format(METRICS_PORT_NUM), + "--discovery.port={0}".format(discovery_port), + "--port={0}".format(discovery_port), + "--rollup.disabletxpoolgossip=true", + ] + + if len(existing_el_clients) > 0: + cmd.append( + "--bootnodes=" + + ",".join( + [ + ctx.enode + for ctx in existing_el_clients[: constants.MAX_ENODE_ENTRIES] + ] + ) + ) + + # if len(extra_params) > 0: + # # this is a repeated, we convert it into Starlark + # cmd.extend([param for param in extra_params]) + + cmd_str = " ".join(cmd) + if network not in constants.PUBLIC_NETWORKS: + subcommand_strs = [ + init_datadir_cmd_str, + cmd_str, + ] + command_str = " && ".join(subcommand_strs) + else: + command_str = cmd_str + + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data, + constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, + } + # if persistent: + # files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + # persistent_key="data-{0}".format(service_name), + # size=el_volume_size, + # ) + return ServiceConfig( + image=image, + ports=used_ports, + cmd=[command_str], + files=files, + entrypoint=ENTRYPOINT_ARGS, + ) + + +def new_op_geth_launcher( + el_cl_genesis_data, + jwt_file, + network, + network_id, +): + return struct( + el_cl_genesis_data=el_cl_genesis_data, + jwt_file=jwt_file, + network=network, + network_id=network_id, + ) diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index b035b0b..4f467fa 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -40,6 +40,8 @@ def input_parser(plan, input_args): ], network_params=struct( network=result["network_params"]["network"], + network_id=result["network_params"]["network_id"], + seconds_per_slot=result["network_params"]["seconds_per_slot"], ), ) @@ -93,6 +95,8 @@ def default_input_args(input_args): def default_network_params(): return { "network": "kurtosis", + "network_id": "2151908", + "seconds_per_slot": 2, } diff --git a/src/participant.star b/src/participant.star new file mode 100644 index 0000000..1716f82 --- /dev/null +++ b/src/participant.star @@ -0,0 +1,12 @@ +def new_participant( + el_type, + # cl_type, + el_context, + # cl_context, +): + return struct( + el_type=el_type, + # cl_type=cl_type, + el_context=el_context, + # cl_context=cl_context, + ) diff --git a/src/participant_network.star b/src/participant_network.star new file mode 100644 index 0000000..5026878 --- /dev/null +++ b/src/participant_network.star @@ -0,0 +1,45 @@ +el_client_launcher = import_module("./el/el_launcher.star") +# cl_client_launcher = import_module("./cl/cl_launcher.star") +participant_module = import_module("./participant.star") + + +def launch_participant_network( + plan, + participants, + jwt_file, + network_params, + el_cl_data, +): + num_participants = len(participants) + # Launch all execution layer clients + all_el_contexts = el_client_launcher.launch( + plan, + jwt_file, + network_params, + el_cl_data, + participants, + num_participants, + ) + + # all_cl_contexts = cl_client_launcher.launch( + # plan, + # jwt_file, + # ) + all_participants = [] + for index, participant in enumerate(participants): + el_type = participant.el_type + # cl_type = participant.cl_type + + el_context = all_el_contexts[index] + # cl_context = all_cl_contexts[index] + + participant_entry = participant_module.new_participant( + el_type, + # cl_type, + el_context, + # cl_context, + ) + + all_participants.append(participant_entry) + + return all_participants diff --git a/static_files/jwt/jwtsecret b/static_files/jwt/jwtsecret new file mode 100644 index 0000000..8b60cac --- /dev/null +++ b/static_files/jwt/jwtsecret @@ -0,0 +1 @@ +0xdc49981516e8e72b401a63e6405495a32dafc3939b5d6d83cc319ac0388bca1b \ No newline at end of file