Skip to content

builder events module #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 1, 2024
Merged

builder events module #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
997e11e
add builder events module
ltitanb Aug 1, 2024
3aa8dc8
clippy
ltitanb Aug 1, 2024
43ced3d
comments
ltitanb Aug 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ members = [
"crates/metrics",

"tests",
"examples/da_commit",
"examples/*",
]
resolver = "2"

Expand Down
3 changes: 2 additions & 1 deletion bin/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ pub mod prelude {
pub use cb_common::{
commit,
commit::request::SignRequest,
config::{load_module_config, StartModuleConfig},
config::{load_builder_module_config, load_commit_module_config, StartCommitModuleConfig},
pbs::{BuilderEvent, BuilderEventClient, OnBuilderApiEvent},
utils::{initialize_tracing_log, utcnow_ms, utcnow_ns, utcnow_sec, utcnow_us},
};
pub use cb_metrics::provider::MetricsProvider;
Expand Down
6 changes: 6 additions & 0 deletions config.example.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,11 @@ use_grafana = true

[[modules]]
id = "DA_COMMIT"
type = "commit"
docker_image = "test_da_commit"
sleep_secs = 5

[[modules]]
id = "BUILDER_LOG"
type = "events"
docker_image = "test_builder_log"
218 changes: 135 additions & 83 deletions crates/cli/src/docker_init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ use std::{path::Path, vec};

use cb_common::{
config::{
CommitBoostConfig, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV, METRICS_SERVER_ENV,
MODULE_ID_ENV, MODULE_JWT_ENV, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS,
SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_SERVER_ENV,
CommitBoostConfig, ModuleKind, BUILDER_SERVER_ENV, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV,
METRICS_SERVER_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV,
SIGNER_DIR_SECRETS, SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV,
SIGNER_SERVER_ENV,
},
loader::SignerLoader,
utils::random_jwt,
Expand Down Expand Up @@ -51,17 +52,101 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()>
let signer_port = 20000;
let signer_server = format!("cb_signer:{signer_port}");

let builder_events_port = 30000;
let mut builder_events_modules = Vec::new();

// setup pbs service
targets.push(PrometheusTargetConfig {
targets: vec![format!("cb_pbs:{metrics_port}")],
labels: PrometheusLabelsConfig { job: "pbs".to_owned() },
});

let pbs_envs = IndexMap::from([
let mut pbs_envs = IndexMap::from([
get_env_same(CB_CONFIG_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
]);

let mut needs_signer_module = cb_config.pbs.with_signer;

// setup modules
if let Some(modules_config) = cb_config.modules {
for module in modules_config {
// TODO: support modules volumes and network
let module_cid = format!("cb_{}", module.id.to_lowercase());

targets.push(PrometheusTargetConfig {
targets: vec![format!("{module_cid}:{metrics_port}")],
labels: PrometheusLabelsConfig { job: module_cid.clone() },
});

let module_service = match module.kind {
// a commit module needs a JWT and access to the signer network
ModuleKind::Commit => {
needs_signer_module = true;

let jwt = random_jwt();
let jwt_name = format!("CB_JWT_{}", module.id.to_uppercase());

// module ids are assumed unique, so envs dont override each other
let module_envs = IndexMap::from([
get_env_val(MODULE_ID_ENV, &module.id),
get_env_same(CB_CONFIG_ENV),
get_env_interp(MODULE_JWT_ENV, &jwt_name),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(SIGNER_SERVER_ENV, &signer_server),
]);

envs.insert(jwt_name.clone(), jwt.clone());
jwts.insert(module.id.clone(), jwt);

Service {
container_name: Some(module_cid.clone()),
image: Some(module.docker_image),
// TODO: allow service to open ports here
networks: Networks::Simple(vec![
METRICS_NETWORK.to_owned(),
SIGNER_NETWORK.to_owned(),
]),
volumes: vec![config_volume.clone()],
environment: Environment::KvPair(module_envs),
depends_on: DependsOnOptions::Simple(vec!["cb_signer".to_owned()]),
..Service::default()
}
}
// an event module just needs a port to listen on
ModuleKind::Events => {
// module ids are assumed unique, so envs dont override each other
let module_envs = IndexMap::from([
get_env_val(MODULE_ID_ENV, &module.id),
get_env_same(CB_CONFIG_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(BUILDER_SERVER_ENV, &builder_events_port.to_string()),
]);

builder_events_modules.push(format!("{module_cid}:{builder_events_port}"));

Service {
container_name: Some(module_cid.clone()),
image: Some(module.docker_image),
networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]),
volumes: vec![config_volume.clone()],
environment: Environment::KvPair(module_envs),
depends_on: DependsOnOptions::Simple(vec!["cb_pbs".to_owned()]),
..Service::default()
}
}
};

services.insert(module_cid, Some(module_service));
}
};

if !builder_events_modules.is_empty() {
let env = builder_events_modules.join(",");
let (k, v) = get_env_val(BUILDER_SERVER_ENV, &env);
pbs_envs.insert(k, v);
}

let pbs_service = Service {
container_name: Some("cb_pbs".to_owned()),
image: Some(cb_config.pbs.docker_image),
Expand All @@ -77,102 +162,68 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()>

services.insert("cb_pbs".to_owned(), Some(pbs_service));

// setup modules
if let Some(modules_config) = cb_config.modules {
for module in modules_config {
// TODO: support modules volumes and network
let module_cid = format!("cb_{}", module.id.to_lowercase());
// TODO: validate if we have signer modules but not signer config

// setup signer service

if let Some(signer_config) = cb_config.signer {
if needs_signer_module {
let mut volumes = vec![config_volume.clone()];

targets.push(PrometheusTargetConfig {
targets: vec![format!("{module_cid}:{metrics_port}")],
labels: PrometheusLabelsConfig { job: module_cid.clone() },
targets: vec![format!("cb_signer:{metrics_port}")],
labels: PrometheusLabelsConfig { job: "signer".into() },
});

let jwt = random_jwt();
let jwt_name = format!("CB_JWT_{}", module.id.to_uppercase());

// module ids are assumed unique, so envs dont override each other
let module_envs = IndexMap::from([
get_env_val(MODULE_ID_ENV, &module.id),
let mut signer_envs = IndexMap::from([
get_env_same(CB_CONFIG_ENV),
get_env_interp(MODULE_JWT_ENV, &jwt_name),
get_env_same(JWTS_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(SIGNER_SERVER_ENV, &signer_server),
get_env_val(SIGNER_SERVER_ENV, &signer_port.to_string()),
]);

envs.insert(jwt_name.clone(), jwt.clone());
jwts.insert(module.id.clone(), jwt);
// TODO: generalize this, different loaders may not need volumes but eg ports
match signer_config.loader {
SignerLoader::File { key_path } => {
volumes.push(Volumes::Simple(format!("./{}:{}:ro", key_path, SIGNER_KEYS)));
let (k, v) = get_env_val(SIGNER_KEYS_ENV, SIGNER_KEYS);
signer_envs.insert(k, v);
}
SignerLoader::ValidatorsDir { keys_path, secrets_path } => {
volumes.push(Volumes::Simple(format!("{}:{}:ro", keys_path, SIGNER_DIR_KEYS)));
let (k, v) = get_env_val(SIGNER_DIR_KEYS_ENV, SIGNER_DIR_KEYS);
signer_envs.insert(k, v);

volumes.push(Volumes::Simple(format!(
"{}:{}:ro",
secrets_path, SIGNER_DIR_SECRETS
)));
let (k, v) = get_env_val(SIGNER_DIR_SECRETS_ENV, SIGNER_DIR_SECRETS);
signer_envs.insert(k, v);
}
};

// write jwts to env
let jwts_json = serde_json::to_string(&jwts).unwrap().clone();
envs.insert(JWTS_ENV.into(), format!("{jwts_json:?}"));

let module_service = Service {
container_name: Some(module_cid.clone()),
image: Some(module.docker_image),
// TODO: allow service to open ports here
let signer_service = Service {
container_name: Some("cb_signer".to_owned()),
image: Some(signer_config.docker_image),
networks: Networks::Simple(vec![
METRICS_NETWORK.to_owned(),
SIGNER_NETWORK.to_owned(),
]),
volumes: vec![config_volume.clone()],
environment: Environment::KvPair(module_envs),
depends_on: DependsOnOptions::Simple(vec!["cb_signer".to_owned()]),
volumes,
environment: Environment::KvPair(signer_envs),
..Service::default()
};

services.insert(module_cid, Some(module_service));
services.insert("cb_signer".to_owned(), Some(signer_service));
}
};

// TODO: validate if we have signer modules but not signer config

// setup signer service
if let Some(signer_config) = cb_config.signer {
let mut volumes = vec![config_volume.clone()];

targets.push(PrometheusTargetConfig {
targets: vec![format!("cb_signer:{metrics_port}")],
labels: PrometheusLabelsConfig { job: "signer".into() },
});

let mut signer_envs = IndexMap::from([
get_env_same(CB_CONFIG_ENV),
get_env_same(JWTS_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(SIGNER_SERVER_ENV, &signer_port.to_string()),
]);

// TODO: generalize this, different loaders may not need volumes but eg ports
match signer_config.loader {
SignerLoader::File { key_path } => {
volumes.push(Volumes::Simple(format!("./{}:{}:ro", key_path, SIGNER_KEYS)));
let (k, v) = get_env_val(SIGNER_KEYS_ENV, SIGNER_KEYS);
signer_envs.insert(k, v);
}
SignerLoader::ValidatorsDir { keys_path, secrets_path } => {
volumes.push(Volumes::Simple(format!("{}:{}:ro", keys_path, SIGNER_DIR_KEYS)));
let (k, v) = get_env_val(SIGNER_DIR_KEYS_ENV, SIGNER_DIR_KEYS);
signer_envs.insert(k, v);

volumes
.push(Volumes::Simple(format!("{}:{}:ro", secrets_path, SIGNER_DIR_SECRETS)));
let (k, v) = get_env_val(SIGNER_DIR_SECRETS_ENV, SIGNER_DIR_SECRETS);
signer_envs.insert(k, v);
}
};

// write jwts to env
let jwts_json = serde_json::to_string(&jwts)?.clone();
envs.insert(JWTS_ENV.into(), format!("{jwts_json:?}"));

let signer_service = Service {
container_name: Some("cb_signer".to_owned()),
image: Some(signer_config.docker_image),
networks: Networks::Simple(vec![METRICS_NETWORK.to_owned(), SIGNER_NETWORK.to_owned()]),
volumes,
environment: Environment::KvPair(signer_envs),
..Service::default()
};

services.insert("cb_signer".to_owned(), Some(signer_service));
};
} else if needs_signer_module {
panic!("Signer module required but no signer config provided");
}

// setup metrics services
// TODO: make this metrics optional?
Expand Down Expand Up @@ -274,6 +325,7 @@ fn get_env_interp(k: &str, v: &str) -> (String, Option<SingleValue>) {
get_env_val(k, &format!("${{{v}}}"))
}

// FOO=bar
fn get_env_val(k: &str, v: &str) -> (String, Option<SingleValue>) {
(k.into(), Some(SingleValue::String(v.into())))
}
Expand Down
7 changes: 7 additions & 0 deletions crates/common/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,18 @@ rust-version.workspace = true
# ethereum
alloy = { workspace = true, features = ["ssz"] }
ethereum_ssz.workspace = true
ssz_types.workspace = true
ethereum_ssz_derive.workspace = true
ethereum_serde_utils.workspace = true
ethereum-types.workspace = true

# networking
axum.workspace = true
reqwest.workspace = true

# async / threads
tokio.workspace = true

# serialization
toml.workspace = true
serde.workspace = true
Expand Down
Loading