diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index 1470a525e..21cb61d3e 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "827ad6c500961f9bae4b5548a1bc3f6f801842de52bd471142c5743546a42cdd", + "checksum": "8d88dc93eade41b932c3475a04b3c57f0309791f0a2146dffe9869567ae774b3", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -9794,6 +9794,10 @@ "id": "futures-util 0.3.30", "target": "futures_util" }, + { + "id": "human_bytes 0.4.3", + "target": "human_bytes" + }, { "id": "humantime 2.1.0", "target": "humantime" @@ -13936,6 +13940,43 @@ }, "license": "MIT OR Apache-2.0" }, + "human_bytes 0.4.3": { + "name": "human_bytes", + "version": "0.4.3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/human_bytes/0.4.3/download", + "sha256": "91f255a4535024abf7640cb288260811fc14794f62b063652ed349f9a6c2348e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "human_bytes", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "human_bytes", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "si-units" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.4.3" + }, + "license": "BSD-2-Clause" + }, "humantime 2.1.0": { "name": "humantime", "version": "2.1.0", @@ -15847,6 +15888,10 @@ "id": "ic-sys 0.9.0", "target": "ic_sys" }, + { + "id": "ic-transport-types 0.37.1", + "target": "ic_transport_types" + }, { "id": "ic-utils 0.37.0", "target": "ic_utils" diff --git a/Cargo.lock b/Cargo.lock index a1cb41113..5676f67de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1956,6 +1956,7 @@ dependencies = [ "fs-err", "futures", "futures-util", + "human_bytes", "humantime", "ic-base-types", "ic-canister-client", @@ -2776,6 +2777,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "human_bytes" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f255a4535024abf7640cb288260811fc14794f62b063652ed349f9a6c2348e" + [[package]] name = "humantime" version = "2.1.0" @@ -3175,6 +3182,7 @@ dependencies = [ "ic-registry-transport", "ic-sns-wasm", "ic-sys", + "ic-transport-types", "ic-utils 0.37.0", "log", "pkcs11", diff --git a/Cargo.toml b/Cargo.toml index 88a06ec85..3b7efe3f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,6 +142,7 @@ ic-nervous-system-root = { git = "https://github.com/dfinity/ic.git", rev = "7de ic-nervous-system-clients = { git = "https://github.com/dfinity/ic.git", rev = "7dee90107a88b836fc72e78993913988f4f73ca2" } ic-sns-wasm = { git = "https://github.com/dfinity/ic.git", rev = "7dee90107a88b836fc72e78993913988f4f73ca2" } cycles-minting-canister = { git = "https://github.com/dfinity/ic.git", rev = "7dee90107a88b836fc72e78993913988f4f73ca2" } +ic-transport-types = "0.37.1" ic-utils = "0.37.0" include_dir = "0.7.4" itertools = "0.13.0" @@ -199,6 +200,7 @@ url = "2.5.2" urlencoding = "2.1.3" warp = "0.3" wiremock = "0.6.0" +human_bytes = "0.4" # dre-canisters dependencies ic-cdk-timers = { git = "https://github.com/dfinity/cdk-rs.git", rev = "59795716487fbb8a9910ac503bcea1e0cb08c932" } diff --git a/rs/cli/Cargo.toml b/rs/cli/Cargo.toml index de3044fb1..b0e4ea937 100644 --- a/rs/cli/Cargo.toml +++ b/rs/cli/Cargo.toml @@ -74,6 +74,7 @@ clap_complete = "4.5.8" cryptoki = { workspace = true } keyring = { workspace = true } comfy-table = { workspace = true } +human_bytes = { workspace = true } [dev-dependencies] actix-rt = { workspace = true } diff --git a/rs/cli/src/commands/mod.rs b/rs/cli/src/commands/mod.rs index 3bb1c4540..2c9bfe29b 100644 --- a/rs/cli/src/commands/mod.rs +++ b/rs/cli/src/commands/mod.rs @@ -16,6 +16,7 @@ use proposals::Proposals; use propose::Propose; use qualify::QualifyCmd; use registry::Registry; +use update_authorized_subnets::UpdateAuthorizedSubnets; use update_unassigned_nodes::UpdateUnassignedNodes; use upgrade::Upgrade; use url::Url; @@ -38,6 +39,7 @@ mod propose; pub mod qualify; mod registry; mod subnet; +mod update_authorized_subnets; mod update_unassigned_nodes; pub mod upgrade; mod version; @@ -163,6 +165,9 @@ pub enum Subcommands { /// Qualification Qualify(QualifyCmd), + + /// Manage authorized subnets + UpdateAuthorizedSubnets(UpdateAuthorizedSubnets), } pub trait ExecutableCommand { @@ -269,6 +274,7 @@ impl ExecutableCommand for Args { Subcommands::Completions(c) => c.require_ic_admin(), Subcommands::Qualify(c) => c.require_ic_admin(), Subcommands::NodeMetrics(c) => c.require_ic_admin(), + Subcommands::UpdateAuthorizedSubnets(c) => c.require_ic_admin(), } } @@ -292,6 +298,7 @@ impl ExecutableCommand for Args { Subcommands::Completions(c) => c.execute(ctx).await, Subcommands::Qualify(c) => c.execute(ctx).await, Subcommands::NodeMetrics(c) => c.execute(ctx).await, + Subcommands::UpdateAuthorizedSubnets(c) => c.execute(ctx).await, } } @@ -315,6 +322,7 @@ impl ExecutableCommand for Args { Subcommands::Completions(c) => c.validate(cmd), Subcommands::Qualify(c) => c.validate(cmd), Subcommands::NodeMetrics(c) => c.validate(cmd), + Subcommands::UpdateAuthorizedSubnets(c) => c.validate(cmd), } } } diff --git a/rs/cli/src/commands/update_authorized_subnets.rs b/rs/cli/src/commands/update_authorized_subnets.rs new file mode 100644 index 000000000..bcf1bc79a --- /dev/null +++ b/rs/cli/src/commands/update_authorized_subnets.rs @@ -0,0 +1,155 @@ +use std::{ + collections::BTreeMap, + fs::File, + io::{BufRead, BufReader}, + path::PathBuf, + sync::Arc, +}; + +use clap::{error::ErrorKind, Args}; +use ic_management_types::Subnet; +use ic_registry_subnet_type::SubnetType; +use ic_types::PrincipalId; +use itertools::Itertools; +use log::info; + +use crate::ic_admin::{ProposeCommand, ProposeOptions}; + +use super::ExecutableCommand; + +const DEFAULT_CANISTER_LIMIT: u64 = 60_000; +const DEFAULT_STATE_SIZE_BYTES_LIMIT: u64 = 322_122_547_200; // 300GB + +#[derive(Args, Debug)] +pub struct UpdateAuthorizedSubnets { + /// Path to csv file containing the blacklist. + #[clap(default_value = "./facts-db/non_public_subnets.csv")] + path: PathBuf, + + /// Canister num limit for marking a subnet as non public + #[clap(default_value_t = DEFAULT_CANISTER_LIMIT)] + canister_limit: u64, + + /// Size limit for marking a subnet as non public in bytes + #[clap(default_value_t = DEFAULT_STATE_SIZE_BYTES_LIMIT)] + state_size_limit: u64, +} + +impl ExecutableCommand for UpdateAuthorizedSubnets { + fn require_ic_admin(&self) -> super::IcAdminRequirement { + super::IcAdminRequirement::Detect + } + + fn validate(&self, cmd: &mut clap::Command) { + if !self.path.exists() { + cmd.error(ErrorKind::InvalidValue, format!("Path `{}` not found", self.path.display())) + .exit(); + } + + if !self.path.is_file() { + cmd.error( + ErrorKind::InvalidValue, + format!("Path `{}` found, but is not a file", self.path.display()), + ); + } + } + + async fn execute(&self, ctx: crate::ctx::DreContext) -> anyhow::Result<()> { + let csv_contents = self.parse_csv()?; + info!("Found following elements: {:?}", csv_contents); + + let registry = ctx.registry().await; + let subnets = registry.subnets().await?; + let mut excluded_subnets = BTreeMap::new(); + + let human_bytes = human_bytes::human_bytes(self.state_size_limit as f64); + let agent = ctx.create_ic_agent_canister_client(None)?; + + for subnet in subnets.values() { + if subnet.subnet_type.eq(&SubnetType::System) { + excluded_subnets.insert(subnet.principal, "System subnet".to_string()); + continue; + } + + let subnet_principal_string = subnet.principal.to_string(); + if let Some((_, description)) = csv_contents.iter().find(|(short_id, _)| subnet_principal_string.starts_with(short_id)) { + excluded_subnets.insert(subnet.principal, description.to_owned()); + continue; + } + + let subnet_metrics = agent.read_state_subnet_metrics(&subnet.principal).await?; + + if subnet_metrics.num_canisters >= self.canister_limit { + excluded_subnets.insert(subnet.principal, format!("Subnet has more than {} canisters", self.canister_limit)); + continue; + } + + if subnet_metrics.canister_state_bytes >= self.state_size_limit { + excluded_subnets.insert(subnet.principal, format!("Subnet has more than {} state size", human_bytes)); + } + } + + let summary = construct_summary(&subnets, &excluded_subnets)?; + + let authorized = subnets + .keys() + .filter(|subnet_id| !excluded_subnets.contains_key(subnet_id)) + .cloned() + .collect(); + + let ic_admin = ctx.ic_admin(); + ic_admin + .propose_run( + ProposeCommand::SetAuthorizedSubnetworks { subnets: authorized }, + ProposeOptions { + title: Some("Update list of public subnets".to_string()), + summary: Some(summary), + motivation: None, + }, + ) + .await?; + + Ok(()) + } +} + +impl UpdateAuthorizedSubnets { + fn parse_csv(&self) -> anyhow::Result> { + let contents = BufReader::new(File::open(&self.path)?); + let mut ret = vec![]; + for line in contents.lines() { + let content = line?; + if content.starts_with("subnet id") { + info!("Skipping header line in csv"); + continue; + } + + let (id, desc) = content.split_once(',').ok_or(anyhow::anyhow!("Failed to parse line: {}", content))?; + ret.push((id.to_string(), desc.to_string())) + } + + Ok(ret) + } +} + +fn construct_summary(subnets: &Arc>, excluded_subnets: &BTreeMap) -> anyhow::Result { + Ok(format!( + "Updating the list of authorized subnets to: + +| Subnet id | Public | Description | +| --------- | ------ | ----------- | +{}", + subnets + .values() + .map(|s| { + let excluded_desc = excluded_subnets.get(&s.principal); + format!( + "| {} | {} | {} |", + s.principal, + excluded_desc.is_none(), + excluded_desc.map(|s| s.to_string()).unwrap_or_default() + ) + }) + .join("\n") + )) +} diff --git a/rs/cli/src/ic_admin.rs b/rs/cli/src/ic_admin.rs index 1d6c54766..5c4d835c3 100644 --- a/rs/cli/src/ic_admin.rs +++ b/rs/cli/src/ic_admin.rs @@ -655,6 +655,9 @@ pub enum ProposeCommand { nodes: Vec, version: String, }, + SetAuthorizedSubnetworks { + subnets: Vec, + }, } impl ProposeCommand { @@ -742,6 +745,7 @@ impl ProposeCommand { vec!["--version".to_string(), version.to_string()], ] .concat(), + Self::SetAuthorizedSubnetworks { subnets } => subnets.iter().flat_map(|s| ["--subnets".to_string(), s.to_string()]).collect::>(), } } } diff --git a/rs/ic-canisters/Cargo.toml b/rs/ic-canisters/Cargo.toml index b0f110637..3fa7e85a4 100644 --- a/rs/ic-canisters/Cargo.toml +++ b/rs/ic-canisters/Cargo.toml @@ -34,3 +34,4 @@ thiserror = { workspace = true } url = { workspace = true } ic-sns-wasm = { workspace = true } trustworthy-node-metrics = { workspace = true } +ic-transport-types = { workspace = true } diff --git a/rs/ic-canisters/src/lib.rs b/rs/ic-canisters/src/lib.rs index c15f20018..7196aba4c 100644 --- a/rs/ic-canisters/src/lib.rs +++ b/rs/ic-canisters/src/lib.rs @@ -6,10 +6,12 @@ use ic_agent::identity::Secp256k1Identity; use ic_agent::Agent; use ic_agent::Identity; use ic_base_types::CanisterId; +use ic_base_types::PrincipalId; use ic_canister_client::Agent as CanisterClientAgent; use ic_canister_client::Sender; use ic_canister_client_sender::SigKeys; use ic_sys::utility_command::UtilityCommand; +use ic_transport_types::SubnetMetrics; use parallel_hardware_identity::ParallelHardwareIdentity; use serde::Deserialize; use std::path::PathBuf; @@ -98,6 +100,13 @@ impl IcAgentCanisterClient { .build()?; Ok(Self { agent }) } + + pub async fn read_state_subnet_metrics(&self, subnet_id: &PrincipalId) -> anyhow::Result { + self.agent + .read_state_subnet_metrics(candid::Principal::from_str(subnet_id.to_string().as_str())?) + .await + .map_err(|e| anyhow::anyhow!(e)) + } } #[derive(Clone, CandidType, Deserialize, Debug)]