feat: subnet authorization with dre

Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "827ad6c500961f9bae4b5548a1bc3f6f801842de52bd471142c5743546a42cdd",
+  "checksum": "8d88dc93eade41b932c3475a04b3c57f0309791f0a2146dffe9869567ae774b3",
   "crates": {
     "actix-codec 0.5.2": {
       "name": "actix-codec",
@@ -9794,6 +9794,10 @@
               "id": "futures-util 0.3.30",
               "target": "futures_util"
             },
+            {
+              "id": "human_bytes 0.4.3",
+              "target": "human_bytes"
+            },
             {
               "id": "humantime 2.1.0",
               "target": "humantime"
@@ -13936,6 +13940,43 @@
       },
       "license": "MIT OR Apache-2.0"
     },
+    "human_bytes 0.4.3": {
+      "name": "human_bytes",
+      "version": "0.4.3",
+      "repository": {
+        "Http": {
+          "url": "https://static.crates.io/crates/human_bytes/0.4.3/download",
+          "sha256": "91f255a4535024abf7640cb288260811fc14794f62b063652ed349f9a6c2348e"
+        }
+      },
+      "targets": [
+        {
+          "Library": {
+            "crate_name": "human_bytes",
+            "crate_root": "src/lib.rs",
+            "srcs": [
+              "**/*.rs"
+            ]
+          }
+        }
+      ],
+      "library_target_name": "human_bytes",
+      "common_attrs": {
+        "compile_data_glob": [
+          "**"
+        ],
+        "crate_features": {
+          "common": [
+            "default",
+            "si-units"
+          ],
+          "selects": {}
+        },
+        "edition": "2018",
+        "version": "0.4.3"
+      },
+      "license": "BSD-2-Clause"
+    },
     "humantime 2.1.0": {
       "name": "humantime",
       "version": "2.1.0",
@@ -15847,6 +15888,10 @@
               "id": "ic-sys 0.9.0",
               "target": "ic_sys"
             },
+            {
+              "id": "ic-transport-types 0.37.1",
+              "target": "ic_transport_types"
+            },
             {
               "id": "ic-utils 0.37.0",
               "target": "ic_utils"

Cargo.toml

@@ -142,6 +142,7 @@ ic-nervous-system-root = { git = "https://github.com/dfinity/ic.git", rev = "7de
 ic-nervous-system-clients = { git = "https://github.com/dfinity/ic.git", rev = "7dee90107a88b836fc72e78993913988f4f73ca2" }
 ic-sns-wasm = { git = "https://github.com/dfinity/ic.git", rev = "7dee90107a88b836fc72e78993913988f4f73ca2" }
 cycles-minting-canister = { git = "https://github.com/dfinity/ic.git", rev = "7dee90107a88b836fc72e78993913988f4f73ca2" }
+ic-transport-types = "0.37.1"
 ic-utils = "0.37.0"
 include_dir = "0.7.4"
 itertools = "0.13.0"
@@ -199,6 +200,7 @@ url = "2.5.2"
 urlencoding = "2.1.3"
 warp = "0.3"
 wiremock = "0.6.0"
+human_bytes = "0.4"
 
 # dre-canisters dependencies
 ic-cdk-timers = { git = "https://github.com/dfinity/cdk-rs.git", rev = "59795716487fbb8a9910ac503bcea1e0cb08c932" }

rs/cli/Cargo.toml

@@ -74,6 +74,7 @@ clap_complete = "4.5.8"
 cryptoki = { workspace = true }
 keyring = { workspace = true }
 comfy-table = { workspace = true }
+human_bytes = { workspace = true }
 
 [dev-dependencies]
 actix-rt = { workspace = true }

rs/cli/src/commands/mod.rs

@@ -16,6 +16,7 @@ use proposals::Proposals;
 use propose::Propose;
 use qualify::QualifyCmd;
 use registry::Registry;
+use update_authorized_subnets::UpdateAuthorizedSubnets;
 use update_unassigned_nodes::UpdateUnassignedNodes;
 use upgrade::Upgrade;
 use url::Url;
@@ -38,6 +39,7 @@ mod propose;
 pub mod qualify;
 mod registry;
 mod subnet;
+mod update_authorized_subnets;
 mod update_unassigned_nodes;
 pub mod upgrade;
 mod version;
@@ -163,6 +165,9 @@ pub enum Subcommands {
 
     /// Qualification
     Qualify(QualifyCmd),
+
+    /// Manage authorized subnets
+    UpdateAuthorizedSubnets(UpdateAuthorizedSubnets),
 }
 
 pub trait ExecutableCommand {
@@ -269,6 +274,7 @@ impl ExecutableCommand for Args {
             Subcommands::Completions(c) => c.require_ic_admin(),
             Subcommands::Qualify(c) => c.require_ic_admin(),
             Subcommands::NodeMetrics(c) => c.require_ic_admin(),
+            Subcommands::UpdateAuthorizedSubnets(c) => c.require_ic_admin(),
         }
     }
 
@@ -292,6 +298,7 @@ impl ExecutableCommand for Args {
             Subcommands::Completions(c) => c.execute(ctx).await,
             Subcommands::Qualify(c) => c.execute(ctx).await,
             Subcommands::NodeMetrics(c) => c.execute(ctx).await,
+            Subcommands::UpdateAuthorizedSubnets(c) => c.execute(ctx).await,
         }
     }
 
@@ -315,6 +322,7 @@ impl ExecutableCommand for Args {
             Subcommands::Completions(c) => c.validate(cmd),
             Subcommands::Qualify(c) => c.validate(cmd),
             Subcommands::NodeMetrics(c) => c.validate(cmd),
+            Subcommands::UpdateAuthorizedSubnets(c) => c.validate(cmd),
         }
     }
 }

rs/cli/src/commands/update_authorized_subnets.rs

@@ -0,0 +1,158 @@
+use std::{
+    collections::BTreeMap,
+    fs::File,
+    io::{BufRead, BufReader},
+    path::PathBuf,
+    sync::Arc,
+};
+
+use clap::{error::ErrorKind, Args};
+use ic_management_types::Subnet;
+use ic_registry_subnet_type::SubnetType;
+use ic_types::PrincipalId;
+use itertools::Itertools;
+use log::info;
+
+use crate::ic_admin::{ProposeCommand, ProposeOptions};
+
+use super::ExecutableCommand;
+
+const DEFAULT_CANISTER_LIMIT: u64 = 60_000;
+const DEFAULT_STATE_SIZE_BYTES_LIMIT: u64 = 322_122_547_200; // 300GB
+
+#[derive(Args, Debug)]
+pub struct UpdateAuthorizedSubnets {
+    /// Path to csv file containing the blacklist.
+    #[clap(default_value = "./facts-db/non_public_subnets.csv")]
+    path: PathBuf,
+
+    /// Canister num limit for marking a subnet as non public
+    #[clap(default_value_t = DEFAULT_CANISTER_LIMIT)]
+    canister_limit: u64,
+
+    /// Size limit for marking a subnet as non public in bytes
+    #[clap(default_value_t = DEFAULT_STATE_SIZE_BYTES_LIMIT)]
+    state_size_limit: u64,
+}
+
+impl ExecutableCommand for UpdateAuthorizedSubnets {
+    fn require_ic_admin(&self) -> super::IcAdminRequirement {
+        super::IcAdminRequirement::Detect
+    }
+
+    fn validate(&self, cmd: &mut clap::Command) {
+        if !self.path.exists() {
+            cmd.error(ErrorKind::InvalidValue, format!("Path `{}` not found", self.path.display()))
+                .exit();
+        }
+
+        if !self.path.is_file() {
+            cmd.error(
+                ErrorKind::InvalidValue,
+                format!("Path `{}` found, but is not a file", self.path.display()),
+            );
+        }
+    }
+
+    async fn execute(&self, ctx: crate::ctx::DreContext) -> anyhow::Result<()> {
+        let csv_contents = self.parse_csv()?;
+        info!("Found following elements: {:?}", csv_contents);
+
+        let registry = ctx.registry().await;
+        let subnets = registry.subnets().await?;
+        let mut excluded_subnets = BTreeMap::new();
+
+        let human_bytes = human_bytes::human_bytes(self.state_size_limit as f64);
+        let agent = ctx.create_ic_agent_canister_client(None)?;
+
+        for subnet in subnets.values() {
+            if subnet.subnet_type.eq(&SubnetType::System) {
+                excluded_subnets.insert(subnet.principal.clone(), "System subnet".to_string());
+                continue;
+            }
+
+            let subnet_principal_string = subnet.principal.to_string();
+            if let Some((_, description)) = csv_contents.iter().find(|(short_id, _)| subnet_principal_string.starts_with(short_id)) {
+                excluded_subnets.insert(subnet.principal.clone(), description.to_owned());
+                continue;
+            }
+
+            let subnet_metrics = agent.read_state_subnet_metrics(&subnet.principal).await?;
+
+            if subnet_metrics.num_canisters >= self.canister_limit {
+                excluded_subnets.insert(
+                    subnet.principal.clone(),
+                    format!("Subnet has more than {} canisters", self.canister_limit),
+                );
+                continue;
+            }
+
+            if subnet_metrics.canister_state_bytes >= self.state_size_limit {
+                excluded_subnets.insert(subnet.principal.clone(), format!("Subnet has more than {} state size", human_bytes));
+            }
+        }
+
+        let summary = construct_summary(&subnets, &excluded_subnets)?;
+
+        let authorized = subnets
+            .keys()
+            .filter(|subnet_id| !excluded_subnets.contains_key(subnet_id))
+            .cloned()
+            .collect();
+
+        let ic_admin = ctx.ic_admin();
+        ic_admin
+            .propose_run(
+                ProposeCommand::SetAuthorizedSubnetworks { subnets: authorized },
+                ProposeOptions {
+                    title: Some("Update list of public subnets".to_string()),
+                    summary: Some(summary),
+                    motivation: None,
+                },
+            )
+            .await?;
+
+        Ok(())
+    }
+}
+
+impl UpdateAuthorizedSubnets {
+    fn parse_csv(&self) -> anyhow::Result<Vec<(String, String)>> {
+        let contents = BufReader::new(File::open(&self.path)?);
+        let mut ret = vec![];
+        for line in contents.lines() {
+            let content = line?;
+            if content.starts_with("subnet id") {
+                info!("Skipping header line in csv");
+                continue;
+            }
+
+            let (id, desc) = content.split_once(',').ok_or(anyhow::anyhow!("Failed to parse line: {}", content))?;
+            ret.push((id.to_string(), desc.to_string()))
+        }
+
+        Ok(ret)
+    }
+}
+
+fn construct_summary(subnets: &Arc<BTreeMap<PrincipalId, Subnet>>, excluded_subnets: &BTreeMap<PrincipalId, String>) -> anyhow::Result<String> {
+    Ok(format!(
+        "Updating the list of authorized subnets to:
+
+| Subnet id | Public | Description |
+| --------- | ------ | ----------- |
+{}",
+        subnets
+            .values()
+            .map(|s| {
+                let excluded_desc = excluded_subnets.get(&s.principal);
+                format!(
+                    "| {} | {} | {} |",
+                    s.principal.to_string(),
+                    excluded_desc.is_none(),
+                    excluded_desc.map(|s| s.to_string()).unwrap_or_default()
+                )
+            })
+            .join("\n")
+    ))
+}

rs/cli/src/ic_admin.rs

@@ -655,6 +655,9 @@ pub enum ProposeCommand {
         nodes: Vec<PrincipalId>,
         version: String,
     },
+    SetAuthorizedSubnetworks {
+        subnets: Vec<PrincipalId>,
+    },
 }
 
 impl ProposeCommand {
@@ -742,6 +745,7 @@ impl ProposeCommand {
                 vec!["--version".to_string(), version.to_string()],
             ]
             .concat(),
+            Self::SetAuthorizedSubnetworks { subnets } => subnets.iter().flat_map(|s| ["--subnets".to_string(), s.to_string()]).collect::<Vec<_>>(),
         }
     }
 }

rs/ic-canisters/Cargo.toml

@@ -34,3 +34,4 @@ thiserror = { workspace = true }
 url = { workspace = true }
 ic-sns-wasm = { workspace = true }
 trustworthy-node-metrics = { workspace = true }
+ic-transport-types = { workspace = true }

rs/ic-canisters/src/lib.rs

@@ -6,10 +6,12 @@ use ic_agent::identity::Secp256k1Identity;
 use ic_agent::Agent;
 use ic_agent::Identity;
 use ic_base_types::CanisterId;
+use ic_base_types::PrincipalId;
 use ic_canister_client::Agent as CanisterClientAgent;
 use ic_canister_client::Sender;
 use ic_canister_client_sender::SigKeys;
 use ic_sys::utility_command::UtilityCommand;
+use ic_transport_types::SubnetMetrics;
 use parallel_hardware_identity::ParallelHardwareIdentity;
 use serde::Deserialize;
 use std::path::PathBuf;
@@ -98,6 +100,13 @@ impl IcAgentCanisterClient {
             .build()?;
         Ok(Self { agent })
     }
+
+    pub async fn read_state_subnet_metrics(&self, subnet_id: &PrincipalId) -> anyhow::Result<SubnetMetrics> {
+        self.agent
+            .read_state_subnet_metrics(candid::Principal::from_str(subnet_id.to_string().as_str())?)
+            .await
+            .map_err(|e| anyhow::anyhow!(e))
+    }
 }
 
 #[derive(Clone, CandidType, Deserialize, Debug)]