feat(CI): qualifying as github job

.github/workflows/qualify.yaml

@@ -0,0 +1,47 @@
+name: Qualification
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "The version that should be qualified"
+        type: string
+        default: ""
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  qualify:
+    runs-on:
+      labels: dre-runner-custom
+    container: ghcr.io/dfinity/dre/actions-runner:7efd87b0eac3ebd255be7efe00a3b39b0f9e9fc1
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "🔍 Check if the version is set"
+        shell: bash
+        run: |
+          if [ -z "${{ github.event.inputs.version }}" ]; then
+            echo "Error: 'version' is required and cannot be empty."
+            exit 1
+          fi
+
+      - name: "☁️ Setup runner"
+        uses: ./.github/workflows/manage-runner-pre
+
+      # This should be done in runner.Dockerfile
+      - name: "Remove utsns and netns from containers.conf"
+        shell: bash
+        run: |
+          sudo rm /etc/containers/containers.conf
+
+      - name: "✨ Running qualification"
+        shell: bash
+        env:
+          MANUALY_TTL_FARM: 1
+        run: |
+          mkdir -p ~/.config/dfx/identity/xnet-testing/
+          echo "${{ secrets.XNET_PRINCIPAL_KEY }}" > ~/.config/dfx/identity/xnet-testing/identity.pem
+          bazel run //rs/qualifier -- "${{ github.event.inputs.version }}"

rs/qualifier/src/cli.rs

@@ -5,6 +5,7 @@ use clap::Parser;
 use ic_nervous_system_common_test_keys::TEST_NEURON_1_OWNER_KEYPAIR;
 use tokio::process::Command;
 const TEST_NEURON_1_IDENTITY_PATH: &str = ".config/dfx/identity/test_neuron_1/identity.pem";
+const XNET_TESTING_IDENTITY_PATH: &str = ".config/dfx/identity/xnet-testing/identity.pem";
 
 #[derive(Parser)]
 #[clap(about, version)]
@@ -38,7 +39,7 @@ pub struct Args {
 }
 
 impl Args {
-    pub fn ensure_key(&self) -> anyhow::Result<(u64, PathBuf)> {
+    pub fn ensure_test_key(&self) -> anyhow::Result<(u64, PathBuf)> {
         let key_pair = &TEST_NEURON_1_OWNER_KEYPAIR;
         let path = dirs::home_dir()
             .ok_or(anyhow::anyhow!("No home dir present"))?
@@ -52,6 +53,16 @@ impl Args {
         Ok((449479075714955186, path))
     }
 
+    pub fn ensure_xnet_test_key(&self) -> anyhow::Result<()> {
+        let path = dirs::home_dir()
+            .ok_or(anyhow::anyhow!("No home dir present"))?
+            .join(PathBuf::from_str(XNET_TESTING_IDENTITY_PATH)?);
+        match path.exists() {
+            true => Ok(()),
+            false => anyhow::bail!("Missing xnet-testing identity on path: {}", path.display()),
+        }
+    }
+
     pub async fn ensure_git(&self) -> anyhow::Result<()> {
         std::fs::create_dir_all(&self.ic_repo_path)?;
 

rs/qualifier/src/ict_util.rs

@@ -1,17 +1,24 @@
-use std::{path::PathBuf, process::Stdio, str::FromStr};
+use std::{path::PathBuf, process::Stdio, str::FromStr, time::Duration};
 
 use itertools::Itertools;
 use log::info;
+use reqwest::ClientBuilder;
 use serde_json::Value;
 use tokio::{
     io::{AsyncBufReadExt, BufReader},
-    process::{ChildStdout, Command},
+    process::Command,
     sync::mpsc::Sender,
 };
 use tokio_util::sync::CancellationToken;
 
 use crate::Message;
 
+// Each 30s set the ttl for a testnet to 90 seconds
+const FARM_GROUP_KEEPALIVE_TTL_SEC: u64 = 90;
+const KEEPALIVE_PERIOD: Duration = Duration::from_secs(30);
+const KEEPALIVE_PERIOD_ERROR: Duration = Duration::from_secs(5);
+pub const FARM_BASE_URL: &str = "https://farm.dfinity.systems";
+
 pub async fn ict(ic_git: PathBuf, config: String, token: CancellationToken, sender: Sender<Message>) -> anyhow::Result<()> {
     let ic_config = PathBuf::from_str("/tmp/ic_config.json")?;
     std::fs::write(&ic_config, &config)?;
@@ -35,19 +42,8 @@ pub async fn ict(ic_git: PathBuf, config: String, token: CancellationToken, send
         .current_dir(&ic_git)
         .spawn()?;
 
-    if let Some(taken) = child.stdout.as_mut() {
-        wait_data(taken, token.clone(), sender).await?;
-    }
-
-    token.cancelled().await;
-    info!("Received shutdown, killing testnet");
-    child.kill().await?;
-
-    Ok(())
-}
-
-async fn wait_data(stdout: &mut ChildStdout, token: CancellationToken, sender: Sender<Message>) -> anyhow::Result<()> {
-    let mut stdout_reader = BufReader::new(stdout).lines();
+    let mut stdout = child.stdout.take().ok_or(anyhow::anyhow!("Couldn't take stdout"))?;
+    let mut stdout_reader = BufReader::new(&mut stdout).lines();
 
     let target = "Testnet is being deployed, please wait ...";
     let logs;
@@ -70,6 +66,21 @@ async fn wait_data(stdout: &mut ChildStdout, token: CancellationToken, sender: S
         if token.is_cancelled() {
             return Ok(());
         }
+
+        match child.try_wait() {
+            Ok(Some(s)) => {
+                let stderr = child.stderr.take().ok_or(anyhow::anyhow!("Unable to get stderr"))?;
+                let mut lines = BufReader::new(stderr).lines();
+
+                let mut all = vec![];
+                while let Some(line) = lines.next_line().await? {
+                    all.push(line)
+                }
+
+                anyhow::bail!("Finished early with status code {:?} and error: \n{}", s, all.iter().join("\n"))
+            }
+            _ => continue,
+        }
     }
 
     sender
@@ -79,20 +90,40 @@ async fn wait_data(stdout: &mut ChildStdout, token: CancellationToken, sender: S
 
     info!("Building testnet...");
     let mut whole_config = vec![];
+    let deployment_name;
     loop {
         let line = stdout_reader.next_line().await?;
         if let Some(line) = line {
             whole_config.push(line.trim().to_string());
             let config = whole_config.iter().join("");
 
-            if serde_json::from_str::<Value>(&config).is_ok() {
+            if let Ok(v) = serde_json::from_str::<Value>(&config) {
+                deployment_name = v["farm"]["group"]
+                    .as_str()
+                    .ok_or(anyhow::anyhow!("Failed to find 'farm.group'"))?
+                    .to_string();
                 break;
             }
         }
 
         if token.is_cancelled() {
             return Ok(());
         }
+
+        match child.try_wait() {
+            Ok(Some(s)) => {
+                let stderr = child.stderr.take().ok_or(anyhow::anyhow!("Unable to get stderr"))?;
+                let mut lines = BufReader::new(stderr).lines();
+
+                let mut all = vec![];
+                while let Some(line) = lines.next_line().await? {
+                    all.push(line)
+                }
+
+                anyhow::bail!("Finished early with status code {:?} and error: \n{}", s, all.iter().join("\n"))
+            }
+            _ => continue,
+        }
     }
 
     let config = whole_config.iter().join("");
@@ -101,5 +132,39 @@ async fn wait_data(stdout: &mut ChildStdout, token: CancellationToken, sender: S
         .await
         .map_err(|_| anyhow::anyhow!("Failed to send data across channels"))?;
 
+    child.stdout = Some(stdout);
+
+    if std::env::var("MANUALY_TTL_FARM").is_ok() {
+        tokio::select! {
+            _ = keep_testnet_alive(deployment_name, token.clone()) => {},
+            _ = token.cancelled() => {}
+        }
+    } else {
+        token.cancelled().await;
+    }
+
+    info!("Received shutdown, killing testnet");
+    child.kill().await?;
+
+    Ok(())
+}
+
+async fn keep_testnet_alive(group_name: String, token: CancellationToken) -> anyhow::Result<()> {
+    let client = ClientBuilder::new().timeout(Duration::from_secs(15)).build()?;
+    let ttl_url = format!("{}/group/{}/ttl/{}", FARM_BASE_URL, &group_name, FARM_GROUP_KEEPALIVE_TTL_SEC);
+    info!("Will be using following ttl link: {}", ttl_url);
+
+    while !token.is_cancelled() {
+        let resp_future = client.put(&ttl_url).send().await;
+
+        match resp_future {
+            Ok(r) => match r.error_for_status() {
+                Ok(_) => tokio::time::sleep(KEEPALIVE_PERIOD).await,
+                _ => tokio::time::sleep(KEEPALIVE_PERIOD_ERROR).await,
+            },
+            _ => tokio::time::sleep(KEEPALIVE_PERIOD_ERROR).await,
+        }
+    }
+
     Ok(())
 }

rs/qualifier/src/main.rs

@@ -29,7 +29,7 @@ async fn main() -> anyhow::Result<()> {
     // Check if farm is reachable. If not, error
     let client = ClientBuilder::new().timeout(Duration::from_secs(30)).build()?;
     client
-        .get("https://kibana.testnet.dfinity.network")
+        .get("https://kibana.testnet.dfinity.network/")
         .send()
         .await
         .map_err(|e| anyhow::anyhow!("Checking connectivity failed: {}", e.to_string()))?
@@ -39,9 +39,10 @@ async fn main() -> anyhow::Result<()> {
     let args = Args::parse();
     info!("Running qualification for {}", args.version_to_qualify);
     info!("Generating keys for farm testnets...");
-    let (neuron_id, private_key_pem) = args.ensure_key()?;
+    let (neuron_id, private_key_pem) = args.ensure_test_key()?;
     info!("Principal key created");
 
+    args.ensure_xnet_test_key()?;
     // Take in one version and figure out what is the base version
     //
     // To find the initial version we could take NNS version?
@@ -133,22 +134,22 @@ async fn main() -> anyhow::Result<()> {
     // we have to extract the neuron pem file to use with dre
     let token = CancellationToken::new();
     let (sender, mut receiver) = mpsc::channel(2);
-    let handle = tokio::spawn(ict(args.ic_repo_path.clone(), config, token.clone(), sender));
-
-    qualify(
-        &mut receiver,
-        private_key_pem,
-        neuron_id,
-        NETWORK_NAME,
-        initial_version,
-        args.version_to_qualify.to_string(),
-    )
-    .await?;
+
+    tokio::select! {
+        res = ict(args.ic_repo_path.clone(), config, token.clone(), sender) => res?,
+        res = qualify(
+            &mut receiver,
+            private_key_pem,
+            neuron_id,
+            NETWORK_NAME,
+            initial_version,
+            args.version_to_qualify.to_string(),
+        ) => res?
+    };
 
     info!("Finished qualifier run for: {}", args.version_to_qualify);
 
     token.cancel();
-    handle.await??;
     Ok(())
 }