[VER-1297] UST: Support for specifying elasticsearch hosts

rs/ic_fondue/src/prod_tests/bootstrap.rs

@@ -240,7 +240,6 @@ pub fn create_config_disk_images(
         let img_path = PathBuf::from(&node.node_path).join(CONF_IMG_FNAME);
 
         let mut cmd = Command::new("build-bootstrap-config-image.sh");
-
         cmd.arg(img_path.clone())
             .arg("--ic_registry_local_store")
             .arg(&ic_registry_local_store_path)
@@ -249,6 +248,11 @@ pub fn create_config_disk_images(
             .arg("--accounts_ssh_authorized_keys")
             .arg(ctx.authorized_ssh_accounts_dir.path());
 
+        if !ctx.journalbeat_hosts.is_empty() {
+            cmd.arg("--journalbeat_hosts")
+                .arg(ctx.journalbeat_hosts.join(","));
+        }
+
         let output = cmd
             .output()
             .expect("could not spawn image creation process");

rs/ic_fondue/src/prod_tests/cli.rs

@@ -105,6 +105,12 @@ The sha256 hash sum of the base image."#
         (file/file.pub) that are installed on the IC-OS by default."#
     )]
     pub authorized_ssh_accounts: Option<PathBuf>,
+    #[structopt(
+        long = "journalbeat-hosts",
+        help = r#"A comma-separated list of hostname/port-pairs that journalbeat
+        should use as target hosts. (e.g. "host1.target.com:443,host2.target.com:443")"#
+    )]
+    pub journalbeat_hosts: Option<String>,
 }
 
 impl CliArgs {
@@ -145,6 +151,8 @@ impl CliArgs {
             None => vec![],
         };
 
+        let journalbeat_hosts = parse_journalbeat_hosts(self.journalbeat_hosts)?;
+
         Ok(ValidatedCliArgs {
             log_base_dir: self.log_base_dir,
             log_level,
@@ -161,6 +169,7 @@ impl CliArgs {
             ignore_pattern,
             skip_pattern,
             authorized_ssh_accounts,
+            journalbeat_hosts,
         })
     }
 }
@@ -190,6 +199,7 @@ pub struct ValidatedCliArgs {
     pub ignore_pattern: Option<Regex>,
     pub skip_pattern: Option<Regex>,
     pub authorized_ssh_accounts: Vec<AuthorizedSshAccount>,
+    pub journalbeat_hosts: Vec<String>,
 }
 
 pub type PrivateKeyFileContent = Vec<u8>;
@@ -245,13 +255,33 @@ fn is_valid_ssh_key_dir<P: AsRef<Path>>(p: P) -> Result<Vec<AuthorizedSshAccount
     Ok(res)
 }
 
+/// Checks whether the input string as the form [hostname:port{,hostname:port}]
+fn parse_journalbeat_hosts(s: Option<String>) -> Result<Vec<String>> {
+    const HOST_START: &str = r#"^(([[:alnum:]]|[[:alnum:]][[:alnum:]\-]*[[:alnum:]])\.)*"#;
+    const HOST_STOP: &str = r#"([[:alnum:]]|[[:alnum:]][[:alnum:]\-]*[[:alnum:]])"#;
+    const PORT: &str = r#":[[:digit:]]{2,5}$"#;
+    let s = match s {
+        Some(s) => s,
+        None => return Ok(vec![]),
+    };
+    let rgx = format!("{}{}{}", HOST_START, HOST_STOP, PORT);
+    let rgx = Regex::new(&rgx).unwrap();
+    let mut res = vec![];
+    for target in s.trim().split(',') {
+        if !rgx.is_match(target) {
+            bail!("Invalid journalbeat host: '{}'", s);
+        }
+        res.push(target.to_string());
+    }
+    Ok(res)
+}
+
 #[cfg(test)]
 #[cfg(target_os = "linux")]
 mod tests {
+    use super::{is_valid_ssh_key_dir, parse_journalbeat_hosts};
     use std::{fs::OpenOptions, path::Path, process::Command};
 
-    use super::is_valid_ssh_key_dir;
-
     #[test]
     fn valid_key_dir_is_valid_key_dir() {
         let tempdir = tempfile::tempdir().expect("Could not create a temp dir");
@@ -294,4 +324,55 @@ mod tests {
             .output()
             .expect("Could not execute ssh-keygen");
     }
+
+    #[test]
+    fn invalid_journalbeat_hostnames_are_rejected() {
+        let invalid_hostnames = &[
+            "sub.domain.tld:1a23",
+            "sub.domain-.tld:123",
+            "sub.domain-.tld:aaa",
+            "sub.domain-.tld:1a2",
+            "sub.-domain.tld:123",
+            "sub.-domain.tl.:123",
+            ".:123",
+            ":123",
+            "sub.domain.tld:",
+            "sub.domain.tld",
+        ];
+
+        for hostname in invalid_hostnames {
+            let hostname = Some(hostname.to_string());
+            assert!(parse_journalbeat_hosts(hostname).is_err())
+        }
+
+        for i in 0..invalid_hostnames.len() {
+            let s = Some(invalid_hostnames[i..].join(","));
+            assert!(parse_journalbeat_hosts(s).is_err())
+        }
+    }
+
+    #[test]
+    fn valid_journalbeat_hostnames_are_accepted() {
+        let invalid_hostnames = &[
+            "sub.domain.tld:123",
+            "sub.domain.tld:12",
+            "sub.domain.tld:123",
+            "sub.domain.tld:1234",
+            "sub.domain.tld:12345",
+            "sub.do-main.tld:123",
+            "sub.do--main.tld:123",
+            "s-ub.domain.tl:123",
+        ];
+
+        for hostname in invalid_hostnames {
+            let hostname = Some(hostname.to_string());
+            assert!(parse_journalbeat_hosts(hostname).is_ok())
+        }
+
+        for i in 0..invalid_hostnames.len() {
+            let s = Some(invalid_hostnames[i..].join(","));
+            let res = parse_journalbeat_hosts(s).expect("Could not parse journalbeat hosts!");
+            assert_eq!(res.len(), invalid_hostnames.len() - i);
+        }
+    }
 }

rs/ic_fondue/src/prod_tests/driver_setup.rs

@@ -75,6 +75,7 @@ pub fn create_driver_context_from_cli(
         logs_base_dir: cli_args.log_base_dir,
         authorized_ssh_accounts_dir: ssh_key_dir,
         authorized_ssh_accounts: cli_args.authorized_ssh_accounts,
+        journalbeat_hosts: cli_args.journalbeat_hosts,
     }
 }
 
@@ -146,6 +147,7 @@ pub struct DriverContext {
     pub logs_base_dir: Option<PathBuf>,
     pub authorized_ssh_accounts_dir: Arc<TempDir>,
     pub authorized_ssh_accounts: Vec<AuthorizedSshAccount>,
+    pub journalbeat_hosts: Vec<String>,
 }
 
 impl DriverContext {

rs/tests/run-farm-based-system-tests.sh

@@ -83,6 +83,11 @@ if [ -z "${SSH_KEY_DIR:-}" ]; then
     ssh-keygen -t ed25519 -N '' -f "$SSH_KEY_DIR/admin"
 fi
 
+JOURNALBEAT_HOSTS=()
+if [ -n "${TEST_ES_HOSTNAMES:-}" ]; then
+    JOURNALBEAT_HOSTS+=("--journalbeat-hosts" "${TEST_ES_HOSTNAMES//[[:space:]]/}")
+fi
+
 RCLONE_ARGS=("--git-rev" "$GIT_REVISION" "--out=$ARTIFACT_DIR" "--unpack" "--mark-executable")
 # prod-test-driver and (NNS) canisters
 "${CI_PROJECT_DIR}"/gitlab-ci/src/artifacts/rclone_download.py --remote-path=canisters "${RCLONE_ARGS[@]}"
@@ -112,7 +117,8 @@ DEV_IMG_SHA256=$(curl "${DEV_IMG_SHA256_URL}" | sed -E 's/^([0-9a-fA-F]+)\s.*/\1
         --base-img-sha256 "${DEV_IMG_SHA256}" \
         --nns-canister-path "${ARTIFACT_DIR}" \
         --authorized-ssh-accounts "${SSH_KEY_DIR}" \
-        --result-file "${RESULT_FILE}" 2>&1
+        --result-file "${RESULT_FILE}" \
+        "${JOURNALBEAT_HOSTS[@]}" 2>&1
 } && RES=0 || RES=$?
 
 # Export spans to Honeycomb if the script is run by a CI pipeline.