Merge branch 'eclipse-opendut:development' into development

.ci/docker/edgar/Dockerfile

@@ -1,3 +1,12 @@
+FROM rust:1-buster AS builder
+
+WORKDIR /src
+COPY ./ /src
+RUN echo "Show build context size" && du -sh /src
+RUN cargo build --package opendut-theo --release
+RUN ls /src/target/release/opendut-theo
+
+
 FROM ubuntu:22.04
 
 RUN apt update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -10,6 +19,9 @@ RUN curl --verbose https://raw.githubusercontent.com/gdraheim/docker-systemctl-r
 RUN echo "01beb201d2045c5e548d012bde9b6ae6113392a57bbea7b3e81131aac995f77a /usr/bin/systemctl" | sha256sum --check --status
 RUN chmod +x /usr/bin/systemctl
 
+# copy theo binary after installing dependencies to prevent reinstalling when build context changes (proper use of docker cache)
+COPY --from=builder /src/target/release/opendut-theo /usr/local/bin/opendut-theo
+
 WORKDIR /opt
 RUN mkdir /opt/signal/
 COPY ./.ci/docker/edgar/scripts/* /opt

.ci/docker/edgar/Dockerfile.dockerignore

@@ -0,0 +1,7 @@
+# Do not include everything in edgars build context
+target/
+.vagrant/
+.github/
+.git/
+.env*
+doc/

.ci/docker/edgar/Readme.md

@@ -43,7 +43,7 @@ awk 'NF {sub(/\r/, ""); printf "%s\\n",$0;}' resources/development/tls/insecure-
 ```
 docker compose build
 docker compose up -d
-docker exec -ti edgar_router /opt/wait_until_ready.sh
+docker exec -ti edgar-leader /opt/wait_until_ready.sh
 docker exec -ti edgar-peer-1 /opt/wait_until_ready.sh
 docker exec -ti edgar-peer-1 /opt/pingall.sh
 docker exec -ti edgar-peer-1 python3 /opt/delete_peers.py

.ci/docker/edgar/docker-compose.yml

@@ -2,12 +2,12 @@ version: "3.9"
 
 services:
 
-  router:
-    container_name: edgar_router  # defined DNS for the container
+  leader:
+    container_name: edgar-leader  # defined DNS for the container
     build:
       context: ../../..
       dockerfile: ./.ci/docker/edgar/Dockerfile
-    command: /opt/managed.sh router
+    command: /opt/managed.sh leader
     #command: sleep infinity
     volumes:
       - ../../../target/ci/distribution/x86_64-unknown-linux-gnu/:/opt/artifacts

.ci/docker/edgar/scripts/functions.sh

@@ -88,15 +88,15 @@ check_timeout() {
 }
 
 
-debug_show_peers_requesting_router_ip() {
+debug_show_peers_requesting_leader_ip() {
   while true; do
-    lookups=$(grep router_ip.txt logs.txt | nl | awk '{print $1}' | tail -n1)
+    lookups=$(grep leader_ip.txt logs.txt | nl | awk '{print $1}' | tail -n1)
     num_lookups=${lookups:-0}
-    echo "${num_lookups} of ${OPENDUT_EDGAR_REPLICAS} peers fetched the router_ip address."
+    echo "${num_lookups} of ${OPENDUT_EDGAR_REPLICAS} peers fetched the leader_ip address."
     if [ "${num_lookups}" == "${OPENDUT_EDGAR_REPLICAS}" ]; then
       break
     else
-      echo "Waiting for peers to fetch router_ip address."
+      echo "Waiting for peers to fetch leader_ip address."
       sleep 1
     fi
   done

.ci/docker/edgar/scripts/managed.sh

@@ -53,7 +53,13 @@ PEER_ID=$(uuidgen)
 NAME="${OPENDUT_EDGAR_CLUSTER_NAME}_$(hostname)"
 echo "Creating peer with name $NAME and id $PEER_ID"
 opendut-cleo create peer --name "$NAME" --id "$PEER_ID"
-opendut-cleo create device --peer-id "$PEER_ID" --name device-"$NAME" --interface eth0 --location "$NAME" --tags "$OPENDUT_EDGAR_CLUSTER_NAME"
+
+DEVICE_INTERFACE="dut0"
+DEVICE_ADDRESS=$(ip -json address show dev eth0 | jq --raw-output '.[0].addr_info[0].local' | sed --expression 's#32#33#')  # derive from existing address, by replacing '32' with '33'
+ip link add $DEVICE_INTERFACE type dummy
+ip address add "$DEVICE_ADDRESS/24" dev "$DEVICE_INTERFACE"
+ip link set dev $DEVICE_INTERFACE up
+opendut-cleo create device --peer-id "$PEER_ID" --name device-"$NAME" --interface "$DEVICE_INTERFACE" --location "$NAME" --tags "$OPENDUT_EDGAR_CLUSTER_NAME"
 
 PEER_SETUP_KEY=$(opendut-cleo generate-peer-setup --id "$PEER_ID")
 echo "Setting up peer with setup key $PEER_SETUP_KEY"
@@ -76,7 +82,7 @@ while ! cleo_count_connected_peers_in_cluster "$expected_peer_count" "$OPENDUT_E
   sleep 3
 done
 
-if [ "$1" == "router" ]; then
+if [ "$1" == "leader" ]; then
   DEVICES="$(opendut-cleo list --output=json devices | jq --arg NAME "$OPENDUT_EDGAR_CLUSTER_NAME" -r '.[] | select(.tags==$NAME).name' | xargs echo)"
   echo "Enumerating devices to join cluster: $DEVICES"
 
@@ -91,6 +97,10 @@ if [ "$1" == "router" ]; then
   CLUSTER_ID=$(echo "$RESPONSE" | jq -r '.id')
   echo "Creating cluster deployment for id=$CLUSTER_ID"
   opendut-cleo create cluster-deployment --id "$CLUSTER_ID"
+
+  # TODO: wait until opendut bridge device exists?
+  # TODO: wait until gre device exists?
+
   echo "Success" | tee -a > /opt/signal/success.txt
 
 fi

.ci/docker/edgar/scripts/pingall.sh

@@ -2,27 +2,49 @@
 
 source "$(dirname "$0")/functions.sh"
 
-ping_all_peers() {
+ping_all_netbird_peers() {
+  REQUIRED_SUCCESS="$1"
   IPS=$(netbird status --json | jq -r '.peers.details[].netbirdIp')
   for ip in $IPS
   do
-    fping -c1 -t500 "$ip" || { echo "$ip did not respond"; sleep 3; }
+    if [ "$REQUIRED_SUCCESS" == "true" ]; then
+      fping --count=1 --timeout=1000 --retry=5 "$ip" || { echo "$ip did not respond"; return 1; }
+    else
+      fping --count=1 --timeout=1000 --retry=5 "$ip" || { echo "$ip did not respond"; sleep 10; }
+    fi
+  done
+}
+
+ping_all_dut_interfaces() {
+  REQUIRED_SUCCESS="$1"
+  IPS=$(wg show all endpoints | grep -Eo '192.168.32.[0-9]+' | sed -e 's#32#33#')
+  for ip in $IPS
+  do
+    if [ "$REQUIRED_SUCCESS" == "true" ]; then
+      fping --count=1 --timeout=1000 --retry=5 "$ip" || { echo "$ip did not respond"; return 1; }
+    else
+      fping --count=1 --timeout=1000 --retry=5 "$ip" || { echo "$ip did not respond"; sleep 10; }
+    fi
   done
 }
 
 
 wait_for_peers_to_connect
 
 echo "first ping may take multiple seconds"
-ping_all_peers
-ping_all_peers
+ping_all_netbird_peers "false"
+ping_all_netbird_peers "false"
 
 set -e  # exit on error
 set -x  # print each command
 
 echo "-------------------------------------------------------------------------"
 echo "Running ping test"
-ping_all_peers
+echo "Pinging NetBird peers..."
+ping_all_netbird_peers "true"
+
+#echo "Pinging DUT interfaces..."  #TODO
+#ping_all_dut_interfaces "true"
 
 echo "SUCCESS"
 exit 0

.ci/docker/edgar/scripts/prepare.sh

@@ -5,38 +5,38 @@ trap die_with_error TERM
 
 
 if [ -n "$1" ] ; then
-  /opt/opendut-edgar/opendut-edgar setup --no-confirm unmanaged --setup-key "$NETBIRD_SETUP_KEY" --management-url "${NETBIRD_MANAGEMENT_API}" --router=local
+  /opt/opendut-edgar/opendut-edgar setup --no-confirm unmanaged --setup-key "$NETBIRD_SETUP_KEY" --management-url "${NETBIRD_MANAGEMENT_API}" --leader=local
 
   while ! netbird status | grep IP; do
     echo "Waiting for netbird to start up..."
     sleep 1
   done
-  router_ip=$(netbird status | grep IP | grep -Eo "[0-9]+.[0-9]+.[0-9]+.[0-9]+")
+  leader_ip=$(netbird status | grep IP | grep -Eo "[0-9]+.[0-9]+.[0-9]+.[0-9]+")
 
-  echo "$router_ip" > router_ip.txt
+  echo "$leader_ip" > leader_ip.txt
   python3 -m http.server 2> logs.txt &
   python3 ip_provider.py 2> provider.txt &
 
-  debug_show_peers_requesting_router_ip
+  debug_show_peers_requesting_leader_ip
   wait_for_peers_to_connect
 
-  /opt/opendut-edgar/opendut-edgar setup --no-confirm unmanaged --setup-key "$NETBIRD_SETUP_KEY" --management-url "${NETBIRD_MANAGEMENT_API}" --router=local
+  /opt/opendut-edgar/opendut-edgar setup --no-confirm unmanaged --setup-key "$NETBIRD_SETUP_KEY" --management-url "${NETBIRD_MANAGEMENT_API}" --leader=local
   echo setting bridge ip
   ip a a 192.168.100.1/24 dev br-opendut
 
 else
-  echo waiting for router to come up
-  while ! curl -sf "http://edgar_router:8000" --output /dev/null; do
-    echo "Waiting for router to start up..."
+  echo waiting for leader to come up
+  while ! curl -sf "http://edgar-leader:8000" --output /dev/null; do
+    echo "Waiting for leader to start up..."
     sleep 3
   done
 
-  router_ip=$(curl --silent http://edgar_router:8000/router_ip.txt)
-  echo "Using router router_ip address $router_ip"
-  /opt/opendut-edgar/opendut-edgar setup --no-confirm unmanaged --setup-key "$NETBIRD_SETUP_KEY" --management-url "${NETBIRD_MANAGEMENT_API}" --router="$router_ip"
+  leader_ip=$(curl --silent http://edgar-leader:8000/leader_ip.txt)
+  echo "Using leader leader_ip address $leader_ip"
+  /opt/opendut-edgar/opendut-edgar setup --no-confirm unmanaged --setup-key "$NETBIRD_SETUP_KEY" --management-url "${NETBIRD_MANAGEMENT_API}" --leader="$leader_ip"
 
   echo fetching bridge_ip
-  bridge_ip=$(curl --silent http://edgar_router:5000/)
+  bridge_ip=$(curl --silent http://edgar-leader:5000/)
   bridge_ip="192.168.100.${bridge_ip}/24"
   echo "Got bridge ip ${bridge_ip}"
   ip a a "${bridge_ip}" dev br-opendut

.ci/docker/keycloak/keycloak_functions.sh

@@ -1,11 +1,24 @@
 #!/bin/bash
 
 wait_for_keycloak() {
+  local timeout="${1:-600}"
+  local sleep_time="${2:-5}"
+
+  START_TIME="$(date +%s)"
+  END_TIME=$((START_TIME + timeout))
+
   # wait until keycloak is ready and returns a status code < 400
-  while ! curl --silent --fail "$KEYCLOAK_URL" --output /dev/null; do
+  while ! curl --silent --fail --connect-timeout 2 --max-time 2 "$KEYCLOAK_URL" --output /dev/null; do
+    local now
+    now=$(date +%s)
+    if [ "$now" -gt "$END_TIME" ]; then
+      echo "Timeout while waiting for Keycloak to start up at: '$KEYCLOAK_URL'"
+      return 1
+    fi
     echo "Waiting for Keycloak to start up..."
-    sleep 5
+    sleep "$sleep_time"
   done
+
   echo "Keycloak ready"
 }
 kcadm() { local cmd="$1" ; shift ; "$KCADM_PATH" "$cmd" --config /tmp/kcadm.config "$@" ; }

.ci/docker/keycloak/provision.sh

@@ -6,7 +6,7 @@ main() {
   #set -ex
 
   echo "provisioning keycloak"
-  wait_for_keycloak
+  wait_for_keycloak || { echo "Keycloak not ready, exiting"; exit 1;}
   kcauth
 
   # https://docs.netbird.io/selfhosted/identity-providers#keycloak

.ci/docker/theo/src/commands/dev.rs

@@ -86,7 +86,7 @@ impl DevCli {
                     .arg("--rm")
                     //.arg("--entrypoint=\"\"")
                     .arg("-it")
-                    .arg("router")
+                    .arg("leader")
                     .arg("bash")
                     .run()?;
             }

.ci/docker/theo/src/commands/edgar.rs

@@ -31,9 +31,9 @@ impl TestEdgarCli {
                 docker_compose_down(DockerCoreServices::Edgar.as_str(), false)?;
                 docker_compose_build(DockerCoreServices::Edgar.as_str())?;
                 start_edgar_in_docker()?;
-                wait_for_edgar_router_provisioned()?;
-                println!("Edgar router is provisioned. Checking if all peers respond to ping...");
-                check_edgar_router_ping_all()?;
+                wait_for_edgar_leader_provisioned()?;
+                println!("Edgar leader is provisioned. Checking if all peers respond to ping...");
+                check_edgar_leader_ping_all()?;
             }
             TaskCli::Stop => {
                 docker_compose_down(DockerCoreServices::Edgar.as_str(), false)?;
@@ -57,61 +57,61 @@ fn start_edgar_in_docker() -> Result<i32, Error> {
 }
 
 
-fn wait_for_edgar_router_provisioned() -> crate::Result {
+fn wait_for_edgar_leader_provisioned() -> crate::Result {
     let timeout = Duration::from_secs(TIMEOUT_SECONDS);
     let start = std::time::Instant::now();
-    let mut edgar_logs = check_edgar_router_logs()?;
+    let mut edgar_logs = check_edgar_leader_logs()?;
     let mut first_check = true;
 
-    while !check_edgar_router_done()? {
+    while !check_edgar_leader_done()? {
         let duration = start.elapsed();
         if duration > timeout {
             return Err(anyhow!(
-                TheoError::Timeout(String::from("An error occurred while waiting for the Edgar router to deploy."))
+                TheoError::Timeout(String::from("An error occurred while waiting for the Edgar leader to deploy."))
             ));
         }
-        let new_edgar_logs = check_edgar_router_logs()?;
+        let new_edgar_logs = check_edgar_leader_logs()?;
         if first_check {
             first_check = false;
         } else if new_edgar_logs.chars().count() == edgar_logs.chars().count() {
-            println!("No progress in the logs. Check 'docker logs edgar_router'.");
+            println!("No progress in the logs. Check 'docker logs edgar-leader'.");
         }
         edgar_logs = new_edgar_logs.to_string();
-        println!("{:^width$} seconds - Waiting for edgar router to be deployed...", duration.as_secs(), width=6);
+        println!("{:^width$} seconds - Waiting for edgar leader to be deployed...", duration.as_secs(), width=6);
         sleep(Duration::from_secs(SLEEP_TIME_SECONDS));
     }
     Ok(())
 }
 
-fn check_edgar_router_logs() -> Result<String, Error> {
+fn check_edgar_leader_logs() -> Result<String, Error> {
     let command_output = DockerCommand::new()
         .arg("logs")
-        .arg("edgar_router")
-        .expect_output("Failed to get edgar router logs.")?;
+        .arg("edgar-leader")
+        .expect_output("Failed to get edgar leader logs.")?;
     let output = String::from_utf8(command_output.stdout)?;
     Ok(output)
 }
 
-fn check_edgar_router_done() -> Result<bool, Error> {
-    let exists = DockerCommand::exists("edgar_router");
+fn check_edgar_leader_done() -> Result<bool, Error> {
+    let exists = DockerCommand::exists("edgar-leader");
     if !exists {
-        Err(TheoError::DockerCommandFailed("Edgar router container has terminated or does not exists!".to_string()).into())
+        Err(TheoError::DockerCommandFailed("Edgar leader container has terminated or does not exists!".to_string()).into())
     } else {
-        check_edgar_router_provisioning_finished()
+        check_edgar_leader_provisioning_finished()
     }
 }
 
-fn check_edgar_router_provisioning_finished() -> Result<bool, Error> {
-    let command_output = DockerCommand::new_exec("edgar_router")
+fn check_edgar_leader_provisioning_finished() -> Result<bool, Error> {
+    let command_output = DockerCommand::new_exec("edgar-leader")
         .arg("ls")
         .arg("/opt/signal/success.txt")
-        .expect_output("Failed to check if edgar router was provisioned.");
+        .expect_output("Failed to check if edgar leader was provisioned.");
     DockerCommand::check_output_status(command_output)
 }
 
 
-fn check_edgar_router_ping_all() -> Result<i32, Error> {
-    DockerCommand::new_exec("edgar_router")
+fn check_edgar_leader_ping_all() -> Result<i32, Error> {
+    DockerCommand::new_exec("edgar-leader")
         .arg("/opt/pingall.sh")
         .expect_status("Failed to check if all EDGAR peers respond to ping.")
 }

.ci/docker/theo/src/commands/testenv.rs

@@ -80,6 +80,7 @@ impl TestenvCli {
             }
             TaskCli::Destroy => {
                 docker_compose_down(DockerCoreServices::Firefox.as_str(), true)?;
+                docker_compose_down(DockerCoreServices::Edgar.as_str(), true)?;
                 docker_compose_down(DockerCoreServices::Carl.as_str(), true)?;
                 docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), true)?;
                 docker_compose_down(DockerCoreServices::Netbird.as_str(), true)?;

.ci/docker/theo/src/commands/vagrant.rs

@@ -33,6 +33,8 @@ pub enum TaskCli {
     Destroy,
     #[command(about = "Reload virtual machine.", alias = "restart")]
     Reload,
+    #[command(about = "Reload virtual machine.")]
+    Status,
     #[command(about = "Run firefox remotely on the virtual machine (x11forwarding).")]
     Firefox,
     #[command(about = "Alternatives to run firefox remotely.")]
@@ -73,6 +75,9 @@ impl VagrantCli {
             TaskCli::Reload => {
                 Command::vagrant().arg("reload").run();
             }
+            TaskCli::Status => {
+                Command::vagrant().arg("status").run();
+            }
             TaskCli::Other => {
                 let project_root = PathBuf::project_path_buf();
                 let vagrant_file_path = project_root.join(".ci/docker/Vagrantfile");

.ci/docker/theo/src/core/docker/compose.rs

@@ -63,5 +63,5 @@ pub(crate) fn docker_compose_network_delete() -> Result<i32, Error> {
         .arg("network")
         .arg("rm")
         .arg("opendut_network")
-        .expect_status("Failed to create docker network.")
+        .expect_status("Failed to delete docker network.")
 }

.github/workflows/job-run-testenv.yaml

@@ -62,7 +62,7 @@ jobs:
 
       - name: Wait for all peers
         run: |
-          docker exec edgar_router /opt/wait_until_ready.sh
+          docker exec edgar-leader /opt/wait_until_ready.sh
 
       - name: Run ping all script on one of the peers
         run: |