Merge branch 'development' into mqtt

.ci/docker/carl/docker-compose.yml

@@ -16,7 +16,6 @@ services:
       - "../../../resources/development/tls/carl.pem:/etc/opendut/tls/carl.pem"
       - "../../../resources/development/tls/carl.key:/etc/opendut/tls/carl.key"
       # to test toml config file: - "./config.toml:/etc/opendut/carl.toml"
-
     environment:
       - OPENDUT_CARL_NETWORK_TLS_CA=/etc/opendut/tls/ca.pem
       - OPENDUT_CARL_NETWORK_REMOTE_HOST=carl

.ci/docker/telemetry/docker-compose.yml

@@ -59,11 +59,15 @@ services:
      - ./otel-collector.yml:/etc/collector-gateway.yaml
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
+     - "../../../resources/development/tls/:/opendut-tls/:ro"
+   environment:
+     - "SSL_CERT_FILE=/opendut-tls/insecure-development-ca.pem"
    command: [ "--config=/etc/collector-gateway.yaml" ]
    user: "0"
    depends_on:
      - tempo
      - loki
+   restart: on-failure
    networks:
      opendutnet:
        ipv4_address: 192.168.32.220

.ci/docker/telemetry/otel-collector.yml

@@ -1,14 +1,16 @@
 receivers:
- otlp:
-   protocols:
-     http:
-       endpoint: 0.0.0.0:4318
-     grpc:
-       endpoint: 0.0.0.0:4317
- docker_stats:
-   endpoint: unix:///var/run/docker.sock
-   collection_interval: 15s
-   metrics:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+      grpc:
+        endpoint: 0.0.0.0:4317
+        auth:
+          authenticator: oidc
+  docker_stats:
+    endpoint: unix:///var/run/docker.sock
+    collection_interval: 15s
+    metrics:
 processors:
   batch:
 
@@ -25,19 +27,19 @@ processors:
         value: test
 
 exporters:
- debug:
-   verbosity: detailed
- prometheus:
+  debug:
+    verbosity: detailed
+  prometheus:
     endpoint: 0.0.0.0:8889
     const_labels:
       project: opendut
     resource_to_telemetry_conversion: # include information about specific containers
       enabled: true
- otlp/tempo:
+  otlp/tempo:
     endpoint: tempo:4317
     tls:
       insecure: true
- loki:
+  loki:
     endpoint: "http://loki:3100/loki/api/v1/push"
     tls:
       insecure: true
@@ -48,27 +50,30 @@ exporters:
       job: true
 
 extensions:
- health_check:
- pprof:
-   endpoint: :1888
- zpages:
-   endpoint: :55679
+  health_check:
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+  oidc:
+    issuer_url: https://keycloak/realms/opendut
+    audience: account
 
 service:
- extensions: [pprof, zpages, health_check]
- telemetry:
+  extensions: [ pprof, zpages, health_check, oidc ]
+  telemetry:
     logs:
-        level: "debug"
- pipelines:
-   traces:
-     receivers: [otlp]
-     processors: [batch]
-     exporters: [debug, otlp/tempo]
-   metrics:
-     receivers: [otlp, docker_stats]
-     processors: [batch]
-     exporters: [debug, prometheus]
-   logs:
-     receivers: [otlp]
-     processors: [batch, resource, attributes]
-     exporters: [debug, loki]
+      level: "debug"
+  pipelines:
+    traces:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ debug, otlp/tempo ]
+    metrics:
+      receivers: [ otlp, docker_stats ]
+      processors: [ batch ]
+      exporters: [ debug, prometheus ]
+    logs:
+      receivers: [ otlp ]
+      processors: [ batch, resource, attributes ]
+      exporters: [ debug, loki ]

.ci/docker/theo/src/commands/dev.rs

@@ -47,10 +47,10 @@ impl DevCli {
                 docker_compose_network_create()?;
 
                 println!("Starting services...");
-                docker_compose_up_expose_ports(DockerCoreServices::Firefox.as_str(), &expose)?;
-                docker_compose_up_expose_ports(DockerCoreServices::Keycloak.as_str(), &expose)?;
+                docker_compose_up_expose_ports(DockerCoreServices::Firefox.as_str(), expose)?;
+                docker_compose_up_expose_ports(DockerCoreServices::Keycloak.as_str(), expose)?;
                 crate::core::docker::keycloak::wait_for_keycloak_provisioned()?;
-                start_netbird(&expose)?;
+                start_netbird(expose)?;
                 crate::core::docker::netbird::wait_for_netbird_api_key()?;
 
                 println!("Stopping carl in container (if present).");
@@ -119,13 +119,13 @@ fn start_carl_traefik_forwarder() -> Result<i32, Error> {
 
     let mut command = DockerCommand::new();
     command.arg("compose")
-        .arg("-f")
+        .arg("--file")
         .arg(format!(".ci/docker/{}/docker-compose.yml", DockerCoreServices::CarlOnHost));
 
     if running_in_opendut_vm() {
-        command.arg("-f").arg(format!(".ci/docker/{}/vm.yml", DockerCoreServices::CarlOnHost));
+        command.arg("--file").arg(format!(".ci/docker/{}/vm.yml", DockerCoreServices::CarlOnHost));
     } else {
-        command.arg("-f").arg(format!(".ci/docker/{}/localhost.yml", DockerCoreServices::CarlOnHost));
+        command.arg("--file").arg(format!(".ci/docker/{}/localhost.yml", DockerCoreServices::CarlOnHost));
     };
     command.arg("--env-file")
         .arg(".env-theo")
@@ -134,6 +134,6 @@ fn start_carl_traefik_forwarder() -> Result<i32, Error> {
 
     command
         .arg("up")
-        .arg("-d")
+        .arg("--detach")
         .expect_status("Failed to execute compose command for netbird traefik forwarder.")
 }

.ci/docker/theo/src/commands/testenv.rs

@@ -55,11 +55,11 @@ pub struct DestroyArgs {
 }
 
 impl TestenvCli {
-    pub(crate) fn default_handling(&self) -> crate::Result {
+    pub(crate) fn default_handling(self) -> crate::Result {
         load_theo_environment_variables();
         DockerCommand::new().docker_checks()?;
 
-        match &self.task {
+        match self.task {
             TaskCli::Build => {
                 Self::docker_compose_build_testenv_services()?;
             }
@@ -71,21 +71,21 @@ impl TestenvCli {
                 if !skip_firefox {
                     docker_compose_up_expose_ports(DockerCoreServices::Firefox.as_str(), expose)?;
                 }
+                docker_compose_up_expose_ports(DockerCoreServices::Keycloak.as_str(), expose)?;
+                crate::core::docker::keycloak::wait_for_keycloak_provisioned()?;
                 if !skip_telemetry {
                     docker_compose_up_expose_ports(DockerCoreServices::Telemetry.as_str(), expose)?;
                 }
-                docker_compose_up_expose_ports(DockerCoreServices::Keycloak.as_str(), expose)?;
-                crate::core::docker::keycloak::wait_for_keycloak_provisioned()?;
                 start_netbird(expose)?;
                 crate::core::docker::netbird::wait_for_netbird_api_key()?;
 
-                println!("Stopping carl traefik forward (if present).");
+                println!("Stopping CARL Traefik forward (if present).");
                 docker_compose_down(DockerCoreServices::CarlOnHost.as_str(), false)?;
 
                 start_carl_in_docker()?;
                 show_error_if_unhealthy_containers_were_found()?;
 
-                println!("Go to OpenDuT Browser at http://localhost:3000/")
+                println!("Go to OpenDuT Browser at http://localhost:3000/");
             }
             TaskCli::Stop => {
                 docker_compose_down(DockerCoreServices::Keycloak.as_str(), false)?;
@@ -148,7 +148,7 @@ fn start_carl_in_docker() -> Result<i32, Error> {
         .add_common_args(DockerCoreServices::Carl.as_str())
         .add_netbird_api_key_to_env()?
         .arg("up")
-        .arg("-d")
-        .expect_status("Failed to execute compose command for edgar.")
+        .arg("--detach")
+        .expect_status("Failed to execute compose command for CARL.")
 }
 

.ci/docker/theo/src/core/docker/command.rs

@@ -67,7 +67,7 @@ impl DockerCommand {
 
     pub(crate) fn add_common_args(&mut self, compose_dir: &str) -> &mut Self {
         self.arg("compose")
-            .arg("-f")
+            .arg("--file")
             .arg(format!("./.ci/docker/{}/docker-compose.yml", compose_dir))
             .arg("--env-file")
             .arg(".env-theo")

.ci/docker/theo/src/core/docker/compose.rs

@@ -14,25 +14,25 @@ pub(crate) fn docker_compose_build(compose_dir: &str) -> Result<i32, Error> {
         .expect_status(format!("Failed to execute docker compose build for directory: {}.", compose_dir).as_str())
 }
 
-pub fn docker_compose_up_expose_ports(compose_dir: &str, expose: &bool) -> crate::Result {
+pub fn docker_compose_up_expose_ports(compose_dir: &str, expose: bool) -> crate::Result {
     let mut command = DockerCommand::new();
     command.arg("compose")
-        .arg("-f")
+        .arg("--file")
         .arg(format!(".ci/docker/{}/docker-compose.yml", compose_dir));
 
-    if determine_if_ports_shall_be_exposed(*expose) {
-        command.arg("-f")
+    if determine_if_ports_shall_be_exposed(expose) {
+        command.arg("--file")
             .arg(format!(".ci/docker/{}/expose_ports.yml", compose_dir))
     } else {
-        command.arg("-f")
+        command.arg("--file")
             .arg(format!(".ci/docker/{}/localhost_ports.yml", compose_dir))
     };
     command.arg("--env-file")
         .arg(".env-theo")
         .arg("--env-file")
         .arg(".env")
         .arg("up")
-        .arg("-d")
+        .arg("--detach")
         .expect_status(&format!("Failed to execute docker compose command for {}.", compose_dir))?;
     Ok(())
 }
@@ -52,7 +52,7 @@ pub(crate) fn docker_compose_down(compose_dir: &str, delete_volumes: bool) -> Re
 pub(crate) fn docker_compose_network_create() -> Result<i32, Error> {
     DockerCommand::new()
         .arg("compose")
-        .arg("-f")
+        .arg("--file")
         .arg(format!("./.ci/docker/{}/docker-compose.yml", DockerCoreServices::Network))
         .arg("up")
         .arg("--force-recreate")

.ci/docker/theo/src/core/docker/mod.rs

@@ -35,22 +35,22 @@ pub fn show_error_if_unhealthy_containers_were_found() -> Result<(), Error> {
     }
 }
 
-pub fn start_netbird(expose: &bool) -> Result<i32, Error> {
+pub fn start_netbird(expose: bool) -> Result<i32, Error> {
     let mut command = DockerCommand::new();
     command.arg("compose")
-        .arg("-f")
+        .arg("--file")
         .arg(".ci/docker/netbird/docker-compose.yml");
 
-    if determine_if_ports_shall_be_exposed(*expose) {
-        command.arg("-f")
+    if determine_if_ports_shall_be_exposed(expose) {
+        command.arg("--file")
             .arg(".ci/docker/netbird/expose_ports.yml");
     };
     command.arg("--env-file")
         .arg(".env-theo")
         .arg("--env-file")
         .arg(".env")
         .arg("up")
-        .arg("-d")
+        .arg("--detach")
         .expect_status("Failed to execute compose command for netbird")
 }
 

.ci/xtask/src/core/util.rs

@@ -65,6 +65,7 @@ pub fn init_tracing() -> crate::Result {
         .with_max_level(tracing::Level::TRACE)
         .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
         .with_env_filter(tracing_filter)
+        .with_writer(std::io::stderr)
         .compact()
         .init();
     Ok(())