From 0b682ad6f8b45796bec3c9a774b26364fc4b874c Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Wed, 22 Dec 2021 14:53:51 -0800 Subject: [PATCH] Revert "Revert "chore: Update k8s-e2e to use new helm chart (#10521)" (#10566)" This reverts commit 738e6c29d3da379c72ceb19800c943c0a17d0fbf. --- lib/k8s-e2e-tests/src/lib.rs | 6 +- lib/k8s-e2e-tests/tests/vector-agent.rs | 273 ++++++++--------- lib/k8s-e2e-tests/tests/vector-aggregator.rs | 36 +-- .../tests/vector-dd-agent-aggregator.rs | 35 +-- lib/k8s-e2e-tests/tests/vector.rs | 285 +++++++----------- lib/k8s-test-framework/src/framework.rs | 2 + lib/k8s-test-framework/src/vector.rs | 4 + scripts/deploy-chart-test.sh | 23 +- 8 files changed, 264 insertions(+), 400 deletions(-) diff --git a/lib/k8s-e2e-tests/src/lib.rs b/lib/k8s-e2e-tests/src/lib.rs index ffd8472065f93..43a275e2f6125 100644 --- a/lib/k8s-e2e-tests/src/lib.rs +++ b/lib/k8s-e2e-tests/src/lib.rs @@ -30,7 +30,7 @@ pub fn get_namespace() -> String { .map(|num| (num as char).to_ascii_lowercase()) .collect(); - format!("test-vector-{}", id) + format!("vector-{}", id) } pub fn get_namespace_appended(namespace: &str, suffix: &str) -> String { @@ -236,9 +236,9 @@ pub async fn smoke_check_first_line(log_reader: &mut Reader) { .read_line() .await .expect("unable to read first line"); - let expected_pat = "INFO vector::app: Log level is enabled. level=\"info\"\n"; + let expected_pat = "INFO vector::app: Log level is enabled."; assert!( - first_line.ends_with(expected_pat), + first_line.contains(expected_pat), "Expected a line ending with {:?} but got {:?}; vector might be malfunctioning", expected_pat, first_line diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index fe9ba2f52f2ab..587aeb75cd0bb 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -13,54 +13,14 @@ use k8s_test_framework::{ }; use tracing::{debug, info}; -const HELM_VALUES_LOWER_GLOB: &str = indoc! {r#" - kubernetesLogsSource: - rawConfig: | - glob_minimum_cooldown_ms = 5000 +const HELM_VALUES_AGENT: &str = indoc! {r#" + role: "Agent" "#}; -const HELM_VALUES_CUSTOM_CONFIG: &str = indoc! {r#" - customConfig: - data_dir: "/vector-data-dir" - sources: - host_metrics: - type: host_metrics - filesystem: - devices: - excludes: ["binfmt_misc"] - filesystems: - excludes: ["binfmt_misc"] - mountpoints: - excludes: ["*/proc/sys/fs/binfmt_misc"] - internal_metrics: - type: internal_metrics - kubernetes_logs: - type: kubernetes_logs - glob_minimum_cooldown_ms: 5000 - sinks: - prometheus_sink: - type: prometheus_exporter - inputs: ["host_metrics", "internal_metrics"] - address: 0.0.0.0:9090 - stdout: - type: console - inputs: ["kubernetes_logs"] - encoding: json -"#}; - -const HELM_VALUES_STDOUT_SINK: &str = indoc! {r#" - sinks: - stdout: - type: "console" - inputs: ["kubernetes_logs"] - target: "stdout" - encoding: "json" -"#}; - -const HELM_VALUES_ADDITIONAL_CONFIGMAP: &str = indoc! {r#" - extraConfigDirSources: - - configMap: - name: vector-agent-config +const HELM_VALUES_EXISTING_CONFIGMAP: &str = indoc! {r#" + role: "Agent" + existingConfigMaps: + - vector-agent-config "#}; const CUSTOM_RESOURCE_VECTOR_CONFIG: &str = indoc! {r#" @@ -70,19 +30,22 @@ const CUSTOM_RESOURCE_VECTOR_CONFIG: &str = indoc! {r#" name: vector-agent-config data: vector.toml: | + data_dir = "/vector-data-dir" + [api] + enabled = false + [sources.kubernetes_logs] + type = "kubernetes_logs" [sinks.stdout] type = "console" inputs = ["kubernetes_logs"] - target = "stdout" encoding = "json" "#}; -/// This test validates that vector-agent picks up logs at the simplest case +/// This test validates that vector picks up logs at the simplest case /// possible - a new pod is deployed and prints to stdout, and we assert that -/// vector picks that up - but with the new `customConfig` way of passing the -/// sink configuration. +/// vector picks that up #[tokio::test] -async fn simple_custom_config() -> Result<(), Box> { +async fn default_agent() -> Result<(), Box> { let _guard = lock(); init(); @@ -94,12 +57,13 @@ async fn simple_custom_config() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_CUSTOM_CONFIG, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -179,7 +143,7 @@ async fn simple_custom_config() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly merges a log message that +/// This test validates that vector properly merges a log message that /// kubernetes has internally split into multiple partial log lines. #[tokio::test] async fn partial_merge() -> Result<(), Box> { @@ -194,13 +158,13 @@ async fn partial_merge() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -281,7 +245,7 @@ async fn partial_merge() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent picks up preexisting logs - logs that +/// This test validates that vector picks up preexisting logs - logs that /// existed before vector was deployed. #[tokio::test] async fn preexisting() -> Result<(), Box> { @@ -323,13 +287,13 @@ async fn preexisting() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -385,7 +349,7 @@ async fn preexisting() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent picks up multiple log lines, and that +/// This test validates that vector picks up multiple log lines, and that /// they arrive at the proper order. #[tokio::test] async fn multiple_lines() -> Result<(), Box> { @@ -401,13 +365,13 @@ async fn multiple_lines() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -489,7 +453,7 @@ async fn multiple_lines() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly annotates log events with pod +/// This test validates that vector properly annotates log events with pod /// and namespace metadata obtained from the k8s API. #[tokio::test] async fn metadata_annotation() -> Result<(), Box> { @@ -504,13 +468,13 @@ async fn metadata_annotation() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -638,7 +602,7 @@ async fn metadata_annotation() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly filters out the logs that are +/// This test validates that vector properly filters out the logs that are /// requested to be excluded from collection, based on k8s API `Pod` labels. #[tokio::test] async fn pod_filtering() -> Result<(), Box> { @@ -654,13 +618,13 @@ async fn pod_filtering() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -835,7 +799,7 @@ async fn pod_filtering() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly filters out the logs by the +/// This test validates that vector properly filters out the logs by the /// custom selectors, based on k8s API `Pod` labels and annotations. #[tokio::test] async fn custom_selectors() -> Result<(), Box> { @@ -848,24 +812,33 @@ async fn custom_selectors() -> Result<(), Box> { let override_name = get_override_name(&namespace, "vector-agent"); const CONFIG: &str = indoc! {r#" - kubernetesLogsSource: - rawConfig: | - glob_minimum_cooldown_ms = 5000 - extra_label_selector = "my_custom_negative_label_selector!=my_val" - extra_field_selector = "metadata.name!=test-pod-excluded-by-name" + role: "Agent" + customConfig: + data_dir: "/vector-data-dir" + api: + enabled: true + address: 127.0.0.1:8686 + sources: + kubernetes_logs: + type: kubernetes_logs + extra_label_selector: "my_custom_negative_label_selector!=my_val" + extra_field_selector: "metadata.name!=test-pod-excluded-by-name" + sinks: + stdout: + type: console + inputs: [kubernetes_logs] + encoding: + codec: json "#}; let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { - custom_helm_values: vec![ - &config_override_name(&override_name, true), - CONFIG, - HELM_VALUES_STDOUT_SINK, - ], + custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() }, ) @@ -1040,7 +1013,7 @@ async fn custom_selectors() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly filters out the logs from +/// This test validates that vector properly filters out the logs from /// particular containers that are requested to be excluded from collection, /// based on k8s API `Pod` annotations. #[tokio::test] @@ -1056,13 +1029,13 @@ async fn container_filtering() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -1204,7 +1177,7 @@ async fn container_filtering() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly filters out the logs matching +/// This test validates that vector properly filters out the logs matching /// the exclusion glob patterns specified at the `kubernetes_logs` /// configuration. #[tokio::test] @@ -1217,27 +1190,33 @@ async fn glob_pattern_filtering() -> Result<(), Box> { let framework = make_framework(); let override_name = get_override_name(&namespace, "vector-agent"); - let config: &str = &format!( - indoc! {r#" - kubernetesLogsSource: - rawConfig: | - exclude_paths_glob_patterns = ["/var/log/pods/{}_test-pod_*/excluded/**"] - glob_minimum_cooldown_ms = 5000 - "#}, - pod_namespace - ); + const CONFIG: &str = indoc! {r#" + role: "Agent" + customConfig: + data_dir: "/vector-data-dir" + api: + enabled: true + address: 127.0.0.1:8686 + sources: + kubernetes_logs: + type: kubernetes_logs + exclude_paths_glob_patterns: ["/var/log/pods/*_test-pod_*/excluded/**"] + sinks: + stdout: + type: console + inputs: [kubernetes_logs] + encoding: + codec: json + "#}; let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { - custom_helm_values: vec![ - &config_override_name(&override_name, true), - config, - HELM_VALUES_STDOUT_SINK, - ], + custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() }, ) @@ -1377,7 +1356,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly collects logs from multiple +/// This test validates that vector properly collects logs from multiple /// `Namespace`s and `Pod`s. #[tokio::test] async fn multiple_ns() -> Result<(), Box> { @@ -1393,13 +1372,13 @@ async fn multiple_ns() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -1515,11 +1494,11 @@ async fn multiple_ns() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent helm chart properly allows +/// This test validates that vector helm chart properly allows /// configuration via an additional config file, i.e. it can combine the managed /// and custom config files. #[tokio::test] -async fn additional_config_file() -> Result<(), Box> { +async fn existing_config_file() -> Result<(), Box> { let _guard = lock(); init(); @@ -1531,13 +1510,13 @@ async fn additional_config_file() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_ADDITIONAL_CONFIGMAP, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_EXISTING_CONFIGMAP, ], custom_resource: CUSTOM_RESOURCE_VECTOR_CONFIG, }, @@ -1617,7 +1596,7 @@ async fn additional_config_file() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent properly exposes metrics in +/// This test validates that vector properly exposes metrics in /// a Prometheus scraping format. #[tokio::test] async fn metrics_pipeline() -> Result<(), Box> { @@ -1632,13 +1611,13 @@ async fn metrics_pipeline() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_STDOUT_SINK, - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -1772,7 +1751,7 @@ async fn metrics_pipeline() -> Result<(), Box> { Ok(()) } -/// This test validates that vector-agent chart properly exposes host metrics +/// This test validates that vector chart properly exposes host metrics /// out of the box. #[tokio::test] async fn host_metrics() -> Result<(), Box> { @@ -1786,12 +1765,13 @@ async fn host_metrics() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), - HELM_VALUES_LOWER_GLOB, + HELM_VALUES_AGENT, ], ..Default::default() }, @@ -1848,20 +1828,17 @@ async fn simple_checkpoint() -> Result<(), Box> { let vector = framework .helm_chart( "test-vector", - "vector-agent", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { - custom_helm_values: vec![HELM_VALUES_STDOUT_SINK, HELM_VALUES_LOWER_GLOB], + custom_helm_values: vec![HELM_VALUES_AGENT], ..Default::default() }, ) .await?; framework - .wait_for_rollout( - "test-vector", - "daemonset/vector-agent", - vec!["--timeout=60s"], - ) + .wait_for_rollout("test-vector", "daemonset/vector", vec!["--timeout=60s"]) .await?; let test_namespace = framework @@ -1890,7 +1867,7 @@ async fn simple_checkpoint() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs("test-vector", "daemonset/vector-agent")?; + let mut log_reader = framework.logs("test-vector", "daemonset/vector")?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -1921,19 +1898,15 @@ async fn simple_checkpoint() -> Result<(), Box> { assert!(got_marker); framework - .restart_rollout("test-vector", "daemonset/vector-agent", vec![]) + .restart_rollout("test-vector", "daemonset/vector", vec![]) .await?; // We need to wait for the new pod to start framework - .wait_for_rollout( - "test-vector", - "daemonset/vector-agent", - vec!["--timeout=60s"], - ) + .wait_for_rollout("test-vector", "daemonset/vector", vec!["--timeout=60s"]) .await?; got_marker = false; // We need to start reading from the newly started pod - let mut log_reader = framework.logs("test-vector", "daemonset/vector-agent")?; + let mut log_reader = framework.logs("test-vector", "daemonset/vector")?; look_for_log_line(&mut log_reader, |val| { if val["kubernetes"]["pod_namespace"] != "test-vector-test-pod" { return FlowControlCommand::GoOn; diff --git a/lib/k8s-e2e-tests/tests/vector-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-aggregator.rs index f5347e3add879..a3bb1201dd82b 100644 --- a/lib/k8s-e2e-tests/tests/vector-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-aggregator.rs @@ -1,25 +1,8 @@ -use indoc::indoc; use k8s_e2e_tests::*; use k8s_test_framework::{lock, vector::Config as VectorConfig}; -const HELM_VALUES_DUMMY_TOPOLOGY: &str = indoc! {r#" - sources: - dummy: - type: "demo_logs" - format: "shuffle" - lines: ["Hello world"] - interval: 60 # once a minute - - sinks: - stdout: - type: "console" - inputs: ["dummy"] - target: "stdout" - encoding: "json" -"#}; - -/// This test validates that vector-aggregator can deploy with the default -/// settings and a dummy topology. +/// This test validates that vector can deploy with the default +/// aggregator settings. #[tokio::test] async fn dummy_topology() -> Result<(), Box> { init(); @@ -32,13 +15,11 @@ async fn dummy_topology() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-aggregator", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { - custom_helm_values: vec![ - &config_override_name(&override_name, false), - HELM_VALUES_DUMMY_TOPOLOGY, - ], + custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() }, ) @@ -69,8 +50,9 @@ async fn metrics_pipeline() -> Result<(), Box> { let vector = framework .helm_chart( &namespace, - "vector-aggregator", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs index cd7ea5919457a..55f4208d6783e 100644 --- a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs @@ -3,27 +3,6 @@ use k8s_e2e_tests::*; use k8s_test_framework::{lock, namespace, test_pod, vector::Config as VectorConfig}; use serde_json::Value; -const HELM_VALUES_DDOG_AGG_TOPOLOGY: &str = indoc! {r#" - service: - type: ClusterIP - ports: - - name: datadog - port: 8080 - protocol: TCP - targetPort: 8080 - sources: - datadog-agent: - type: datadog_agent - address: 0.0.0.0:8080 - - sinks: - stdout: - type: console - inputs: ["datadog-agent"] - target: stdout - encoding: json -"#}; - /// This test validates that vector-aggregator can deploy with the default /// settings and a dummy topology. #[tokio::test] @@ -60,8 +39,7 @@ async fn datadog_to_vector() -> Result<(), Box> { kubelet_tls_verify: false logs_config.use_http: true logs_config.logs_no_ssl: true - logs_config.logs_dd_url: {}:8080 - logs_config.use_v2_api: false + logs_config.logs_dd_url: {}:8282 listeners: - name: kubelet config_providers: @@ -76,13 +54,11 @@ async fn datadog_to_vector() -> Result<(), Box> { let _vector = framework .helm_chart( &namespace, - "vector-aggregator", - "https://packages.timber.io/helm/nightly/", + "vector", + "vector", + "https://helm.vector.dev", VectorConfig { - custom_helm_values: vec![ - &config_override_name(&override_name, false), - HELM_VALUES_DDOG_AGG_TOPOLOGY, - ], + custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() }, ) @@ -99,6 +75,7 @@ async fn datadog_to_vector() -> Result<(), Box> { .helm_chart( &datadog_namespace, "datadog", + "datadog", "https://helm.datadoghq.com", // VectorConfig is a generic config container VectorConfig { diff --git a/lib/k8s-e2e-tests/tests/vector.rs b/lib/k8s-e2e-tests/tests/vector.rs index 655f9862fe8e2..e90e1c63e1a22 100644 --- a/lib/k8s-e2e-tests/tests/vector.rs +++ b/lib/k8s-e2e-tests/tests/vector.rs @@ -1,168 +1,61 @@ -use indoc::formatdoc; +use indoc::{formatdoc, indoc}; use k8s_e2e_tests::*; use k8s_test_framework::{ lock, namespace, test_pod, vector::Config as VectorConfig, wait_for_resource::WaitFor, }; -fn helm_values_stdout_sink(aggregator_override_name: &str, agent_override_name: &str) -> String { - if is_multinode() { - formatdoc!( - r#" - global: - vector: - commonEnvKV: - VECTOR_REQUIRE_HEALTHY: true - - vector-agent: - fullnameOverride: "{}" - kubernetesLogsSource: - rawConfig: | - glob_minimum_cooldown_ms = 5000 - vectorSink: - host: "{}" - dataVolume: - hostPath: - path: /var/lib/{}-vector/ - - extraVolumeMounts: - - name: var-lib - mountPath: /var/writablelib - readOnly: false - - lifecycle: - preStop: - exec: - command: - - sh - - -c - - rm -rf /var/writablelib/{}-vector - - vector-aggregator: - fullnameOverride: "{}" - vectorSource: - sourceId: vector - - sinks: - stdout: - type: "console" - inputs: ["vector"] - target: "stdout" - encoding: "json" - "#, - agent_override_name, - aggregator_override_name, - agent_override_name, - agent_override_name, - aggregator_override_name - ) - } else { - formatdoc!( - r#" - global: - vector: - commonEnvKV: - VECTOR_REQUIRE_HEALTHY: true - - vector-agent: - fullnameOverride: "{}" - kubernetesLogsSource: - rawConfig: | - glob_minimum_cooldown_ms = 5000 - vectorSink: - host: "{}" - - vector-aggregator: - fullnameOverride: "{}" - vectorSource: - sourceId: vector - +fn helm_values_stdout_sink(agent_override_name: &str) -> String { + formatdoc!( + r#" + role: Agent + fullnameOverride: "{}" + env: + - name: VECTOR_REQUIRE_HEALTHY + value: true + customConfig: + data_dir: "/vector-data-dir" + api: + enabled: true + address: 127.0.0.1:8686 + sources: + kubernetes_logs: + type: kubernetes_logs sinks: - stdout: - type: "console" - inputs: ["vector"] - target: "stdout" - encoding: "json" + vector: + type: vector + inputs: [kubernetes_logs] + address: aggregator-vector:6000 + version: "2" "#, - agent_override_name, - aggregator_override_name, - aggregator_override_name - ) - } + agent_override_name, + ) } -fn helm_values_haproxy(aggregator_override_name: &str, agent_override_name: &str) -> String { - if is_multinode() { - formatdoc!( - r#" - global: - vector: - commonEnvKV: - VECTOR_REQUIRE_HEALTHY: true - - vector-agent: - fullnameOverride: "{}" - kubernetesLogsSource: - rawConfig: | - glob_minimum_cooldown_ms = 5000 - vectorSink: - host: "{}-haproxy" - dataVolume: - hostPath: - path: /var/lib/{}-vector/ - - vector-aggregator: - fullnameOverride: "{}" - vectorSource: - sourceId: vector - - sinks: - stdout: - type: "console" - inputs: ["vector"] - target: "stdout" - encoding: "json" - - haproxy: +fn helm_values_haproxy(agent_override_name: &str) -> String { + formatdoc!( + r#" + role: Agent + fullnameOverride: "{}" + env: + - name: VECTOR_REQUIRE_HEALTHY + value: true + customConfig: + data_dir: "/vector-data-dir" + api: enabled: true - "#, - agent_override_name, - aggregator_override_name, - agent_override_name, - aggregator_override_name - ) - } else { - formatdoc!( - r#" - global: - vector: - commonEnvKV: - VECTOR_REQUIRE_HEALTHY: true - - vector-agent: - fullnameOverride: "{}" - vectorSink: - host: "{}-haproxy" - - vector-aggregator: - fullnameOverride: "{}" - vectorSource: - sourceId: vector - + address: 127.0.0.1:8686 + sources: + kubernetes_logs: + type: kubernetes_logs sinks: - stdout: - type: "console" - inputs: ["vector"] - target: "stdout" - encoding: "json" - - haproxy: - enabled: true + vector: + type: vector + inputs: [kubernetes_logs] + address: aggregator-vector-haproxy:6000 + version: "2" "#, - agent_override_name, - aggregator_override_name, - aggregator_override_name - ) - } + agent_override_name, + ) } /// This test validates that vector picks up logs with an agent and @@ -175,19 +68,15 @@ async fn logs() -> Result<(), Box> { let namespace = get_namespace(); let pod_namespace = get_namespace_appended(&namespace, "test-pod"); let framework = make_framework(); - let aggregator_override_name = get_override_name(&namespace, "vector-aggregator"); let agent_override_name = get_override_name(&namespace, "vector-agent"); - let vector = framework + let vector_aggregator = framework .helm_chart( &namespace, "vector", - "https://packages.timber.io/helm/nightly/", + "aggregator", + "https://helm.vector.dev", VectorConfig { - custom_helm_values: vec![&helm_values_stdout_sink( - &aggregator_override_name, - &agent_override_name, - )], ..Default::default() }, ) @@ -196,15 +85,28 @@ async fn logs() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", agent_override_name), + &format!("statefulset/aggregator-vector"), vec!["--timeout=60s"], ) .await?; + let vector_agent = framework + .helm_chart( + &namespace, + "vector", + "agent", + "https://helm.vector.dev", + VectorConfig { + custom_helm_values: vec![&helm_values_stdout_sink(&agent_override_name)], + ..Default::default() + }, + ) + .await?; + framework .wait_for_rollout( &namespace, - &format!("statefulset/{}", aggregator_override_name), + &format!("daemonset/{}", agent_override_name), vec!["--timeout=60s"], ) .await?; @@ -234,10 +136,7 @@ async fn logs() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs( - &namespace, - &format!("statefulset/{}", aggregator_override_name), - )?; + let mut log_reader = framework.logs(&namespace, &format!("statefulset/aggregator-vector"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -270,33 +169,36 @@ async fn logs() -> Result<(), Box> { drop(test_pod); drop(test_namespace); - drop(vector); + drop(vector_agent); + drop(vector_aggregator); Ok(()) } /// This test validates that vector picks up logs with an agent and /// delivers them to the aggregator through an HAProxy load balancer. #[tokio::test] -async fn logs_haproxy() -> Result<(), Box> { +async fn haproxy() -> Result<(), Box> { let _guard = lock(); init(); let namespace = get_namespace(); let pod_namespace = get_namespace_appended(&namespace, "test-pod"); let framework = make_framework(); - let aggregator_override_name = get_override_name(&namespace, "vector-aggregator"); let agent_override_name = get_override_name(&namespace, "vector-agent"); - let vector = framework + const CONFIG: &str = indoc! {r#" + haproxy: + enabled: true + "#}; + + let vector_aggregator = framework .helm_chart( &namespace, "vector", - "https://packages.timber.io/helm/nightly/", + "aggregator", + "https://helm.vector.dev", VectorConfig { - custom_helm_values: vec![&helm_values_haproxy( - &aggregator_override_name, - &agent_override_name, - )], + custom_helm_values: vec![CONFIG], ..Default::default() }, ) @@ -305,7 +207,7 @@ async fn logs_haproxy() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", agent_override_name), + &format!("statefulset/aggregator-vector"), vec!["--timeout=60s"], ) .await?; @@ -313,7 +215,28 @@ async fn logs_haproxy() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("statefulset/{}", aggregator_override_name), + &format!("deployment/aggregator-vector-haproxy"), + vec!["--timeout=60s"], + ) + .await?; + + let vector_agent = framework + .helm_chart( + &namespace, + "vector", + "agent", + "https://helm.vector.dev", + VectorConfig { + custom_helm_values: vec![&helm_values_haproxy(&agent_override_name)], + ..Default::default() + }, + ) + .await?; + + framework + .wait_for_rollout( + &namespace, + &format!("daemonset/{}", agent_override_name), vec!["--timeout=60s"], ) .await?; @@ -343,10 +266,7 @@ async fn logs_haproxy() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs( - &namespace, - &format!("statefulset/{}", aggregator_override_name), - )?; + let mut log_reader = framework.logs(&namespace, &format!("statefulset/aggregator-vector"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -379,6 +299,7 @@ async fn logs_haproxy() -> Result<(), Box> { drop(test_pod); drop(test_namespace); - drop(vector); + drop(vector_agent); + drop(vector_aggregator); Ok(()) } diff --git a/lib/k8s-test-framework/src/framework.rs b/lib/k8s-test-framework/src/framework.rs index 93ed45fd735d6..42bc7399221a6 100644 --- a/lib/k8s-test-framework/src/framework.rs +++ b/lib/k8s-test-framework/src/framework.rs @@ -24,6 +24,7 @@ impl Framework { &self, namespace: &str, helm_chart: &str, + release_name: &str, helm_repo: &str, config: vector::Config<'_>, ) -> Result> { @@ -32,6 +33,7 @@ impl Framework { self.interface.deploy_chart_command.as_str(), namespace, helm_chart, + release_name, config, Some(env), )?; diff --git a/lib/k8s-test-framework/src/vector.rs b/lib/k8s-test-framework/src/vector.rs index fa9b8910168e6..55f6dbabc7f98 100644 --- a/lib/k8s-test-framework/src/vector.rs +++ b/lib/k8s-test-framework/src/vector.rs @@ -11,6 +11,7 @@ pub struct CommandBuilder { interface_command: String, namespace: String, helm_chart: String, + release_name: String, custom_helm_values_files: Vec, custom_resource_file: Option, custom_env: Option>, @@ -26,6 +27,7 @@ impl up_down::CommandBuilder for CommandBuilder { }) .arg(&self.namespace) .arg(&self.helm_chart) + .arg(&self.release_name) .stdin(Stdio::null()); command.env( @@ -68,6 +70,7 @@ pub fn manager( interface_command: &str, namespace: &str, helm_chart: &str, + release_name: &str, config: Config<'_>, custom_env: Option>, ) -> Result> { @@ -88,6 +91,7 @@ pub fn manager( interface_command: interface_command.to_owned(), namespace: namespace.to_owned(), helm_chart: helm_chart.to_owned(), + release_name: release_name.to_owned(), custom_helm_values_files, custom_resource_file, custom_env, diff --git a/scripts/deploy-chart-test.sh b/scripts/deploy-chart-test.sh index 4f05a222a3a97..3021052a1bbf1 100755 --- a/scripts/deploy-chart-test.sh +++ b/scripts/deploy-chart-test.sh @@ -15,7 +15,7 @@ set -euo pipefail # # Deploy: # -# $ CHART_REPO=https://helm.testmaterial.tld scripts/deploy-public-chart-test.sh up test-namespace-qwerty chart +# $ CHART_REPO=https://helm.testmaterial.tld scripts/deploy-public-chart-test.sh up test-namespace-qwerty chart release # # Teardown: # @@ -33,6 +33,9 @@ NAMESPACE="${2:?"Specify the namespace as the second argument"}" # The helm chart to manage HELM_CHART="${3:?"Specify the helm chart name as the third argument"}" +# Release name for chart install +RELEASE_NAME="${4:?"Specify the release name as the fourth argument"}" + # Allow overriding kubectl with something like `minikube kubectl --`. VECTOR_TEST_KUBECTL="${VECTOR_TEST_KUBECTL:-"kubectl"}" @@ -61,7 +64,7 @@ up() { $VECTOR_TEST_HELM repo add "$CUSTOM_HELM_REPO_LOCAL_NAME" "$CHART_REPO" --force-update || true $VECTOR_TEST_HELM repo update - $VECTOR_TEST_KUBECTL create namespace "$NAMESPACE" + $VECTOR_TEST_KUBECTL create namespace "$NAMESPACE" --dry-run -o yaml | $VECTOR_TEST_KUBECTL apply -f - if [[ -n "$CUSTOM_RESOURCE_CONFIGS_FILE" ]]; then $VECTOR_TEST_KUBECTL create --namespace "$NAMESPACE" -f "$CUSTOM_RESOURCE_CONFIGS_FILE" @@ -80,9 +83,10 @@ up() { # overwriting console output. split-container-image "$CONTAINER_IMAGE" HELM_VALUES+=( - --set "global.vector.commonEnvKV.LOG=info" - --set "global.vector.image.repository=$CONTAINER_IMAGE_REPOSITORY" - --set "global.vector.image.tag=$CONTAINER_IMAGE_TAG" + --set "env[0].name=VECTOR_LOG" + --set "env[0].value=info" + --set "image.repository=$CONTAINER_IMAGE_REPOSITORY" + --set "image.tag=$CONTAINER_IMAGE_TAG" ) set -x @@ -90,9 +94,8 @@ up() { --atomic \ --namespace "$NAMESPACE" \ "${HELM_VALUES[@]}" \ - "$HELM_CHART" \ - "$CUSTOM_HELM_REPO_LOCAL_NAME/$HELM_CHART" \ - --devel + "$RELEASE_NAME" \ + "$CUSTOM_HELM_REPO_LOCAL_NAME/$HELM_CHART" { set +x; } &>/dev/null } @@ -105,7 +108,9 @@ down() { $VECTOR_TEST_HELM delete --namespace "$NAMESPACE" "$HELM_CHART" fi - $VECTOR_TEST_KUBECTL delete namespace "$NAMESPACE" + if $VECTOR_TEST_KUBECTL get namespace "$NAMESPACE" &>/dev/null; then + $VECTOR_TEST_KUBECTL delete namespace "$NAMESPACE" + fi } case "$COMMAND" in