From 66cc6ff52dc7fa85ea73d50027693d82892155f4 Mon Sep 17 00:00:00 2001 From: manrodrigues Date: Mon, 11 Dec 2023 20:14:51 -0300 Subject: [PATCH 1/3] initial tests for otlp backend --- python-test/README.md | 2 +- .../integration_config_file_otel.feature | 312 ++++++++++++++++++ ... integration_config_file_pktvisor.feature} | 4 +- python-test/features/steps/agent_groups.py | 8 +- python-test/features/steps/agents.py | 113 +++---- python-test/features/steps/configs.py | 4 +- python-test/features/steps/local_agent.py | 82 ++--- python-test/features/steps/policies.py | 5 +- python-test/features/steps/sink.py | 5 +- 9 files changed, 394 insertions(+), 141 deletions(-) create mode 100644 python-test/features/integration_config_file_otel.feature rename python-test/features/{integration_config_file.feature => integration_config_file_pktvisor.feature} (99%) diff --git a/python-test/README.md b/python-test/README.md index 8224530ae..8829d079f 100644 --- a/python-test/README.md +++ b/python-test/README.md @@ -88,7 +88,7 @@ Then fill in the correct values: - Bool - If true, uses orb_address as base to api and mqtt address using orb.live pattern. If false, requires you to add the corresponding addresses. - Default value: `true` -- **backend_type**: +- **sink_backend_type**: - Str - Sink backend type - Default value: `prometheus` diff --git a/python-test/features/integration_config_file_otel.feature b/python-test/features/integration_config_file_otel.feature new file mode 100644 index 000000000..a0ee05f17 --- /dev/null +++ b/python-test/features/integration_config_file_otel.feature @@ -0,0 +1,312 @@ +@integration_config_files_otel_backend @AUTORETRY +Feature: Integration tests using agent with otlp as backend provided via config file + + +@smoke_otel_backend +Scenario: provisioning agent with otel backend and applying 1 policy (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 1 orb tag(s) (lower case) + And 1 policies with otel backend and yaml format are applied to the group + And a new agent is created with 0 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: True] + And otel state is running + Then 1 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 1 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 180 seconds + And remove the agent .yaml generated on each scenario + + +@smoke_otel_backend +Scenario: provisioning agent with otel backend and applying 2 policies (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 1 orb tag(s) (lower case) + And 2 policies with otel backend and yaml format are applied to the group + And a new agent is created with 0 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: True] + And otel state is running + Then 2 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 2 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 180 seconds + And remove the agent .yaml generated on each scenario + + +@smoke_otel_backend +Scenario: removing policy from agent with otel backend (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 1 orb tag(s) (lower case) + And 2 policies with otel backend and yaml format are applied to the group + And a new agent is created with 0 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: True] + And otel state is running + Then 2 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 2 policies are applied and all has status running + When one of applied policies is removed + Then referred policy must not be listed on the orb policies list + And no dataset should be linked to the removed policy anymore + And 1 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 policies are applied and all has status running + And container logs should inform that removed policy was stopped and removed within 30 seconds + And the container logs that were output after the policy have been removed contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after the policy have been removed does not contain the message "scraped and published telemetry" referred to deleted policy anymore + + +@smoke_otel_backend +Scenario: Remove group to which agent with otel backend is linked + Given the Orb user has a registered account + And the Orb user logs in + And a new agent is created with 0 orb tag(s) + And an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with 3 agent tags and has status online. [Overwrite default: False. Paste only file: True] + And otel state is waiting + And referred agent is subscribed to 1 group + And otel state is waiting + And this agent's heartbeat shows that 1 groups are matching the agent + And that a sink with default configuration type already exists + And 2 policies with otel backend and yaml format are applied to the group + And this agent's heartbeat shows that 2 policies are applied and all has status running + When 1 group(s) to which the agent is linked is removed + Then the container logs should contain the message "completed RPC unsubscription to group" within 30 seconds + And the container logs contain the message "policy no longer used by any group, removing" referred to each policy within 30 seconds + And this agent's heartbeat shows that 0 policies are applied to the agent + And this agent's heartbeat shows that 0 groups are matching the agent + And no dataset should be linked to the removed group anymore + And 0 dataset(s) have validity valid and 2 have validity invalid in 30 seconds + + +@smoke_otel_backend +Scenario: Remotely restart agent with otel backend and policies applied + Given the Orb user has a registered account + And the Orb user logs in + And 1 Agent Group(s) is created with 1 orb tag(s) (lower case) + And that a sink with default configuration type already exists + And 1 policies with otel backend and yaml format are applied to the group + And a new agent is created with 0 orb tag(s) + And an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: True] + And this agent's heartbeat shows that 1 groups are matching the agent + And otel state is running + And this agent's heartbeat shows that 1 policies are applied and all has status running + When remotely restart the agent + Then the container logs that were output after reset the agent contain the message "otel process stopped" within 30 seconds + And the container logs should contain the message "all backends and comms were restarted" within 30 seconds + And the container logs that were output after reset the agent contain the message "removing policies" within 30 seconds + And the container logs that were output after reset the agent contain the message "resetting backend" within 30 seconds + And the container logs that were output after reset the agent contain the message "all backends and comms were restarted" within 30 seconds + And the container logs that were output after reset the agent contain the message "policy applied successfully" referred to each applied policy within 30 seconds + And the container logs that were output after reset the agent contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds + + +@smoke_otel_backend @config_file @auto_provision +Scenario: agent otel with only agent tags subscription to a group with policies created after provision the agent (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + When an agent(backend_type:otel, settings: {}) is self-provisioned via a configuration file on port available with 3 agent tags and has status online. [Overwrite default: False. Paste only file: True] + And 1 Agent Group(s) is created with all tags contained in the agent + And 2 policies with otel backend and yaml format are applied to the group + And otel state is running + Then 2 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And this agent's heartbeat shows that 2 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + + +@smoke_otel_backend @config_file @auto_provision +Scenario: agent otel with only agent tags subscription to a group with policies created before provision the agent (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 1 orb tag(s) (lower case) + And 3 simple policies otel are applied to the group + And a new agent is created with 0 orb tag(s) + When an agent(backend_type:otel, settings: {}) is self-provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: False] + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + + +@smoke_otel_backend @config_file @auto_provision +Scenario: agent otel with mixed tags subscription to a group with policies created after provision the agent (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + When an agent(backend_type:otel, settings: {}) is self-provisioned via a configuration file on port available with 3 agent tags and has status online. [Overwrite default: False. Paste only file: True] + And otel state is waiting + And edit the orb tags on agent and use 2 orb tag(s) + And 1 Agent Group(s) is created with all tags contained in the agent + And otel state is waiting + And 3 policies with otel backend and yaml format are applied to the group + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + + +@smoke_otel_backend @config_file @auto_provision +Scenario: agent otel with mixed tags subscription to a group with policies created before provision the agent (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 2 orb tag(s) (lower case) + And 3 simple policies otel are applied to the group + And a new agent is created with 2 orb tag(s) + When an agent(backend_type:otel, settings: {}) is self-provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: False] + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + +@smoke_otel_backend @config_file +Scenario: agent otel with only agent tags subscription to a group with policies created after provision the agent (config file - auto_provision=false) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And a new agent is created with 0 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with 3 agent tags and has status online. [Overwrite default: False. Paste only file: True] + And otel state is waiting + And 1 Agent Group(s) is created with all tags contained in the agent + And 3 policies with otel backend and yaml format are applied to the group + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + +#@smoke_otel_backend @config_file +@MUTE +Scenario: agent otel with only agent tags subscription to a group with policies created before provision the agent (config file - auto_provision=false) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 1 orb tag(s) (lower case) + And 3 simple policies otel are applied to the group + And a new agent is created with 0 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: False] + And otel state is waiting + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And otel state is running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + +@smoke_otel_backend @config_file +Scenario: agent otel with mixed tags subscription to a group with policies created after provision the agent (config file - auto_provision=false) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And a new agent is created with 2 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with 3 agent tags and has status online. [Overwrite default: False. Paste only file: False] + And edit the orb tags on agent and use 2 orb tag(s) + And 1 Agent Group(s) is created with all tags contained in the agent + And 3 policies with otel backend and yaml format are applied to the group + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + +#@smoke_otel_backend @config_file +@MUTE +Scenario: agent otel with mixed tags subscription to a group with policies created before provision the agent (config file - auto_provision=false) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 2 orb tag(s) (lower case) + And 3 simple policies otel are applied to the group + And a new agent is created with 2 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: False. Paste only file: False] + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And referred sink must have active state on response within 120 seconds + And remove the agent .yaml generated on each scenario + + +########### provisioning agents without specify otel configs on backend + + +@smoke_otel_backend @config_file @otel_configs @auto_provision +Scenario: provisioning agent without specify path to otel config file (config file - auto_provision=true) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And 1 Agent Group(s) is created with 1 orb tag(s) (lower case) + And 3 policies with otel backend and yaml format are applied to the group + And a new agent is created with 0 orb tag(s) + When an agent(backend_type:otel, settings: {}) is self-provisioned via a configuration file on port available with matching 1 group agent tags and has status online. [Overwrite default: True. Paste only file: True. Use specif backend config {"config_file":"None"}] + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And remove the agent .yaml generated on each scenario + + +@smoke_otel_backend @config_file @otel_configs +Scenario: provisioning agent without specify path to otel config file (config file - auto_provision=false) + Given the Orb user has a registered account + And the Orb user logs in + And that a sink with default configuration type already exists + And a new agent is created with 2 orb tag(s) + When an agent(backend_type:otel, settings: {}) is provisioned via a configuration file on port available with 3 agent tags and has status online. [Overwrite default: True. Paste only file: True. Use specif backend config {"config_file":"None"}] + And edit the orb tags on agent and use 2 orb tag(s) + And 1 Agent Group(s) is created with all tags contained in the agent + And 3 policies with otel backend and yaml format are applied to the group + And otel state is running + Then 3 dataset(s) have validity valid and 0 have validity invalid in 30 seconds + And this agent's heartbeat shows that 1 groups are matching the agent + And the container logs should contain the message "completed RPC subscription to group" within 30 seconds + And this agent's heartbeat shows that 3 policies are applied and all has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds + And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And remove the agent .yaml generated on each scenario diff --git a/python-test/features/integration_config_file.feature b/python-test/features/integration_config_file_pktvisor.feature similarity index 99% rename from python-test/features/integration_config_file.feature rename to python-test/features/integration_config_file_pktvisor.feature index abe3c58eb..cfb4b8752 100644 --- a/python-test/features/integration_config_file.feature +++ b/python-test/features/integration_config_file_pktvisor.feature @@ -1,5 +1,5 @@ @integration_config_files @AUTORETRY -Feature: Integration tests using agent provided via config file +Feature: Integration tests using agent with pktvisor as backend provided via config file @private @auto_provision @@ -79,7 +79,7 @@ Scenario: provisioning agent without specify pktvisor path to config file (confi And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And remove the agent .yaml generated on each scenario -@smoke @config_file @pktvisor_configs @bla +@smoke @config_file @pktvisor_configs Scenario: provisioning agent without specify pktvisor binary path and path to config file (config file - auto_provision=false) Given the Orb user has a registered account And the Orb user logs in diff --git a/python-test/features/steps/agent_groups.py b/python-test/features/steps/agent_groups.py index ac06353e4..cd50b3e04 100644 --- a/python-test/features/steps/agent_groups.py +++ b/python-test/features/steps/agent_groups.py @@ -8,11 +8,12 @@ return_api_put_response) from behave import given, then, step from hamcrest import * -import requests from random import sample import json import random +from logger import Logger +log = Logger().logger_instance() configs = TestConfig.configs() agent_group_name_prefix = 'test_group_name_' agent_group_description = "This is an agent group" @@ -46,8 +47,8 @@ def create_agent_group_matching_agent(context, amount_of_agent_groups, amount_of f"Agent:{context.agent}") for group in range(int(amount_of_agent_groups)): agent_group_name = agent_group_name_prefix + random_string() - agent_group_data = generate_group_with_valid_json(context.token, agent_group_name, group_description, - tags_to_group, context.agent_groups) + generate_group_with_valid_json(context.token, agent_group_name, group_description, + tags_to_group, context.agent_groups) @step("{amount_of_agent_groups} Agent Group(s) is created with {orb_tags} orb tag(s) (lower case)") @@ -535,6 +536,7 @@ def generate_group_with_valid_json(token, agent_group_name, group_description, t """ agent_group_data = create_agent_group(token, agent_group_name, group_description, tags_to_group) + log.debug(f"Created agent group data: {agent_group_data}") group_id = agent_group_data['id'] agent_groups[group_id] = agent_group_name diff --git a/python-test/features/steps/agents.py b/python-test/features/steps/agents.py index e53e08007..7680983e3 100644 --- a/python-test/features/steps/agents.py +++ b/python-test/features/steps/agents.py @@ -385,72 +385,20 @@ def provision_agent_using_config_file(context, backend_type, settings, provision context.container_id = run_agent_config_file(context.agent_file_name, overwrite_default, paste_only_file) if context.container_id not in context.containers_id.keys(): context.containers_id[context.container_id] = str(context.port) - log = f"web server listening on localhost:{context.port}" - agent_started, logs = get_logs_and_check(context.container_id, log, element_to_check="log") - assert_that(agent_started, equal_to(True), f"Log {log} not found on agent logs. Agent Name: {agent_name}.\n" - f"Logs:{logs}") - context.agent, is_agent_created = check_agent_exists_on_backend(context.token, agent_name, timeout=60) - logs = get_orb_agent_logs(context.container_id) - assert_that(is_agent_created, equal_to(True), f"Agent {agent_name} not found in /agents route." - f"\n Config File (json converted): {safe_config_file}." - f"\nLogs: {logs}.") - context.agent, are_tags_correct = get_agent_tags(context.token, context.agent['id'], tags_on_agent) - assert_that(are_tags_correct, equal_to(True), f"Agent tags created does not match with the required ones. Agent:" - f"{context.agent}. Tags that would be present: {tags_on_agent}.\n" - f"Agent Logs: {logs}") - assert_that(context.agent, is_not(None), f"Agent {agent_name} not correctly created. Logs: {logs}") - agent_id = context.agent['id'] - existing_agents = get_agent(context.token, agent_id) - assert_that(len(existing_agents), greater_than(0), f"Agent not created. Logs: {logs}") - agent_status, context.agent = wait_until_expected_agent_status(context.token, agent_id, status) - assert_that(agent_status, is_(equal_to(status)), - f"Agent did not get '{status}' after 30 seconds, but was '{agent_status}'. \n" - f"Agent: {json.dumps(context.agent, indent=4)}. \n Logs: {logs}") - - -@step("an agent with otel backend is {provision} via a configuration file on port {port} " - "with {agent_tags} agent tags and has status {status}. [Overwrite default: {overwrite_default}. Paste only " - "file: {paste_only_file}]") -def provision_otel_backend_agent_using_config_file(context, provision, port, agent_tags, status, overwrite_default, - paste_only_file, **kwargs): - assert_that(provision, any_of(equal_to("self-provisioned"), equal_to("provisioned")), "Unexpected provision " - "attribute") - overwrite_default = overwrite_default.title() - paste_only_file = paste_only_file.title() - assert_that(overwrite_default, any_of("True", "False"), "Unexpected value for overwrite_default parameter.") - assert_that(paste_only_file, any_of("True", "False"), "Unexpected value for overwrite_default parameter.") - overwrite_default = eval(overwrite_default) - paste_only_file = eval(paste_only_file) - if provision == "provisioned": - auto_provision = "false" - orb_cloud_mqtt_id = context.agent['id'] - orb_cloud_mqtt_key = context.agent['key'] - orb_cloud_mqtt_channel_id = context.agent['channel_id'] - agent_name = context.agent['name'] + if backend_type == "pktvisor": + log_message = f"web server listening on localhost:{context.port}" + agent_started, logs, log_line = get_logs_and_check(context.container_id, log_message, element_to_check="log") + assert_that(agent_started, equal_to(True), f"Log {log_message} not found on agent logs." + f" Agent Name: {agent_name}. Logs:{logs}") else: - auto_provision = "true" - orb_cloud_mqtt_id = None - orb_cloud_mqtt_key = None - orb_cloud_mqtt_channel_id = None - agent_name = f"{agent_name_prefix}{random_string(10)}" - orb_url = configs.get('orb_url') - context.port = return_port_by_availability(context, True) - otel_configs = {"config_file": "default"} - if 'otel_config' in kwargs.keys(): - if "config_file" in kwargs['pkt_config'].keys(): - otel_configs["config_file"] = kwargs['pkt_config']["config_file"] - context.agent_file_name, tags_on_agent, safe_config_file = \ - create_agent_with_otel_backend_config_file(context.token, agent_name, agent_tags, orb_url, context.port, - context.agent_groups, auto_provision, - orb_cloud_mqtt_id, orb_cloud_mqtt_key, orb_cloud_mqtt_channel_id, - overwrite_default, paste_only_file, otel_configs['config_file']) - context.container_id = run_agent_config_file(context.agent_file_name, overwrite_default, paste_only_file) - if context.container_id not in context.containers_id.keys(): - context.containers_id[context.container_id] = str(context.port) - msg = f"Starting GRPC server" - agent_started, logs = get_logs_and_check(context.container_id, msg, element_to_check="msg") - assert_that(agent_started, equal_to(True), f"Log {msg} not found on agent logs. Agent Name: {agent_name}.\n" - f"Logs:{logs}") + log_message = f"Started receiver for OTLP in orb-agent" + agent_started, logs, log_line = get_logs_and_check(context.container_id, log_message, element_to_check="msg") + assert_that(agent_started, equal_to(True), f"Log {log_message} not found on agent logs." + f" Agent Name: {agent_name}. Logs:{logs}") + assert_that(str(log_line.get("port", "")), equal_to(str(context.port)), f"Log {log_message} related to port " + f"{context.port} not found on agent " + f"logs. Logs: {logs}") + context.agent, is_agent_created = check_agent_exists_on_backend(context.token, agent_name, timeout=60) logs = get_orb_agent_logs(context.container_id) assert_that(is_agent_created, equal_to(True), f"Agent {agent_name} not found in /agents route." @@ -626,8 +574,7 @@ def wait_until_expected_backend_state(token, agent_id, backend, state, event=Non :param (obj) event: threading.event """ - agent = get_agent(token, agent_id) - backend_state = agent["last_hb_data"]["backend_state"][backend]['state'] + backend_state, agent = get_backend_info(token, agent_id, backend, "state") if backend_state == state: event.set() return backend_state, agent @@ -647,17 +594,35 @@ def wait_until_expected_backend_error(token, agent_id, backend, error, event=Non :param (obj) event: threading.event """ - agent = get_agent(token, agent_id) - if 'error' in agent["last_hb_data"]["backend_state"][backend].keys(): - backend_error = agent["last_hb_data"]["backend_state"][backend]['error'] - if backend_error == error: - event.set() - return backend_error, agent + backend_error, agent = get_backend_info(token, agent_id, backend, "error") + if backend_error == error: + event.set() return backend_error, agent else: return None, agent +def get_backend_info(token, agent_id, backend, info): + """ + Get the backend state for a specific agent. + + :param str token: Access token for authentication + :param str agent_id: ID of the agent + :param str backend: Name of the agent backend + :param str info: Info requested + :return: The agent backend required info + :rtype: str or None + """ + agent = get_agent(token, agent_id) + try: + last_hb_data = agent.get("last_hb_data", {}) + backend_info = last_hb_data.get("backend_state", {}).get(backend, {}).get(info) + return backend_info, agent + except Exception as e: + log.error(f"Error getting backend {info}: {e}. Agent: {agent}") + return None, agent + + def get_agent(token, agent_id, expected_status_code=200): """ Gets an agent from Orb control plane @@ -783,7 +748,7 @@ def edit_agent(token, agent_id, name, tags, expected_status_code=200): json_request = {"name": name, "orb_tags": tags, "validate_only": False} status_code, response = return_api_put_response(orb_url + '/api/v1/agents/' + agent_id, request_body=json_request, - token=token, verify=verify_ssl_bool) + token=token, verify=verify_ssl_bool) assert_that(status_code, equal_to(expected_status_code), 'Request to edit agent failed with status=' + str(status_code) + ":" + str(response)) diff --git a/python-test/features/steps/configs.py b/python-test/features/steps/configs.py index 571e0b171..20e9e3dd4 100644 --- a/python-test/features/steps/configs.py +++ b/python-test/features/steps/configs.py @@ -46,8 +46,8 @@ def _read_configs(): assert_that(configs.get('password'), has_length(greater_than_or_equal_to(8)), 'Orb password must be at least 8 digits') - configs['backend_type'] = configs.get("backend_type", "prometheus") - if configs['backend_type'] == "otlphttp": + configs['sink_backend_type'] = configs.get("sink_backend_type", "prometheus") + if configs['sink_backend_type'] == "otlphttp": assert_that(configs.get('otlp_publisher_username'), not_none(), 'No otlp_publisher username was provided!') assert_that(configs.get('otlp_publisher_username'), not_(""), 'No otlp_publisher username was provided!') diff --git a/python-test/features/steps/local_agent.py b/python-test/features/steps/local_agent.py index 39a8c020a..fff33ea4c 100644 --- a/python-test/features/steps/local_agent.py +++ b/python-test/features/steps/local_agent.py @@ -11,7 +11,9 @@ from datetime import datetime import ciso8601 from metrics import expected_metrics_by_handlers_and_groups, wait_until_metrics_scraped +from logger import Logger +log = Logger().logger_instance() configs = TestConfig.configs() verify_ssl_bool = eval(configs.get('verify_ssl').title()) @@ -36,8 +38,8 @@ def check_metrics_by_handler(context, handler_type): else: extra_metrics_present = sorted(extra_metrics_present) assert_that(correct_metrics, equal_to(True), f"Metrics are not the expected. " - f"Metrics expected that are not present: {expected_metrics_not_present}." - f"Extra metrics present: {extra_metrics_present}") + f"Metrics expected that are not present: {expected_metrics_not_present}." + f"Extra metrics present: {extra_metrics_present}") @step('the agent container is started on an {status_port} port') @@ -66,7 +68,7 @@ def run_local_agent_container(context, status_port, **kwargs): log = f"web server listening on localhost:{context.port}" else: log = f"unable to bind to localhost:{context.port}" - agent_started, logs = get_logs_and_check(context.container_id, log, element_to_check="log") + agent_started, logs, log_line = get_logs_and_check(context.container_id, log, element_to_check="log") assert_that(agent_started, equal_to(True), f"Log {log} not found on agent logs. Agent Name: {context.agent['name']}." f"\n Logs:{logs}") @@ -100,7 +102,7 @@ def check_agent_logs_considering_timestamp(context, condition, text_to_match, ti considered_timestamp = context.considered_timestamp_reset else: considered_timestamp = context.considered_timestamp - text_found, logs = get_logs_and_check(context.container_id, text_to_match, considered_timestamp, + text_found, logs, log_line = get_logs_and_check(context.container_id, text_to_match, considered_timestamp, timeout=time_to_wait) assert_that(text_found, is_(True), f"Message {text_to_match} was not found in the agent logs!. \n\n" f"Container logs: {json.dumps(logs, indent=4)}") @@ -117,7 +119,7 @@ def check_errors_on_agent_logs(context, type_of_message): @then('the container logs should contain the message "{text_to_match}" within {time_to_wait} seconds') def check_agent_msg_in_logs(context, text_to_match, time_to_wait): - text_found, logs = get_logs_and_check(context.container_id, text_to_match, timeout=time_to_wait) + text_found, logs, log_line = get_logs_and_check(context.container_id, text_to_match, timeout=time_to_wait) assert_that(text_found, is_(True), f"Message {text_to_match} was not found in the agent logs!. \n\n" f"Container logs: {json.dumps(logs, indent=4)}") @@ -126,7 +128,7 @@ def check_agent_msg_in_logs(context, text_to_match, time_to_wait): @then('the container logs should contain "{error_log}" as log within {time_to_wait} seconds') def check_agent_log_in_logs(context, error_log, time_to_wait): error_log = error_log.replace(":port", f":{context.port}") - text_found, logs = get_logs_and_check(context.container_id, error_log, element_to_check="log", timeout=time_to_wait) + text_found, logs, log_line = get_logs_and_check(context.container_id, error_log, element_to_check="log", timeout=time_to_wait) assert_that(text_found, is_(True), f"Log {error_log} was not found in the agent logs!. \n\n" f"Container logs: {json.dumps(logs, indent=4)}") @@ -261,58 +263,30 @@ def get_orb_agent_logs(container_id): return container.logs().decode("utf-8").split("\n") -def check_logs_contain_message(logs, expected_message, event, start_time=0): +def check_logs_contain_entry(logs, element_to_check, expected_entry, start_time=0): """ - Gets the logs from Orb agent container + Check if the logs from Orb agent container contain a specific entry :param (list) logs: list of log lines - :param (str) expected_message: message that we expect to find in the logs - :param (obj) event: threading.event - :param (int) start_time: time to be considered as initial time. Default: None - :returns: (bool) whether expected message was found in the logs + :param (str) element_to_check: key to search in the logs + :param (str) expected_entry: entry that we expect to find in the logs + :param (int) start_time: time to be considered as the initial time. Default: 0 + :returns: (bool) whether the expected entry was found in the logs """ - for log_line in logs: log_line = safe_load_json(log_line) - if log_line is not None and log_line['msg'] == expected_message and isinstance(log_line['ts'], int) and \ - log_line['ts'] > start_time: - event.set() - return event.is_set() - elif log_line is not None and log_line['msg'] == expected_message and isinstance(log_line['ts'], str) and \ - datetime.timestamp(ciso8601.parse_datetime(log_line['ts'])) > start_time: - event.set() - return event.is_set() + if log_line is not None and element_to_check in log_line.keys() and isinstance(log_line['ts'], (int, str)): + log_timestamp = ( + log_line['ts'] + if isinstance(log_line['ts'], int) + else datetime.timestamp(ciso8601.parse_datetime(log_line['ts'])) + ) - return event.is_set() + if expected_entry in log_line.get(element_to_check, '') and log_timestamp > start_time: + return True, log_line - -def check_logs_contain_log(logs, expected_log, event, start_time=0): - """ - Check if the logs from Orb agent container contain specific log - - :param (list) logs: list of log lines - :param (str) expected_log: log that we expect to find in the logs - :param (obj) event: threading.event - :param (int) start_time: time to be considered as initial time. Default: None - :returns: (bool) whether expected message was found in the logs - """ - - for log_line in logs: - log_line = safe_load_json(log_line) - - if log_line is not None and "log" in log_line.keys() and expected_log in log_line['log'] and isinstance( - log_line['ts'], int) and \ - log_line['ts'] > start_time: - event.set() - return event.is_set() - elif log_line is not None and "log" in log_line.keys() and expected_log in log_line['log'] and isinstance( - log_line['ts'], str) and \ - datetime.timestamp(ciso8601.parse_datetime(log_line['ts'])) > start_time: - event.set() - return event.is_set() - - return event.is_set() + return False, None def run_local_agent_from_terminal(command, verify_ssl, pktvisor_port): @@ -386,11 +360,10 @@ def get_logs_and_check(container_id, expected_message, start_time=0, element_to_ """ assert_that(element_to_check, any_of(equal_to("msg"), equal_to("log")), "Unexpected value for element to check.") logs = get_orb_agent_logs(container_id) - if element_to_check == "msg": - text_found = check_logs_contain_message(logs, expected_message, event, start_time) - else: - text_found = check_logs_contain_log(logs, expected_message, event, start_time) - return text_found, logs + message_found, log_line = check_logs_contain_entry(logs, element_to_check, expected_message, start_time) + if message_found is True: + event.set() + return message_found, logs, log_line def run_agent_config_file(agent_name, overwrite_default=False, only_file=False, config_file_path="/opt/orb", @@ -421,6 +394,7 @@ def run_agent_config_file(agent_name, overwrite_default=False, only_file=False, command = f"docker run -d -v {volume} --net=host {agent_image}" else: command = f"docker run -d -v {volume} --net=host {agent_image} run -c {agent_command}" + log.debug(f"Run Agent Command: {command}") args = shlex.split(command) terminal_running = subprocess.Popen(args, stdout=subprocess.PIPE) subprocess_return = terminal_running.stdout.read().decode() diff --git a/python-test/features/steps/policies.py b/python-test/features/steps/policies.py index 38469a9cc..8f3733875 100644 --- a/python-test/features/steps/policies.py +++ b/python-test/features/steps/policies.py @@ -1,5 +1,4 @@ from hamcrest import * -import requests from behave import given, then, step from utils import (random_string, filter_list_by_parameter_start_with, safe_load_json, remove_empty_from_json, \ threading_wait_until, UtilsManager, create_tags_set, is_json, values_to_boolean, @@ -14,7 +13,9 @@ import json import ciso8601 import yaml +from logger import Logger +log = Logger().logger_instance() policy_name_prefix = "test_policy_name_" configs = TestConfig.configs() orb_url = configs.get('orb_url') @@ -363,7 +364,7 @@ def check_agent_logs_for_policies_considering_timestamp(context, condition, text policies_without_message = set(context.list_agent_policies_id).difference(policies_have_expected_message) for policy in policies_without_message: policies_data.append(get_policy(context.token, policy)) - + log.debug(f"Agent logs: {logs}") assert_that(policies_have_expected_message, equal_to(set(context.list_agent_policies_id)), f"Message '{text_to_match}' for policy " f"'{policies_data}'" diff --git a/python-test/features/steps/sink.py b/python-test/features/steps/sink.py index 495db7019..97f749b70 100644 --- a/python-test/features/steps/sink.py +++ b/python-test/features/steps/sink.py @@ -4,7 +4,6 @@ remove_empty_from_json, return_api_get_response, return_api_put_response, return_api_post_response, return_api_delete_response) from hamcrest import * -import requests import threading import yaml @@ -12,12 +11,11 @@ sink_name_prefix = "test_sink_label_name_" orb_url = configs.get('orb_url') verify_ssl_bool = eval(configs.get('verify_ssl').title()) -backend_type = configs.get('backend_type') @given("that the user has the prometheus/grafana credentials") def check_prometheus_grafana_credentials(context): - if backend_type == "otlphttp": + if configs.get("sink_backend_type") == "otlphttp": context.remote_prometheus_endpoint = configs.get('otlp_publisher_endpoint') assert_that(context.remote_prometheus_endpoint, not_none(), 'No remote write endpoint to send otlp ' 'metrics' @@ -53,6 +51,7 @@ def check_prometheus_grafana_credentials(context): @step("a new sink is created") def create_sink(context, **kwargs): + backend_type = configs.get("sink_backend_type") sink_label_name = sink_name_prefix + random_string(10) token = context.token endpoint = context.remote_prometheus_endpoint From ec1c16a5acf4ec6a865aa14734f1036b639e3f70 Mon Sep 17 00:00:00 2001 From: manrodrigues Date: Mon, 11 Dec 2023 20:20:11 -0300 Subject: [PATCH 2/3] initial tests for otlp backend --- .../features/integration_config_file_otel.feature | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python-test/features/integration_config_file_otel.feature b/python-test/features/integration_config_file_otel.feature index a0ee05f17..3584490ef 100644 --- a/python-test/features/integration_config_file_otel.feature +++ b/python-test/features/integration_config_file_otel.feature @@ -123,6 +123,7 @@ Scenario: agent otel with only agent tags subscription to a group with policies And this agent's heartbeat shows that 2 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -143,6 +144,7 @@ Scenario: agent otel with only agent tags subscription to a group with policies And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -165,6 +167,7 @@ Scenario: agent otel with mixed tags subscription to a group with policies creat And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -185,6 +188,7 @@ Scenario: agent otel with mixed tags subscription to a group with policies creat And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -205,6 +209,7 @@ Scenario: agent otel with only agent tags subscription to a group with policies And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -226,6 +231,7 @@ Scenario: agent otel with only agent tags subscription to a group with policies And otel state is running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -246,6 +252,7 @@ Scenario: agent otel with mixed tags subscription to a group with policies creat And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -266,6 +273,7 @@ Scenario: agent otel with mixed tags subscription to a group with policies creat And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario @@ -289,6 +297,7 @@ Scenario: provisioning agent without specify path to otel config file (config fi And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And remove the agent .yaml generated on each scenario @@ -309,4 +318,5 @@ Scenario: provisioning agent without specify path to otel config file (config fi And this agent's heartbeat shows that 3 policies are applied and all has status running And the container logs contain the message "policy applied successfully" referred to each policy within 30 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds + And the container logs that were output after all policies have been applied contain the message "scraped and published telemetry" referred to each applied policy within 180 seconds And remove the agent .yaml generated on each scenario From 58ea2a7fb66da9ea7759c502505c1165fe835a05 Mon Sep 17 00:00:00 2001 From: manrodrigues Date: Mon, 11 Dec 2023 20:55:51 -0300 Subject: [PATCH 3/3] initial tests for otlp backend --- python-test/features/integration_config_file_otel.feature | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python-test/features/integration_config_file_otel.feature b/python-test/features/integration_config_file_otel.feature index 3584490ef..fb39d084e 100644 --- a/python-test/features/integration_config_file_otel.feature +++ b/python-test/features/integration_config_file_otel.feature @@ -213,8 +213,8 @@ Scenario: agent otel with only agent tags subscription to a group with policies And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario -#@smoke_otel_backend @config_file -@MUTE + +@smoke_otel_backend @config_file Scenario: agent otel with only agent tags subscription to a group with policies created before provision the agent (config file - auto_provision=false) Given the Orb user has a registered account And the Orb user logs in @@ -256,8 +256,7 @@ Scenario: agent otel with mixed tags subscription to a group with policies creat And referred sink must have active state on response within 120 seconds And remove the agent .yaml generated on each scenario -#@smoke_otel_backend @config_file -@MUTE +@smoke_otel_backend @config_file Scenario: agent otel with mixed tags subscription to a group with policies created before provision the agent (config file - auto_provision=false) Given the Orb user has a registered account And the Orb user logs in