Automated tests for delete a policy applied to an agent

python-test/features/agentsProvider.feature

@@ -9,18 +9,3 @@ Feature: agent provider
         Then the agent status in Orb should be online
             And the container logs should contain the message "sending capabilities" within 10 seconds
 
-
-    Scenario: Apply two policies to an agent
-        Given the Orb user has a registered account
-            And the Orb user logs in
-            And that an agent already exists and is online
-            And referred agent is subscribed to a group
-            And that a sink already exists
-            And that a policy already exists
-            And that a dataset using referred group, sink and policy ID already exists
-        When a new policy is created
-            And a new dataset is created using referred group, sink and policy ID
-        Then this agent's heartbeat shows that all 2 policies have been successfully applied
-            And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds
-            And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each policy within 180 seconds
-            And referred sink must have active state on response within 10 seconds

python-test/features/datasets.feature

@@ -12,4 +12,5 @@ Feature: datasets creation
     Then the container logs should contain the message "managing agent policy from core" within 10 seconds
       And the container logs should contain the message "policy applied successfully" within 10 seconds
       And the container logs should contain the message "scraped metrics for policy" within 180 seconds
-      And referred sink must have active state on response within 10 seconds
+      And referred sink must have active state on response within 10 seconds
+      And datasets related to all existing policies have validity valid

python-test/features/integration.feature

@@ -0,0 +1,33 @@
+@integration
+Feature: Integration tests
+
+Scenario: Apply two policies to an agent
+    Given the Orb user has a registered account
+        And the Orb user logs in
+        And that an agent already exists and is online
+        And referred agent is subscribed to a group
+        And that a sink already exists
+    When 2 policies are applied to the agent
+    Then this agent's heartbeat shows that 2 policies are successfully applied
+        And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds
+        And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds
+        And referred sink must have active state on response within 10 seconds
+        And datasets related to all existing policies have validity valid
+
+
+Scenario: Remove policy from agent
+    Given the Orb user has a registered account
+        And the Orb user logs in
+        And that an agent already exists and is online
+        And referred agent is subscribed to a group
+        And that a sink already exists
+        And 2 policies are applied to the agent
+        And this agent's heartbeat shows that 2 policies are successfully applied
+    When one of applied policies is removed
+    Then referred policy must not be listed on the orb policies list
+        And datasets related to removed policy has validity invalid
+        And datasets related to all existing policies have validity valid
+        And this agent's heartbeat shows that 1 policies are successfully applied
+        And container logs should inform that removed policy was stopped and removed within 10 seconds
+        And the container logs that were output after the policy have been removed contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds
+        And the container logs that were output after the policy have been removed does not contain the message "scraped metrics for policy" referred to deleted policy anymore

python-test/features/policies.feature

@@ -6,4 +6,4 @@ Feature: policy creation
       And the Orb user logs in
       And that an agent already exists and is online
     When a new policy is created
-    Then referred policy must be listened on the orb policies list
+    Then referred policy must be listed on the orb policies list

python-test/features/steps/control_plane_agents.py

@@ -57,7 +57,7 @@ def clean_agents(context):
     delete_agents(token, agents_filtered_list)
 
 
-@step("this agent's heartbeat shows that all {amount_of_policies} policies have been successfully applied")
+@step("this agent's heartbeat shows that {amount_of_policies} policies are successfully applied")
 def list_policies_applied_to_an_agent(context, amount_of_policies):
     time_waiting = 0
     sleep_time = 0.5
@@ -71,10 +71,8 @@ def list_policies_applied_to_an_agent(context, amount_of_policies):
         time.sleep(sleep_time)
         time_waiting += sleep_time
 
-    assert_that(len(context.list_agent_policies_id), equal_to(int(amount_of_policies)), f'Amount of policies applied to '
-                                                                                     f'this agent failed with '
-                                                                                     f'{context.list_agent_policies_id}'
-                                                                                        f'policies')
+    assert_that(len(context.list_agent_policies_id), equal_to(int(amount_of_policies)),
+                f"Amount of policies applied to this agent failed with {context.list_agent_policies_id} policies")
     assert_that(sorted(context.list_agent_policies_id), equal_to(sorted(context.policies_created.keys())))
 
 
@@ -195,7 +193,7 @@ def generate_agent_name_and_tag(name_agent_prefix, agent_tag_key_prefix, agent_t
     :param (str) agent_tag_value_prefix: prefix to identify tag_value created by tests
     :return: random name, tag_key and tag_value for agent
     """
-    agent_name = agent_name_prefix + random_string(10)
-    agent_tag_key = tag_key_prefix + random_string(4)
-    agent_tag_value = tag_value_prefix + random_string(4)
+    agent_name = name_agent_prefix + random_string(10)
+    agent_tag_key = agent_tag_key_prefix + random_string(4)
+    agent_tag_value = agent_tag_value_prefix + random_string(4)
     return agent_name, agent_tag_key, agent_tag_value

python-test/features/steps/control_plane_datasets.py

@@ -1,4 +1,4 @@
-from behave import given, when, then
+from behave import given, when, then, step
 from utils import random_string, filter_list_by_parameter_start_with
 from hamcrest import *
 import requests
@@ -12,7 +12,7 @@
 
 @when("a new dataset is created using referred group, sink and policy ID")
 def create_new_dataset(context):
-    context.dataset_applied_timestamp = datetime.now().timestamp()
+    context.considered_timestamp = datetime.now().timestamp()
     token = context.token
     agent_groups_id = context.agent_group_data['id']
     sink_id = context.sink['id']
@@ -21,6 +21,23 @@ def create_new_dataset(context):
     context.dataset = create_dataset(token, dataset_name, policy_id, agent_groups_id, sink_id)
 
 
+@step('datasets related to removed policy has validity invalid')
+def check_dataset_status_invalid(context):
+    for dataset_id in context.id_of_datasets_related_to_removed_policy:
+        dataset = get_dataset(context.token, dataset_id)
+        assert_that(dataset['valid'], equal_to(False), f"dataset {dataset_id} status failed with valid"
+                                                       f"equals {dataset['valid']}")
+
+
+@step('datasets related to all existing policies have validity valid')
+def check_dataset_status_valid(context):
+    all_datasets = list_datasets(context.token)
+    for dataset in all_datasets:
+        if dataset["agent_policy_id"] in context.policies_created.keys():
+            assert_that(dataset['valid'], equal_to(True), f"dataset {dataset['id']} status failed with valid "
+                                                          f"equals {dataset['valid']}")
+
+
 def create_dataset(token, name_label, policy_id, agent_group_id, sink_id):
     """
 
@@ -105,3 +122,22 @@ def delete_dataset(token, dataset_id):
 
     assert_that(response.status_code, equal_to(204), 'Request to delete dataset id='
                 + dataset_id + ' failed with status=' + str(response.status_code))
+
+
+def get_dataset(token, dataset_id, expected_status_code=200):
+    """
+    Gets a dataset from Orb control plane
+
+    :param (str) token: used for API authentication
+    :param (str) dataset_id: that identifies dataset to be fetched
+    :param (int) expected_status_code: expected request's status code. Default:200.
+    :returns: (dict) the fetched dataset
+    """
+
+    get_dataset_response = requests.get(base_orb_url + '/api/v1/policies/dataset/' + dataset_id,
+                                       headers={'Authorization': token})
+
+    assert_that(get_dataset_response.status_code, equal_to(expected_status_code),
+                'Request to get policy id=' + dataset_id + ' failed with status=' + str(get_dataset_response.status_code))
+
+    return get_dataset_response.json()