From 59a7ee48cd268d32b064ee5429a1efae80965c8b Mon Sep 17 00:00:00 2001 From: Jon Uriarte Date: Fri, 28 Jun 2024 06:17:08 -0400 Subject: [PATCH] [OSASINFRA] Add multi test configs support Support multiple test configs or job definitions from the calling zuul job, i.e: cifmw_run_test_shiftstack_testconfig: - 4.16_ovnkubernetes_ipi_va1.yaml - 4.16_ovnkubernetes_upi_va1.yaml - 4.16_ovnkubernetes_ipi-proxy.yaml The shiftstack role will run the shiftstack-qa automation for each test config defined in `cifmw_run_test_shiftstack_testconfig`. The artifacts directory includes now an additional directory for each test config, as multiple test configs can run now from a Zuul job. This commit adds the `cifmw_shiftstack_manifests_dir` param as well for the role generated manifest files and the `cifmw_shiftstack_ansible_command_logs_dir` param for the ansible command module output logs. --- roles/shiftstack/README.md | 8 +- roles/shiftstack/defaults/main.yml | 2 + .../shiftstack/molecule/default/converge.yml | 3 +- .../tasks/deploy_shiftstackclient_pod.yml | 8 +- .../shiftstack/tasks/exec_command_in_pod.yml | 16 ++-- .../shiftstack/tasks/pre_test_shiftstack.yml | 18 +++- roles/shiftstack/tasks/test_config.yml | 59 +++++++++++++ roles/shiftstack/tasks/test_shiftstack.yml | 83 ++++++++----------- 8 files changed, 129 insertions(+), 68 deletions(-) create mode 100644 roles/shiftstack/tasks/test_config.yml diff --git a/roles/shiftstack/README.md b/roles/shiftstack/README.md index c01aa2b6f9..4bc49b416b 100644 --- a/roles/shiftstack/README.md +++ b/roles/shiftstack/README.md @@ -2,6 +2,7 @@ Role for triggering Openshift on Openstack QA automation (installation and tests). ## Parameters +* `cifmw_shiftstack_ansible_command_logs_dir`: (*string*) Directory name for the ansible command module output. Defaults to `"{{ cifmw_shiftstack_basedir }}/ansible_command_logs"`. * `cifmw_shiftstack_artifacts_dir`: (*string*) Directory name for the role artifacts. Defaults to `"{{ cifmw_shiftstack_basedir }}/artifacts"`. * `cifmw_shiftstack_basedir`: (*string*) Base directory for the role artifacts and logs. Defaults to `{{ cifmw_basedir }}/tests/shiftstack` (which defaults to `~/ci-framework-data/tests/shiftstack`. * `cifmw_shiftstack_client_pod_name`: (*string*) Pod name for the pod running the Openshift installer and tests. Defaults to `shiftstackclient`. @@ -11,6 +12,7 @@ Role for triggering Openshift on Openstack QA automation (installation and tests * `cifmw_shiftstack_client_pvc_manifest`: (*string*) The file name for the shiftstackclient pvc manifest. Defaults to `"{{ cifmw_shiftstack_client_pod_name }}_pvc.yml"`. * `cifmw_shiftstack_cluster_name`: (*string*) The Openshift cluster name. Defaults to `ostest`. * `cifmw_shiftstack_installation_dir`: (*string*) Directory to place installation files. Defaults to `"{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/installation"`. +* `cifmw_shiftstack_manifests_dir`: (*string*) Directory name for the role generated Openshift manifests. Defaults to `"{{ cifmw_shiftstack_basedir }}/manifests"`. * `cifmw_shiftstack_project_name`: (*string*) The Openstack project name. Defaults to `shiftstack`. * `cifmw_shiftstack_qa_gerrithub_change`: (*string*) The gerrithub change to fetch from the `cifmw_shiftstack_qa_repo` repository (i.e. 'refs/changes/29/1188429/50)'. Defaults to ''. * `cifmw_shiftstack_qa_repo`: (*string*) The repository containing the Openshift on Openstack QA automation. Defaults to `https://review.gerrithub.io/shiftstack/shiftstack-qa`. @@ -24,8 +26,10 @@ The role is imported in the test playbook, i.e. when: cifmw_run_tests: true cifmw_run_tempest: false cifmw_run_test_role: shiftstack -cifmw_run_test_shiftstack_testconfig: 4.15_ovnkubernetes_ipi_va1.yaml +cifmw_run_test_shiftstack_testconfig: + - 4.17_ovnkubernetes_ipi.yaml + - 4.17_ovnkubernetes_upi.yaml cifmw_shiftstack_qa_gerrithub_change: refs/changes/29/1188429/50 #optional -$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_tempest=false cifmw_run_test_role=shiftstack cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50" +$ ansible-playbook deploy-edpm.yml -e 'cifmw_run_tests=true cifmw_run_tempest=false cifmw_run_test_role=shiftstack cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50' -e 'cifmw_run_test_shiftstack_testconfig=["4.17_ovnkubernetes_ipi.yaml","4.17_ovnkubernetes_upi.yaml"]' ``` diff --git a/roles/shiftstack/defaults/main.yml b/roles/shiftstack/defaults/main.yml index 02f88809f0..3956c8a84d 100644 --- a/roles/shiftstack/defaults/main.yml +++ b/roles/shiftstack/defaults/main.yml @@ -17,6 +17,7 @@ # All variables intended for modification should be placed in this file. # All variables within this role should have a prefix of "cifmw_shiftstack" +cifmw_shiftstack_ansible_command_logs_dir: "{{ cifmw_shiftstack_basedir }}/ansible_command_logs" cifmw_shiftstack_artifacts_dir: "{{ cifmw_shiftstack_basedir }}/artifacts" cifmw_shiftstack_basedir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/tests/shiftstack" cifmw_shiftstack_client_pod_image: "quay.io/shiftstack-qe/shiftstack-client:latest" @@ -26,6 +27,7 @@ cifmw_shiftstack_client_pod_namespace: "openstack" cifmw_shiftstack_client_pvc_manifest: "{{ cifmw_shiftstack_client_pod_name }}_pvc.yml" cifmw_shiftstack_cluster_name: "ostest" cifmw_shiftstack_installation_dir: "{{ cifmw_shiftstack_basedir }}/installation" +cifmw_shiftstack_manifests_dir: "{{ cifmw_shiftstack_basedir }}/manifests" cifmw_shiftstack_project_name: "shiftstack" cifmw_shiftstack_qa_gerrithub_change: "" cifmw_shiftstack_qa_repo: "https://review.gerrithub.io/shiftstack/shiftstack-qa" diff --git a/roles/shiftstack/molecule/default/converge.yml b/roles/shiftstack/molecule/default/converge.yml index 57915eb97e..29b935a39e 100644 --- a/roles/shiftstack/molecule/default/converge.yml +++ b/roles/shiftstack/molecule/default/converge.yml @@ -22,7 +22,8 @@ cifmw_path: "{{ ansible_user_dir }}/.crc/bin:{{ ansible_user_dir }}/.crc/bin/oc:{{ ansible_user_dir }}/bin:{{ ansible_env.PATH }}" cifmw_openshift_kubeconfig: "{{ ansible_user_dir }}/.crc/machines/crc/kubeconfig" cifmw_shiftstack_run_playbook: cifmw-gate.yaml - cifmw_run_test_shiftstack_testconfig: cifmw-gate.yaml + cifmw_run_test_shiftstack_testconfig: + - cifmw-gate.yaml cifmw_shiftstack_sc: "crc-csi-hostpath-provisioner" tasks: - name: Include the shiftstack role diff --git a/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml b/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml index 5c6d82ae52..0e6377c2c6 100644 --- a/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml +++ b/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml @@ -17,26 +17,26 @@ - name: Render the pvc manifest ansible.builtin.template: src: templates/shiftstackclient_pvc.yml.j2 - dest: "{{ (cifmw_shiftstack_basedir, cifmw_shiftstack_client_pvc_manifest) | path_join }}" + dest: "{{ (cifmw_shiftstack_manifests_dir, cifmw_shiftstack_client_pvc_manifest) | path_join }}" mode: "0644" - name: Apply the manifest for the PVC creation kubernetes.core.k8s: state: present kubeconfig: "{{ cifmw_openshift_kubeconfig }}" - src: "{{ (cifmw_shiftstack_basedir, cifmw_shiftstack_client_pvc_manifest) | path_join }}" + src: "{{ (cifmw_shiftstack_manifests_dir, cifmw_shiftstack_client_pvc_manifest) | path_join }}" - name: Render the pod manifest from a template ansible.builtin.template: src: "templates/shiftstackclient_pod.yml.j2" - dest: "{{ (cifmw_shiftstack_basedir, cifmw_shiftstack_client_pod_manifest) | path_join }}" + dest: "{{ (cifmw_shiftstack_manifests_dir, cifmw_shiftstack_client_pod_manifest) | path_join }}" mode: "0644" - name: Apply the manifest for the pod creation kubernetes.core.k8s: state: present kubeconfig: "{{ cifmw_openshift_kubeconfig }}" - src: "{{ (cifmw_shiftstack_basedir, cifmw_shiftstack_client_pod_manifest) | path_join }}" + src: "{{ (cifmw_shiftstack_manifests_dir, cifmw_shiftstack_client_pod_manifest) | path_join }}" wait: true wait_condition: type: Ready diff --git a/roles/shiftstack/tasks/exec_command_in_pod.yml b/roles/shiftstack/tasks/exec_command_in_pod.yml index 33650b99de..1b59c88701 100644 --- a/roles/shiftstack/tasks/exec_command_in_pod.yml +++ b/roles/shiftstack/tasks/exec_command_in_pod.yml @@ -21,13 +21,13 @@ PATH: "{{ cifmw_path }}" KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" ansible.builtin.command: - cmd: "oc rsh -n {{ namespace }} {{ pod }} bash -c '{{ command }}'" - register: command_result + cmd: "oc rsh -n {{ namespace }} {{ pod_name }} bash -c '{{ command }}'" + register: pod_command_result rescue: - name: Fail when the command module fails ansible.builtin.fail: - msg: "'{{ command }}' command execution failed, see logs for more information." + msg: '"{{ command }}" command execution failed with exit code "{{ pod_command_result.rc }}" and error: "{{ pod_command_result.stderr }}".' always: - name: Command execution log saving block @@ -37,15 +37,15 @@ ansible.builtin.command: cmd: date +"%Y_%m_%d-%H_%M_%S" register: current_date - - - name: Save the command execution information in the log file in '{{ cifmw_shiftstack_basedir }}' + + - name: Save the command execution information in the log file in '{{ cifmw_shiftstack_ansible_command_logs_dir }}' vars: command_info: namespace: "{{ namespace }}" - pod: "{{ pod }}" + pod_name: "{{ pod_name }}" command: "{{ command }}" - command_result: "{{ command_result }}" + pod_command_result: "{{ pod_command_result }}" ansible.builtin.copy: content: "{{ command_info | to_nice_yaml }}" - dest: "{{ cifmw_shiftstack_basedir }}/{{ current_date.stdout }}.{{ log_file_name }}" + dest: "{{ cifmw_shiftstack_ansible_command_logs_dir }}/{{ current_date.stdout }}.{{ log_file_name }}" mode: "0644" diff --git a/roles/shiftstack/tasks/pre_test_shiftstack.yml b/roles/shiftstack/tasks/pre_test_shiftstack.yml index f145737c0c..26594b64ef 100644 --- a/roles/shiftstack/tasks/pre_test_shiftstack.yml +++ b/roles/shiftstack/tasks/pre_test_shiftstack.yml @@ -13,6 +13,13 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +- name: Fail if the testconfig is not passed + ansible.builtin.fail: + msg: | + "The variable 'cifmw_run_test_shiftstack_testconfig' is not passed and it's required." + when: cifmw_run_test_shiftstack_testconfig is not defined + - name: Remove the shiftstackclient pod if exists kubernetes.core.k8s: state: absent @@ -21,13 +28,18 @@ namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" name: "{{ cifmw_shiftstack_client_pod_name }}" -- name: Remove the shiftstack role artifacts/logs directory (if exists) +- name: Remove the shiftstack role data directory (if exists) ansible.builtin.file: path: "{{ cifmw_shiftstack_basedir }}" state: absent -- name: Create the directory for shiftstack role artifacts/logs ({{ cifmw_shiftstack_basedir }}) +- name: "Create the directory '{{ item }}'" ansible.builtin.file: - path: "{{ cifmw_shiftstack_basedir }}" + path: "{{ item }}" state: directory mode: "0755" + loop: + - "{{ cifmw_shiftstack_basedir }}" + - "{{ cifmw_shiftstack_ansible_command_logs_dir }}" + - "{{ cifmw_shiftstack_manifests_dir }}" + - "{{ cifmw_shiftstack_artifacts_dir }}" diff --git a/roles/shiftstack/tasks/test_config.yml b/roles/shiftstack/tasks/test_config.yml new file mode 100644 index 0000000000..e0a7258e6c --- /dev/null +++ b/roles/shiftstack/tasks/test_config.yml @@ -0,0 +1,59 @@ +--- +# These tasks are intended to be run inside a loop, where `testconfig` +# is the loop parameter and can take different values in each run. +- name: OCP install and test block + vars: + testconfig_dir_name: "{{ testconfig | splitext | first }}" + testconfig_artifacts_dir: "{{ (cifmw_shiftstack_artifacts_dir, testconfig_dir_name) | path_join }}" + + block: + - name: "Initialize the testconfig_result_dict fact for the testconfig '{{ testconfig }}'" + ansible.builtin.set_fact: + testconfig_result_dict: "{{ testconfig_result_dict | combine({ testconfig : 'PASS' }) }}" + + - name: "Check the test configuration file exists in the repository" + vars: + namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" + pod_name: "{{ cifmw_shiftstack_client_pod_name }}" + command: >- + test -f shiftstack-qa/jobs_definitions/{{ testconfig }} + log_file_name: "{{ testconfig }}-find_test_config.log" + ansible.builtin.include_tasks: exec_command_in_pod.yml + + - name: "Create the directory for the test config artifacts '{{ testconfig_artifacts_dir }}'" + ansible.builtin.file: + path: "{{ testconfig_artifacts_dir }}" + state: directory + mode: "0755" + + - name: "Test Openshift on Openstack with the test configuration '{{ testconfig }}'" + vars: + namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" + pod_name: "{{ cifmw_shiftstack_client_pod_name }}" + command: >- + cd shiftstack-qa && + ansible-navigator run playbooks/{{ cifmw_shiftstack_run_playbook }} -e @jobs_definitions/{{ testconfig }} -e ocp_cluster_name={{ cifmw_shiftstack_cluster_name }} -e user_cloud={{ cifmw_shiftstack_project_name }} + ansible.builtin.include_tasks: exec_command_in_pod.yml + + rescue: + - name: "Set failure info for '{{ testconfig }}'" + ansible.builtin.set_fact: + failure_info: + failed_task: "{{ ansible_failed_task.name }}" + result_msg: "{{ ansible_failed_result.msg }}" + + - name: "Update the testconfig_result_dict fact with a failed status for '{{ testconfig }}'" + ansible.builtin.set_fact: + testconfig_result_dict: "{{ testconfig_result_dict | combine({testconfig: {'FAIL': failure_info}}) }}" + + always: + - name: "Copy the artifacts from the pod '{{ cifmw_shiftstack_client_pod_name }}'" + environment: + PATH: "{{ cifmw_path }}" + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + ansible.builtin.command: + cmd: > + oc rsync -n {{ cifmw_shiftstack_client_pod_namespace }} + {{ cifmw_shiftstack_client_pod_name }}:{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/ + {{ testconfig_artifacts_dir }}/ + ignore_errors: true diff --git a/roles/shiftstack/tasks/test_shiftstack.yml b/roles/shiftstack/tasks/test_shiftstack.yml index f69348921d..1bdee361b2 100644 --- a/roles/shiftstack/tasks/test_shiftstack.yml +++ b/roles/shiftstack/tasks/test_shiftstack.yml @@ -17,7 +17,7 @@ - name: Clone the repository '{{ cifmw_shiftstack_qa_repo }}' vars: namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" - pod: "{{ cifmw_shiftstack_client_pod_name }}" + pod_name: "{{ cifmw_shiftstack_client_pod_name }}" command: >- git clone {{ cifmw_shiftstack_qa_repo }} log_file_name: "clone_shiftstack_qa_repo.log" @@ -26,7 +26,7 @@ - name: Fetch the gerrithub change '{{ cifmw_shiftstack_qa_gerrithub_change }}' vars: namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" - pod: "{{ cifmw_shiftstack_client_pod_name }}" + pod_name: "{{ cifmw_shiftstack_client_pod_name }}" command: >- cd shiftstack-qa && git fetch {{ cifmw_shiftstack_qa_repo }} {{ cifmw_shiftstack_qa_gerrithub_change }} && @@ -40,61 +40,44 @@ - name: Install the ansible collections vars: namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" - pod: "{{ cifmw_shiftstack_client_pod_name }}" + pod_name: "{{ cifmw_shiftstack_client_pod_name }}" command: >- cd shiftstack-qa && ansible-galaxy collection install -f -r requirements.yaml log_file_name: "install_collections.log" ansible.builtin.include_tasks: exec_command_in_pod.yml -- name: Check the playbook to be run exists in the repository - vars: - namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" - pod: "{{ cifmw_shiftstack_client_pod_name }}" - command: >- - test -f shiftstack-qa/playbooks/{{ cifmw_shiftstack_run_playbook }} - log_file_name: "find_playbook.log" - ansible.builtin.include_tasks: exec_command_in_pod.yml +- name: Initialize a dictionary to store the result of the test configs + ansible.builtin.set_fact: + testconfig_result_dict: {} -- name: Check the test configuration file exists in the repository - vars: - namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" - pod: "{{ cifmw_shiftstack_client_pod_name }}" - command: >- - test -f shiftstack-qa/jobs_definitions/{{ cifmw_run_test_shiftstack_testconfig }} - log_file_name: "find_test_config.log" - ansible.builtin.include_tasks: exec_command_in_pod.yml +- name: Include test config tasks and loop over '{{ cifmw_run_test_shiftstack_testconfig }}' + ansible.builtin.include_tasks: test_config.yml + loop: "{{ cifmw_run_test_shiftstack_testconfig }}" + loop_control: + loop_var: testconfig -- name: OCP install and test block - block: - - name: Test Openshift on Openstack with the test configuration '{{ cifmw_run_test_shiftstack_testconfig }}' - vars: - namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" - pod: "{{ cifmw_shiftstack_client_pod_name }}" - command: >- - cd shiftstack-qa && - ansible-navigator run playbooks/{{ cifmw_shiftstack_run_playbook }} -e @jobs_definitions/{{ cifmw_run_test_shiftstack_testconfig }} -e ocp_cluster_name={{ cifmw_shiftstack_cluster_name }} -e user_cloud={{ cifmw_shiftstack_project_name }} - ansible.builtin.include_tasks: exec_command_in_pod.yml +- name: Initialize a list to store failed test configs + ansible.builtin.set_fact: + fail_entries: [] - rescue: - - name: Fail task when OCP installation/test fails - ansible.builtin.fail: - msg: "OCP install/test block failed, see logs for more information." +- name: Iterate over the testconfig_result_dict and find failed test configs + vars: + item_key: "{{ item.key }}" + item_value: "{{ item.value }}" + ansible.builtin.set_fact: + fail_entries: "{{ fail_entries + [item_key] }}" + when: > + (item_value is string and item_value == 'FAIL') or + (item_value is mapping and item_value['FAIL'] is defined) + loop: "{{ testconfig_result_dict | dict2items }}" + loop_control: + label: "{{ item.key }}" - always: - - name: Retrieve artifacts from the pod - block: - - name: Create the directory for the artifacts - ansible.builtin.file: - path: "{{ cifmw_shiftstack_artifacts_dir }}" - state: directory - mode: "0755" +- name: Print test configs result + ansible.builtin.debug: + msg: "{{ testconfig_result_dict | to_nice_json }}" - - name: Copy the artifacts from the pod '{{ cifmw_shiftstack_client_pod_name }}' - environment: - PATH: "{{ cifmw_path }}" - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - ansible.builtin.command: - cmd: > - oc rsync -n {{ cifmw_shiftstack_client_pod_namespace }} - {{ cifmw_shiftstack_client_pod_name }}:{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/ - {{ cifmw_shiftstack_artifacts_dir }}/ +- name: Fail if there is any failed test config + ansible.builtin.fail: + msg: "There are FAILED test configs: {{ fail_entries }}" + when: fail_entries | length > 0