Merge pull request #359 from openstack-k8s-operators/must-gather

Add os_must_gather role

ci_framework/playbooks/99-logs.yml

@@ -32,6 +32,10 @@
       ansible.builtin.import_role:
         name: artifacts
 
+    - name: Run os_must_gather
+      ansible.builtin.import_role:
+        name: os_must_gather
+
     - name: Ensure ansible.log is at the expected location
       register: ansible_log_state
       ansible.builtin.stat:

ci_framework/roles/artifacts/tasks/main.yml

@@ -39,11 +39,6 @@
     - always
   ansible.builtin.import_tasks: packages.yml
 
-- name: Gather cluster information
-  tags:
-    - always
-  ansible.builtin.import_tasks: cluster_info.yml
-
 - name: Cleanup artifacts before finishing
   tags:
     - always

ci_framework/roles/os_must_gather/README.md

@@ -0,0 +1,15 @@
+# os_must_gather
+A role to gather openstack/openshift deployment logs using openstack-must-gather tool
+
+## Parameters
+* `cifmw_os_must_gather_image`: (String) The full url of openstack-must-gather container image
+* `cifmw_os_must_gather_output_dir`: (String) The directory to store logs generated by must-gather tool
+* `cifmw_os_must_gather_timeout`: (String) Timeout for must-gather command
+* `cifmw_os_must_gather_host_network`: (Bool) Flag to gather host network data
+
+## Examples
+```
+- name: Gather logs using os-must-gather tool
+  ansible.builtin.include_role:
+    role: os_must_gather
+```

ci_framework/roles/os_must_gather/defaults/main.yml

@@ -0,0 +1,23 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+# All variables intended for modification should be placed in this file.
+# All variables within this role should have a prefix of "cifmw_os_must_gather"
+cifmw_os_must_gather_image: "quay.io/openstack-k8s-operators/openstack-must-gather:latest"
+cifmw_os_must_gather_output_dir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}"
+cifmw_os_must_gather_timeout: "10m"
+cifmw_os_must_gather_host_network: false

ci_framework/roles/os_must_gather/meta/main.yml

@@ -0,0 +1,41 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+galaxy_info:
+  author: CI Framework
+  description: CI Framework Role -- os_must_gather
+  company: Red Hat
+  license: Apache-2.0
+  min_ansible_version: 2.14
+  namespace: edpm
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  platforms:
+    - name: CentOS
+      versions:
+        - 9
+
+  galaxy_tags:
+    - edpm
+
+# List your role dependencies here, one per line. Be sure to remove the '[]' above,
+# if you add dependencies to this list.
+dependencies: []

ci_framework/roles/os_must_gather/molecule/default/converge.yml

@@ -0,0 +1,24 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  vars:
+    cifmw_path: "{{ ansible_user_dir }}/.crc/bin:{{ ansible_user_dir }}/.crc/bin/oc:{{ ansible_user_dir }}/bin:{{ ansible_env.PATH }}"
+    cifmw_openshift_kubeconfig: "{{ ansible_user_dir }}/.crc/machines/crc/kubeconfig"
+  roles:
+    - role: "os_must_gather"

ci_framework/roles/os_must_gather/molecule/default/molecule.yml

@@ -0,0 +1,11 @@
+---
+# Mainly used to override the defaults set in .config/molecule/
+# By default, it uses the "config_podman.yml" - in CI, it will use
+# "config_local.yml".
+log: true
+
+provisioner:
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml

ci_framework/roles/os_must_gather/molecule/default/prepare.yml

@@ -0,0 +1,29 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  vars:
+    cifmw_path: "{{ ansible_user_dir }}/.crc/bin:{{ ansible_user_dir }}/.crc/bin/oc:{{ ansible_user_dir }}/bin:{{ ansible_env.PATH }}"
+    cifmw_install_yamls_out_dir: "{{ ansible_user_dir }}/.ansible"
+  roles:
+    - role: test_deps
+    - role: ci_setup
+  tasks:
+    - name: Ensure CRC is started
+      ansible.builtin.command:
+        cmd: crc start

ci_framework/roles/os_must_gather/tasks/main.yml

@@ -0,0 +1,57 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Ensure directories are present
+  ansible.builtin.file:
+    path: "{{ cifmw_os_must_gather_output_dir }}/{{ item }}"
+    state: directory
+  loop:
+    - logs
+
+- name: Check for oc command
+  ansible.builtin.command: command -v oc
+  environment:
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  register: oc_installed
+  ignore_errors: true
+
+- name: Running openstack-must-gather tool
+  when:
+    - oc_installed is defined
+    - oc_installed.rc == 0
+    - cifmw_openshift_kubeconfig is defined
+  block:
+    - name: Run openstack-must-gather command
+      environment:
+        KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+        PATH: "{{ cifmw_path }}"
+      ansible.builtin.shell:
+        cmd: >-
+          oc adm must-gather --image {{ cifmw_os_must_gather_image }}
+          --timeout {{ cifmw_os_must_gather_timeout }}
+          --host-network={{ cifmw_os_must_gather_host_network }}
+          --dest-dir {{ cifmw_os_must_gather_output_dir }}/logs > {{ cifmw_os_must_gather_output_dir }}/logs/os_must_gather.log
+  rescue:
+    - name: Inspect the cluster after must-gather failure
+      ignore_errors: true
+      environment:
+        KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+        PATH: "{{ cifmw_path }}"
+      ansible.builtin.shell: |
+        mkdir -p {{ cifmw_os_must_gather_output_dir }}/logs/oc_inspect
+        for ns in openstack-operators openstack baremetal-operator-system openshift-machine-api; do
+          oc adm inspect namespace/"${ns}" --dest-dir={{ cifmw_os_must_gather_output_dir }}/logs/oc_inspect || echo "Error getting logs from ${ns}"
+        done

docs/source/roles/os_must_gather.md

@@ -0,0 +1 @@
+../../../ci_framework/roles/os_must_gather/README.md

zuul.d/molecule.yaml

@@ -192,6 +192,16 @@
       - ^molecule-requirements.txt
       - ^ci_framework/roles/operator_deploy/(?!meta|README).*
       - ^ci/playbooks/molecule.*
+- job:
+    name: cifmw-molecule-os_must_gather
+    parent: cifmw-molecule-base-crc
+    vars:
+      TEST_RUN: os_must_gather
+    files:
+      - ^ansible-requirements.txt
+      - ^molecule-requirements.txt
+      - ^ci_framework/roles/os_must_gather/(?!meta|README).*
+      - ^ci/playbooks/molecule.*
 - job:
     name: cifmw-molecule-os_net_setup
     parent: cifmw-molecule-base

zuul.d/projects.yaml

@@ -37,6 +37,7 @@
         - cifmw-molecule-openshift_setup
         - cifmw-molecule-operator_build
         - cifmw-molecule-operator_deploy
+        - cifmw-molecule-os_must_gather
         - cifmw-molecule-os_net_setup
         - cifmw-molecule-pkg_build
         - cifmw-molecule-polarion