GPU role (first part)

roles/cluster/defaults/main.yml

@@ -20,6 +20,7 @@ cluster_common_packages:
   - ncurses-static
   - net-tools
   - openssl
+  - pciutils
   - qt5-qtbase
   - qt5-qtxmlpatterns
   - readline-static
@@ -36,4 +37,5 @@ cluster_common_packages:
   - urw-base35-fonts
   - vim
   - wget
+  - yum-utils
 ...

roles/gpu/README.md

@@ -0,0 +1,20 @@
+# NVidia GPU installation role for Centos 7
+
+This role follows the latest instructions of the newest version of available
+drivers, avaiable at [NVIDIA CUDA Installation Guide for
+Linux](https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Linux.pdf).
+
+## Role outline
+
+- installs `pciutils` tools
+- checks if there is pci device from nvidia and if there is, then it
+- installs on system needed yum packages that can later build the driver
+- downloads the .run driver from nvidia (driver version is defined in defualts)
+- installs and compile the driver module
+- blacklists nouveau
+- installs systemd service file, that automatically loads the driver upons system
+  boot, and that reloads the driver when/if it has failed operating
+
+## TO-DO
+- extensive testing and benchmarking
+- role for development software installation

roles/gpu/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+gpu_cuda_version: '11.7.1'
+gpu_driver_version: '515.65.01'
+gpu_url_directory: 'https://developer.download.nvidia.com/compute/cuda/{{ gpu_cuda_version }}/local_installers/'
+gpu_runfile: 'cuda_{{ gpu_cuda_version }}_{{ gpu_driver_version }}_linux.run'
+...

roles/gpu/files/blacklist-nouveau.conf

@@ -0,0 +1,2 @@
+blacklist nouveau
+options nouveau modeset=0

roles/gpu/files/nvidia-persistenced.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Initialize GPU at the startup of the system
+
+[Service]
+ExecStart=/usr/bin/nvidia-persistenced --verbose
+RestartSec=15
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

roles/gpu/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Restart server because of the pending updates
+  ansible.builtin.reboot:
+    msg: "Reboot initiated by Ansible, because of the pending updates"
+  listen: "reboot_server"
+  become: true
+...

roles/gpu/tasks/configuration.yml

@@ -0,0 +1,23 @@
+---
+- name: Copy blacklist-nouveau.conf file into modprobe.d to disable Nouveau drivers
+  ansible.builtin.copy:
+    src: blacklist-nouveau.conf
+    dest: /etc/modprobe.d/blacklist-nouveau.conf
+    owner: root
+    group: root
+    mode: '0644'
+  become: true
+
+- name: Install NVidia persistence service
+  ansible.builtin.copy:
+    src: nvidia-persistenced.service
+    dest: /etc/systemd/system/nvidia-persistenced.service
+  become: true
+
+- name: Enable a nvidia-persistenced service
+  ansible.builtin.systemd:
+    name: nvidia-persistenced.service
+    state: started
+    enabled: true
+  become: true
+...

roles/gpu/tasks/gpu.yml

@@ -0,0 +1,67 @@
+---
+- name: Install yum requirements for gpu driver installation
+  ansible.builtin.yum:
+    state: 'installed'
+    update_cache: true
+    name:
+      - tar
+      - bzip2
+      - make
+      - automake
+      - gcc
+      - gcc-c++
+      - pciutils
+      - elfutils-libelf-devel
+      - libglvnd-devel
+      - bind-utils
+      - wget
+  become: true
+
+# ansible_kernel variable is not working, as after reboot, still holds old kernel
+- name: Get current kernel version
+  ansible.builtin.command: '/usr/bin/uname -r'
+  register: uname_output
+  failed_when: uname_output.rc != 0
+  when: true
+  become: true
+
+- name: Set kernel version fact
+  ansible.builtin.set_fact:
+    kernel_version: "{{ uname_output.stdout }}"
+
+- name: Install kernel developement package matching running kernel version
+  ansible.builtin.yum:
+    name: 'kernel-devel-{{ kernel_version }}'
+  register: yum_result
+  failed_when: yum_result.rc != 0
+  when: true
+  become: true
+
+- name: Download a driver installation file from NVidia
+  ansible.builtin.get_url:
+    url: '{{ gpu_url_directory }}/{{ gpu_runfile }}'
+    dest: '/root/{{ gpu_runfile }}'
+    mode: '0700'
+  become: true
+
+- name: "Check if driver downloaded"
+  ansible.builtin.stat:
+    path: '/root/{{ gpu_runfile }}'
+  when: true
+  register: driver_downloaded
+  become: true
+
+- name: Install driver from .run file
+  ansible.builtin.command: '/root/{{ gpu_runfile }} --silent --driver'
+  register: install_result
+  failed_when: install_result.rc != 0
+  when: driver_downloaded.stat.exists
+  notify: reboot_server
+  become: true
+
+- name: Remove installation file
+  ansible.builtin.file:
+    path: '/root/{{ gpu_runfile }}'
+    state: absent
+  become: true
+...

roles/gpu/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+- name: Check if system needs to be restarted
+  ansible.builtin.command: '/bin/needs-restarting -r'
+  register: needs_restarting
+  failed_when: 'needs_restarting.rc > 1'
+  changed_when: 'needs_restarting.rc == 1'
+  become: true
+  notify: reboot_server
+
+- name: Flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Check if we have CUDA capable system
+  ansible.builtin.command: 'lspci'
+  register: lspci_nv
+  when: true
+  become: true
+
+- name: Check if we have already configured NVidia devices
+  ansible.builtin.command: 'nvidia-smi'
+  register: detect_devices
+  failed_when: false
+  when: true
+  become: true
+
+- name: Run GPU driver installation role
+  ansible.builtin.include: gpu.yml
+  when: ('"nvidia" in lspci_nv.stdout | lower') and (detect_devices.rc != 0)
+
+- name: Set configuration files for service and modprobe
+  ansible.builtin.include: configuration.yml
+...

single_group_playbooks/cluster_part1.yml

@@ -22,6 +22,7 @@
     - figlet_motd
     - node_exporter
     - cluster
+    - gpu            # needs to run after role 'cluster'
     - resolver
     - coredumps
 ...

single_role_playbooks/gpu.yml

@@ -0,0 +1,6 @@
+---
+- hosts:
+    - compute_vm
+  roles:
+    - gpu
+...

static_inventories/nibbler_cluster.yml

@@ -127,7 +127,7 @@ all:
           hosts:
             nb-vcompute04:
           vars:
-            cloud_flavor: gpu.A40
+            cloud_flavor: gpu.A40_8
             local_volume_size_extra: 1
             slurm_sockets: 32
             slurm_cores_per_socket: 1