Updated subgroup_directories role.

group_vars/talos-cluster/vars.yml

@@ -87,7 +87,7 @@ regular_users:
     sudoers: '%umcg-gcc'
   - user: 'umcg-lifelines-dm'
     groups: ['umcg-lifelines']
-    sudoers: 'umcg-kdelange,kim'
+    sudoers: '%umcg-lifelines-dms'
 pfs_mounts: [
   { pfs: 'umcgst11',
     source: 'gcc-storage001.stor.hpc.local:/ifs/rekencluster',

roles/subgroup_directories/tasks/create_subgroup_directories.yml

@@ -1,78 +1,104 @@
 ---
-- name: 'Get group directories form the ldap cache.'
+#
+##
+### Query LDAP.
+##
+#
+- name: 'Get subgroups with version number form the ldap cache.'
   shell: |
-    ldapsearch -LLL -D '{{ ldap_binddn }}' -w '{{ bindpw }}' -b '{{ ldap_base }}' -H 'ldap://{{ uri_ldap }}' "(ObjectClass=GroupofNames)" dn | tr "=," "\n" | fgrep "{{ group }}" | grep -e "^{{ group }}$" || true
-  register: group_directories
+    ldapsearch -LLL -D '{{ ldap_binddn }}' -w '{{ bindpw }}' -b '{{ ldap_base }}' -H 'ldap://{{ uri_ldap }}' \
+        "(ObjectClass=GroupofNames)" dn \
+      | tr "=," "\n" \
+      | fgrep "{{ group }}" \
+      | grep -v "^{{ group }}$" \
+      | grep -- "-v[0-9][0-9]*$" \
+      || true
+  register: versioned_subgroups
 
-- name: 'Get subgroup directories with version number form the ldap cache.'
-  shell: |
-    ldapsearch -LLL -D '{{ ldap_binddn }}' -w '{{ bindpw }}' -b '{{ ldap_base }}' -H 'ldap://{{ uri_ldap }}' "(ObjectClass=GroupofNames)" dn | tr "=," "\n" | fgrep "{{ group }}" | grep -v "^{{ group }}$" | grep -- "-v[0-9][0-9]*$" || true
-  register: subgroup_directories_version
+- name: 'Create list of versioned subgroups.'
+  set_fact:
+    versioned_subgroups_list: "{% if versioned_subgroups.stdout | length %}{{ versioned_subgroups.stdout.split('\n') | list }}{% endif %}"
 
-- name: 'Get subgroup directories without version number form the ldap cache.'
+- name: 'Get subgroups without version number form the ldap cache.'
   shell: |
-    ldapsearch -LLL -D '{{ ldap_binddn }}' -w '{{ bindpw }}' -b '{{ ldap_base }}' -H 'ldap://{{ uri_ldap }}' "(ObjectClass=GroupofNames)" dn | tr "=," "\n" | fgrep "{{ group }}" | grep -v "^{{ group }}$" | grep -v -- "-v[0-9][0-9]*$"|| true
-  register: subgroup_directories
+    ldapsearch -LLL -D '{{ ldap_binddn }}' -w '{{ bindpw }}' -b '{{ ldap_base }}' -H 'ldap://{{ uri_ldap }}' \
+        "(ObjectClass=GroupofNames)" dn \
+        | tr "=," "\n" \
+        | fgrep "{{ group }}" \
+        | grep -v "^{{ group }}$" \
+        | grep -v -- "-v[0-9][0-9]*$" \
+        || true
+  register: unversioned_subgroups
 
-- name: 'Create list of subgroup_directories without version'
+- name: 'Create list of unversioned subgroups.'
   set_fact:
-    subgroup_directories_split: "{{ subgroup_directories.stdout.split('\n') | list }}"
-  
-- name: 'Create list of subgroup_directories with version'
-  set_fact:
-    subgroup_directories_version_split: "{{ subgroup_directories_version.stdout.split('\n') | list }}"
-
-- name: 'Create "[tmp0*|prm0*]/projects" directory'
+    unversioned_subgroups_list: "{% if unversioned_subgroups.stdout | length %}{{ unversioned_subgroups.stdout.split('\n') | list }}{% endif %}"
+#
+##
+### Create projects and releases folders in which the subgroup dirs will be created.
+##
+#
+- name: 'Create "[tmp*|prm*]/projects" directory.'
   file:
-    path: "/mnt/{{ pfs }}/groups/{{ group }}/{{ lfs }}/projects/"
+    path: "/groups/{{ group }}/{{ lfs }}/projects/"
     owner: "{{ group }}-dm"
     group: "{{ group }}"
     mode: "{{ mode_project }}"
     state: 'directory'
-  when: inventory_hostname in groups['sys-admin-interface']
+  when: unversioned_subgroups_list | length > 0
   become: true
+  become_user: "{{ group }}-dm"
 
-- name: 'Create "[tmp0*|prm0*]/releases" directory '
+- name: 'Create "[tmp*|prm*]/releases" directory.'
   file:
-    path: "/mnt/{{ pfs }}/groups/{{ group }}/{{ lfs }}/releases/"
+    path: "/groups/{{ group }}/{{ lfs }}/releases/"
     owner: "{{ group }}-dm"
     group: "{{ group }}"
     mode: "{{ mode_dataset }}"
     state: 'directory'
-  when: inventory_hostname in groups['sys-admin-interface']
+  when: versioned_subgroups_list | length > 0
   become: true
-
-- name: 'Create "prm08/releases/dataset" directory structure for subgroups with version number'
+  become_user: "{{ group }}-dm"
+#
+##
+### Create subgroup dirs.
+##
+#
+- name: 'Create "[tmp*|prm*]/releases/${dataset}" directory structure for subgroups with version number.'
   file:
-    path: "/mnt/{{ pfs }}/groups/{{ group }}/{{ lfs }}/releases/{{ item | regex_replace('^' + group + '-(.*)-(v[0-9][0-9]*)$', '\\1') }}"
+    path: "/groups/{{ group }}/{{ lfs }}/releases/{{ item | regex_replace('^' + group + '-(.*)-(v[0-9][0-9]*)$', '\\1') }}"
     owner: "{{ group }}-dm"
     group: "{{ group }}"
     mode: "{{ mode_dataset }}"
     state: 'directory'
-  with_items: "{{ subgroup_directories_version_split }}"
-  when: inventory_hostname in groups['sys-admin-interface']
+  with_items: "{{ versioned_subgroups_list }}"
+  when: versioned_subgroups_list | length > 0
   become: true
-
-- name: 'Create "prm08/releases/dataset/version" directory structure for subgroups with version number'
+  become_user: "{{ group }}-dm"
+
+- name: 'Create "[tmp*|prm*]/releases/${dataset}/${version}" directory structure for subgroups with version number.'
   file:
-    path: "/mnt/{{ pfs }}/groups/{{ group }}/{{ lfs }}/releases/{{ item | regex_replace('^' + group + '-(.*)-(v[0-9][0-9]*)$', '\\1') }}/{{ item | regex_replace('^' + group + '-(.*)-(v[0-9][0-9]*)$', '\\2') }}"
+    path: "/groups/{{ group }}/{{ lfs }}/releases/{{ item | regex_replace('^' + group + '-(.*)-(v[0-9][0-9]*)$', '\\1') }}/{{ item | regex_replace('^' + group + '-(.*)-(v[0-9][0-9]*)$', '\\2') }}"
     owner: "{{ group }}-dm"
     group: "{% if item | length %}{{ item }}{% else %}{{ group }}{% endif %}"
-    mode: "{{  mode_version }}"
+    mode: "{{ mode_version }}"
     state: 'directory'
-  with_items: "{{ subgroup_directories_version_split }}"
-  when: inventory_hostname in groups['sys-admin-interface']
+  with_items: "{{ versioned_subgroups_list }}"
+  when: versioned_subgroups_list | length > 0
   become: true
+  become_user: "{{ group }}-dm"
+  ignore_errors: yes # Continue of this specific subgroup failed and try to create other subgroup folders.
 
-- name: 'Create "[tmp0*|prm0*]/projects/" directory structure for subgroups without version number'
+- name: 'Create "[tmp*|prm*]/projects/${project}" directory structure for subgroups without version number.'
   file:
-    path: "/mnt/{{ pfs }}/groups/{{ group }}/{{ lfs }}/projects/{{ item | regex_replace('^' + group + '-(.*)$', '\\1') }}"
+    path: "/groups/{{ group }}/{{ lfs }}/projects/{{ item | regex_replace('^' + group + '-(.*)$', '\\1') }}"
     owner: "{{ group }}-dm"
     group: "{% if item | length %}{{ item }}{% else %}{{ group }}{% endif %}"
     mode: "{{ mode_project }}"
     state: 'directory'
-  with_items: "{{ subgroup_directories_split }}"
-  when: inventory_hostname in groups['sys-admin-interface']
+  with_items: "{{ unversioned_subgroups_list }}"
+  when: unversioned_subgroups_list | length > 0
   become: true
-
+  become_user: "{{ group }}-dm"
+  ignore_errors: yes # Continue of this specific subgroup failed and try to create other subgroup folders.
 ...

roles/subgroup_directories/tasks/main.yml

@@ -1,35 +1,33 @@
 ---
-- name: "loop over prm file systems"
+- name: 'Loop over "prm" file systems.'
   include_tasks:
     file: "{{ playbook_dir }}/roles/subgroup_directories/tasks/create_subgroup_directories.yml"
   vars:
     lfs: "{{ lfs_item.0.lfs }}"
-    pfs: "{{ lfs_item.0.pfs }}"
     group: "{{ lfs_item.1 }}"
     mode_project: '2750'
     mode_version: '2750'
-    mode_dataset: '2750' 
+    mode_dataset: '2750'
   with_subelements:
     - "{{ lfs_mounts | selectattr('lfs', 'search', 'prm[0-9]+$') | list }}"
     - 'groups'
   loop_control:
     loop_var: lfs_item
-  when: inventory_hostname in groups['sys-admin-interface']
+  when: inventory_hostname in groups['user-interface']
 
-- name: "loop over tmp file systems"
+- name: 'Loop over "tmp" file systems.'
   include_tasks:
     file: "{{ playbook_dir }}/roles/subgroup_directories/tasks/create_subgroup_directories.yml"
   vars:
-    lfs: "{{ lfs_item.0.lfs )}}"
-    pfs: "{{ lfs_item.0.pfs }}"
+    lfs: "{{ lfs_item.0.lfs }}"
     group: "{{ lfs_item.1 }}"
     mode_project: '2770'
     mode_version: '2770'
-    mode_dataset: '2750' 
+    mode_dataset: '2750'
   with_subelements:
     - "{{ lfs_mounts | selectattr('lfs', 'search', 'tmp[0-9]+$') | list }}"
     - 'groups'
   loop_control:
     loop_var: lfs_item
-  when: inventory_hostname in groups['sys-admin-interface']
+  when: inventory_hostname in groups['user-interface']
 ...

single_role_playbooks/subgroup_directories.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: sys-admin-interface
+- hosts: user-interface
   roles:
     - subgroup_directories
 ...