From eec60cf999d224e0ec3bdff7f68106208d06415f Mon Sep 17 00:00:00 2001 From: pneerincx Date: Tue, 21 Sep 2021 13:38:00 +0200 Subject: [PATCH 1/5] Removed default from roles/mount_volume/defaults/main.yml and moved it to group_vars/compute_vm.yml as the values previously listed as defaults were for compute nodes. --- group_vars/compute_vm.yml | 10 ++++++++ roles/mount_volume/defaults/main.yml | 23 +++++++++++-------- .../tasks/main.yml | 0 3 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 group_vars/compute_vm.yml rename roles/{nfs_home_client => nfs_client}/tasks/main.yml (100%) diff --git a/group_vars/compute_vm.yml b/group_vars/compute_vm.yml new file mode 100644 index 000000000..328168631 --- /dev/null +++ b/group_vars/compute_vm.yml @@ -0,0 +1,10 @@ +--- +volumes: + - mount_point: '/local' + device: '/dev/vdb' + mounted_owner: root + mounted_group: root + mounted_mode: '0755' + mount_options: 'rw,relatime' + type: ext4 +... diff --git a/roles/mount_volume/defaults/main.yml b/roles/mount_volume/defaults/main.yml index 328168631..4b93da237 100644 --- a/roles/mount_volume/defaults/main.yml +++ b/roles/mount_volume/defaults/main.yml @@ -1,10 +1,13 @@ ---- -volumes: - - mount_point: '/local' - device: '/dev/vdb' - mounted_owner: root - mounted_group: root - mounted_mode: '0755' - mount_options: 'rw,relatime' - type: ext4 -... +# +# Example data structure used by the mount_volume role. +# +#--- +#volumes: +# - mount_point: '/local' +# device: '/dev/vdb' +# mounted_owner: root +# mounted_group: root +# mounted_mode: '0755' +# mount_options: 'rw,relatime' +# type: ext4 +#... diff --git a/roles/nfs_home_client/tasks/main.yml b/roles/nfs_client/tasks/main.yml similarity index 100% rename from roles/nfs_home_client/tasks/main.yml rename to roles/nfs_client/tasks/main.yml From 7664360473ef41e95d82a78bbcdc234db6479c53 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Tue, 21 Sep 2021 13:39:08 +0200 Subject: [PATCH 2/5] Removed unused group_vars/wingedhelix_cluster/secrets.yml.2021-09-13T18:30:42, which was encrypted with the wrong vault password for another cluster. --- .../secrets.yml.2021-09-13T18:30:42 | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 group_vars/wingedhelix_cluster/secrets.yml.2021-09-13T18:30:42 diff --git a/group_vars/wingedhelix_cluster/secrets.yml.2021-09-13T18:30:42 b/group_vars/wingedhelix_cluster/secrets.yml.2021-09-13T18:30:42 deleted file mode 100644 index 8ed63dee8..000000000 --- a/group_vars/wingedhelix_cluster/secrets.yml.2021-09-13T18:30:42 +++ /dev/null @@ -1,60 +0,0 @@ -$ANSIBLE_VAULT;1.2;AES256;nibbler -31396336343062363562376564313732303561666163386638666664313534646364333138653833 -3331383366326138623831613761336436643733643938350a356233346261666235353530623637 -31623361353236306435396433363433316162653865663162313366613935353038313336323136 -3738626333636435630a333130363964323265663063323061616162303730653463353932376464 -39396531353633333365633134343465343934396361656131666438336536653162363262663236 -66353834643561306434633934396535353639393833356636323765346332623632656266343664 -30303262616661656132373335613937303037306239373539363465373563633564663431393732 -37643937636237363631623032653964386136333232653239303565663065333465326137633139 -61383362313939636562316539633364363037333264326361353763653832316564346138636465 -31313038373364316338613730636430353132303535363133643363623733646634646137356431 -36316538376461663139663432323565306139353262386130333866393736356566333634613438 -62383132386337653630616237656639346666383838386538316332353066663861323162313531 -63363961353063303238383733373934646533323462356363613163656561323133626361316237 -63613330636338313061383562303436353534383234343966383832316164386136643563326266 -65653931316364373934653632396635323762643236666239653335616331343331623433646336 -62386531366163353438383763363136343434386165383364653634363336353631366239633237 -62633033326534333562646139376263646434303665386162333538376361366264653737623138 -34333234316134303435646131626130316439633836643133626536303332393038343965306361 -37353664323439333636383062363836383364623237323563386131626263613266646432306230 -63323634643432343134643339303339653536333266373063346262346266383137366266343231 -33343163333061336234666566343233373733356330646466333232316363323134336631346264 -64633236626634383934616437376335663430616131343165386665313335663037353962393830 -33626639316631663839663932326333303736373836346161356136363434656464383666663062 -31343534383733396538336461643761356564616262323464633932616439653933333265323836 -38613836633130653761333463343765366661353666343735396631646134383132393836343535 -34616131396133346338623638363532366262643131363665316237633863376333353532663836 -36323830363338323837376565636563313965316630396134396439373832343837323363383663 -63613834613834616561613230613838313061666365343764393038346336646538356136653962 -36386638643733663133376332346534633134393634653635363964306632656263643033393230 -61323936306363383533636362616237626539613232643432663331313364363066613937393965 -62303433303362396537346432306538653834633634393365373935613639616632303336396462 -62626164373736323032323431313534666662323665653038613939666165663566643661633463 -63663730383664366663333461623666356132643764306364363136663438396365323434306631 -62396161303130393334636164313837656363373330376466646463386265653565383434386162 -32333538313766316331373038653561363731653839353366376630343761383465366662303935 -61393365376535663661326361303238386464646362353836373730326331323265326662626435 -32303330346338623531633338383766323037386533353536613836623935643563656432343639 -36626539396634386662336430643132346538306334666665303039626466373337636533633930 -61303863666435626462383736343463666638323864306238303336653732306566373862393333 -36663536316238313164643632356634653638643439396232313663343464336564386433336130 -37343733343933303532323137366539376635343564303430336661663636663231323266643734 -39373962333633666130376439313764343031623436396230653934346237383931343763306463 -62636333316632616232353534633539346366316137333161316464343264383765396336373638 -64613466636664383230383836313664346632303933373232353165306532623966303231363734 -64393164663861316538303738653333343661303164366666636139656230303938303336626262 -39373261653063623535653436643966346463326165663430323633393665373937376464346635 -37353430343366616636613162633231313961313666613539376431336637303666633531666361 -32383666633430356536386530383836653833636631643830356131623332616261613338326534 -37653266376534303765396532656235313634613633313638386439336337393362313336313531 -32653665343965303662643062376262393736616536393563316464326161353731613136386263 -39643138316365643439643230656563636265663937373864643332373833653832653061613231 -36313733396238343739646662343463323664313734393930323062326263303166633932393732 -33623539353833353234306565393230616334656138666164306664613964613266313633353530 -36333832306264353534396564343533396331633030653066373231323834316232666166333334 -66393130393935323161656230346636356666616563366234353234373932383135626330343130 -65653831306539643561356362386665663663633131343938326136336631656666383338326163 -64623162616131633662393263353861323861623039396237376566653035333138343564323062 -36333538303730653265656636373639393637363432393162346133343564616137623031303864 -366233613734633461313033326336303861 From 2ea6630df68d31c07c6ba61a142807d54362042c Mon Sep 17 00:00:00 2001 From: pneerincx Date: Tue, 21 Sep 2021 13:41:10 +0200 Subject: [PATCH 3/5] Updated Winged Helix for new nfs_server and nfs_client roles. --- group_vars/wingedhelix_cluster/vars.yml | 5 +++-- static_inventories/wingedhelix_hosts.ini | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/group_vars/wingedhelix_cluster/vars.yml b/group_vars/wingedhelix_cluster/vars.yml index b43c3e2ea..dc484123d 100644 --- a/group_vars/wingedhelix_cluster/vars.yml +++ b/group_vars/wingedhelix_cluster/vars.yml @@ -125,10 +125,11 @@ regular_users: # # Shared storage related variables # -lustre_client_networks: 'tcp11(vlan???.lustre),tcp12(vlan???.lustre)' +#lustre_client_networks: 'tcp11(vlan???.lustre),tcp12(vlan???.lustre)' pfs_mounts: [ { pfs: 'umcgst12', - source: '', # this is going to be filled when we have shared NFS server + device: '/dev/vdb', + source: 'lookup nfs_server ip address in storage VLAN', type: 'nfs4', # checked with cat /proc/filesystem rw_options: 'defaults,_netdev,flock', ro_options: 'defaults,_netdev,ro' }, diff --git a/static_inventories/wingedhelix_hosts.ini b/static_inventories/wingedhelix_hosts.ini index 98a5929d1..0f9fd818c 100644 --- a/static_inventories/wingedhelix_hosts.ini +++ b/static_inventories/wingedhelix_hosts.ini @@ -10,6 +10,9 @@ wh-repo [docs] docs +[nfs_server] +wh-sai + [sys_admin_interface] wh-sai From a5bc242baa7fd77c35469593215cecc7cf567652 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Tue, 21 Sep 2021 13:51:13 +0200 Subject: [PATCH 4/5] Refactored nfs_home_server and nfs_home_client roles to more generic nfs_server and nfs_client roles that can also be used for other file systems/folders. Updated mount_volume and shared_storage roles too to make them work with new nfs_* roles. --- cluster.yml | 2 + roles/mount_volume/tasks/main.yml | 15 +++++++ roles/nfs_client/tasks/main.yml | 9 +---- roles/nfs_home_server/defaults/main.yml | 4 -- roles/nfs_home_server/tasks/main.yml | 31 -------------- roles/nfs_home_server/tasks/mount.yml | 40 ------------------- roles/nfs_server/handlers/main.yml | 30 ++++++++++++++ roles/nfs_server/tasks/main.yml | 33 +++++++++++++++ roles/shared_storage/tasks/main.yml | 4 +- single_group_playbooks/cluster_part2.yml | 2 + single_group_playbooks/nfs_server.yml | 14 +++++++ .../sys_admin_interface.yml | 1 - single_role_playbooks/mount_volume.yml | 2 +- 13 files changed, 102 insertions(+), 85 deletions(-) delete mode 100644 roles/nfs_home_server/defaults/main.yml delete mode 100644 roles/nfs_home_server/tasks/main.yml delete mode 100644 roles/nfs_home_server/tasks/mount.yml create mode 100644 roles/nfs_server/handlers/main.yml create mode 100644 roles/nfs_server/tasks/main.yml create mode 100644 single_group_playbooks/nfs_server.yml diff --git a/cluster.yml b/cluster.yml index 4d657a45c..c6ff6b407 100644 --- a/cluster.yml +++ b/cluster.yml @@ -51,6 +51,8 @@ - import_playbook: single_group_playbooks/ldap_server.yml +- import_playbook: single_group_playbooks/nfs_server.yml + - import_playbook: single_group_playbooks/cluster_part2.yml - import_playbook: single_group_playbooks/sys_admin_interface.yml diff --git a/roles/mount_volume/tasks/main.yml b/roles/mount_volume/tasks/main.yml index 5543d2e35..a3a1445aa 100644 --- a/roles/mount_volume/tasks/main.yml +++ b/roles/mount_volume/tasks/main.yml @@ -1,4 +1,19 @@ --- +- name: 'Compile list of volumes to mount for NFS servers.' + ansible.builtin.set_fact: + volumes: "{{ volumes | default([]) + [ volume ] }}" + vars: + volume: + mount_point: "/mnt/{{ item.pfs }}" + device: "{{ item.device }}" + mounted_owner: root + mounted_group: root + mounted_mode: '0755' + mount_options: 'rw,relatime' + type: ext4 + when: inventory_hostname in groups['nfs_server'] | default([]) + with_items: "{{ pfs_mounts | selectattr('type', 'search', 'nfs') | selectattr('device', 'defined') | list }}" + - name: 'Check the mount point.' command: "mountpoint {{ item.mount_point }}" register: mount_point_status diff --git a/roles/nfs_client/tasks/main.yml b/roles/nfs_client/tasks/main.yml index 47124d8e1..c36625e67 100644 --- a/roles/nfs_client/tasks/main.yml +++ b/roles/nfs_client/tasks/main.yml @@ -1,12 +1,7 @@ --- -- name: install nfs utils +- name: Install NFS utils. yum: name: nfs-utils - become: true - -- name: Add fstab entry - lineinfile: - path: /etc/exports - line: /home {{ nfs_server_ip | default(hostvars[groups['user_interface'][0]]['ansible_default_ipv4']['address']) }}:/home /home nfs rw 0 0 + when: pfs_mounts | selectattr('type', 'search', 'nfs') | list | length >= 1 become: true ... \ No newline at end of file diff --git a/roles/nfs_home_server/defaults/main.yml b/roles/nfs_home_server/defaults/main.yml deleted file mode 100644 index 6d341dc60..000000000 --- a/roles/nfs_home_server/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -nfs_network_range: "172.23.40.92/22" -nfs_mountpoint: "/home" -... diff --git a/roles/nfs_home_server/tasks/main.yml b/roles/nfs_home_server/tasks/main.yml deleted file mode 100644 index 233a4f697..000000000 --- a/roles/nfs_home_server/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- import_tasks: mount.yml - -- name: 'Install NFS utils.' - yum: - name: nfs-utils - become: true - -- name: 'Enable nfs service' - systemd: - name: nfs - enabled: yes - become: true - -- name: 'Add share entry to NFS exports.' - lineinfile: - path: /etc/exports - line: "{{nfs_mountpoint}} {{nfs_network_range}}(rw,sync,no_root_squash,no_subtree_check)" - register: etc_exports - become: true - -- name: 'Activate exports' - command: 'exportfs -a' - when: etc_exports.changed - -- name: 'Make sure nfs service is started' - systemd: - state: started - name: nfs - become: true -... \ No newline at end of file diff --git a/roles/nfs_home_server/tasks/mount.yml b/roles/nfs_home_server/tasks/mount.yml deleted file mode 100644 index e3fa79ec7..000000000 --- a/roles/nfs_home_server/tasks/mount.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: Make mount point for the cinder volume. - file: - path: "{{nfs_mountpoint}}" - mode: 0755 - state: directory - owner: root - group: root - become: true - -- name: Check the local mount point. - command: mountpoint "{{nfs_mountpoint}}" - register: mount_local - failed_when: false - -- name: Create an ext4 filesystem on /dev/vdb. - filesystem: - fstype: ext4 - dev: /dev/vdb - when: - mount_local.rc == 1 - become: true - -- name: Mount the volume. - mount: - path: "{{nfs_mountpoint}}" - src: /dev/vdb - fstype: ext4 - opts: rw,relatime - state: present - become: true - -- name: Mount all mountpoints from fstab. - command: mount -a - args: - warn: false - when: - mount_local.rc == 1 - become: true -... diff --git a/roles/nfs_server/handlers/main.yml b/roles/nfs_server/handlers/main.yml new file mode 100644 index 000000000..159ac07d8 --- /dev/null +++ b/roles/nfs_server/handlers/main.yml @@ -0,0 +1,30 @@ +--- +# +# Important: maintain correct handler order. +# Handlers are executed in the order in which they are defined +# and not in the order in which they are listed in a "notify: handler_name" statement! +# +# Restart before reload: an reload after a restart may be redundant but should not fail, +# but the other way around may fail when the impact of changes was too large for a reload. +# +- name: 'Restart nfs-server service.' + systemd: + name: 'nfs-server.service' + state: restarted + daemon_reload: true + become: true + listen: restart_nfs-server + +- name: 'Reload nfs-server service.' + systemd: + name: 'nfs-server.service' + state: reloaded + daemon_reload: true + become: true + listen: reload_nfs-server + +- name: '(Re)export all NFS shares.' + command: 'exportfs -a' + become: true + listen: export_nfs_shares +... \ No newline at end of file diff --git a/roles/nfs_server/tasks/main.yml b/roles/nfs_server/tasks/main.yml new file mode 100644 index 000000000..3a3544d91 --- /dev/null +++ b/roles/nfs_server/tasks/main.yml @@ -0,0 +1,33 @@ +# +# This role may depend on the mount_volume role to mount the filesystems that must be exported over NFS. +# +--- +- name: 'Install NFS utils.' + yum: + name: nfs-utils + notify: + - restart_nfs-server + - export_nfs_shares + become: true + +- name: 'Enable and start nfs-server service.' + systemd: + name: nfs-server.service + state: 'started' + enabled: true + daemon_reload: true + notify: + - export_nfs_shares + become: true + +- name: 'Add NFS share to /etc/exports.' + lineinfile: + path: /etc/exports + line: "/mnt/{{ item.pfs }} {{ network_private_storage_cidr }}(rw,sync,root_squash,no_subtree_check)" + with_items: "{{ pfs_mounts | selectattr('type', 'search', 'nfs') | selectattr('device', 'defined') | list }}" + notify: + - export_nfs_shares + become: true + +- meta: flush_handlers +... \ No newline at end of file diff --git a/roles/shared_storage/tasks/main.yml b/roles/shared_storage/tasks/main.yml index 6dce296fe..cdd9b44c5 100644 --- a/roles/shared_storage/tasks/main.yml +++ b/roles/shared_storage/tasks/main.yml @@ -28,7 +28,9 @@ opts: "{{ item.rw_options }}" state: 'mounted' with_items: "{{ pfs_mounts }}" - when: inventory_hostname in groups['sys_admin_interface'] + when: + - inventory_hostname in groups['sys_admin_interface'] + - inventory_hostname not in groups ['nfs_server'] | default([]) or 'nfs' not in item.type become: true - name: 'Create "home" Logical File System (LFS) on Physical File Systems (PFSs) mounted on SAIs.' diff --git a/single_group_playbooks/cluster_part2.yml b/single_group_playbooks/cluster_part2.yml index edd0c7d4a..5e81fc6f6 100644 --- a/single_group_playbooks/cluster_part2.yml +++ b/single_group_playbooks/cluster_part2.yml @@ -8,5 +8,7 @@ - sshd - ldap # client - regular_users + - lustre_client + - nfs_client - shared_storage ... diff --git a/single_group_playbooks/nfs_server.yml b/single_group_playbooks/nfs_server.yml new file mode 100644 index 000000000..b5d8544f1 --- /dev/null +++ b/single_group_playbooks/nfs_server.yml @@ -0,0 +1,14 @@ +# +# This play depends on: +# * cluster_part1.yml +# +--- +- import_playbook: pre_deploy_checks.yml + +- name: '###==-> Extra roles for NFS server. <-==###' + hosts: + - nfs_server + roles: + - mount_volume + - nfs_server +... diff --git a/single_group_playbooks/sys_admin_interface.yml b/single_group_playbooks/sys_admin_interface.yml index bf6e77223..0955c0291 100644 --- a/single_group_playbooks/sys_admin_interface.yml +++ b/single_group_playbooks/sys_admin_interface.yml @@ -10,7 +10,6 @@ hosts: - sys_admin_interface roles: - - mount_volume - slurm_management - prom_server - grafana diff --git a/single_role_playbooks/mount_volume.yml b/single_role_playbooks/mount_volume.yml index 519e180fa..cf1e4cdc4 100644 --- a/single_role_playbooks/mount_volume.yml +++ b/single_role_playbooks/mount_volume.yml @@ -1,7 +1,7 @@ - name: Mount the ceph volume hosts: + - nfs_server - compute_vm - - sys_admin_interface - deploy_admin_interface - data_transfer roles: From e07d50f6661ca826011ee3a3231dc92f5638554c Mon Sep 17 00:00:00 2001 From: pneerincx Date: Tue, 21 Sep 2021 14:05:08 +0200 Subject: [PATCH 5/5] Fixed linter issues. --- roles/lustre_client/defaults/main.yml | 4 ++-- roles/lustre_client/tasks/install.yml | 3 ++- roles/mount_volume/defaults/main.yml | 21 +++++++++++---------- roles/mount_volume/tasks/main.yml | 2 +- roles/nfs_client/tasks/main.yml | 2 +- roles/nfs_server/handlers/main.yml | 2 +- roles/nfs_server/tasks/main.yml | 5 +++-- 7 files changed, 21 insertions(+), 18 deletions(-) diff --git a/roles/lustre_client/defaults/main.yml b/roles/lustre_client/defaults/main.yml index 80b36f16b..45560a9e5 100644 --- a/roles/lustre_client/defaults/main.yml +++ b/roles/lustre_client/defaults/main.yml @@ -3,5 +3,5 @@ # We cannot define a default for lustre_client_networks here; # Always configure lustre_client_networks in group_vars/[group]/vars.yml # -#lustre_client_networks: 'tcp11(vlanMISSING.lustre),tcp12(vlanMISSING.lustre)' -... \ No newline at end of file +# lustre_client_networks: 'tcp11(vlanMISSING.lustre),tcp12(vlanMISSING.lustre)' +... diff --git a/roles/lustre_client/tasks/install.yml b/roles/lustre_client/tasks/install.yml index 19021aadb..73823715b 100644 --- a/roles/lustre_client/tasks/install.yml +++ b/roles/lustre_client/tasks/install.yml @@ -71,7 +71,8 @@ become: true notify: reboot -- meta: flush_handlers +- name: Flush handlers + meta: flush_handlers - name: Install the Lustre client. yum: diff --git a/roles/mount_volume/defaults/main.yml b/roles/mount_volume/defaults/main.yml index 4b93da237..74693e3b2 100644 --- a/roles/mount_volume/defaults/main.yml +++ b/roles/mount_volume/defaults/main.yml @@ -1,13 +1,14 @@ # # Example data structure used by the mount_volume role. # -#--- -#volumes: -# - mount_point: '/local' -# device: '/dev/vdb' -# mounted_owner: root -# mounted_group: root -# mounted_mode: '0755' -# mount_options: 'rw,relatime' -# type: ext4 -#... +# --- +# volumes: +# - mount_point: '/local' +# device: '/dev/vdb' +# mounted_owner: root +# mounted_group: root +# mounted_mode: '0755' +# mount_options: 'rw,relatime' +# type: ext4 +# ... +# diff --git a/roles/mount_volume/tasks/main.yml b/roles/mount_volume/tasks/main.yml index a3a1445aa..d96368248 100644 --- a/roles/mount_volume/tasks/main.yml +++ b/roles/mount_volume/tasks/main.yml @@ -4,7 +4,7 @@ volumes: "{{ volumes | default([]) + [ volume ] }}" vars: volume: - mount_point: "/mnt/{{ item.pfs }}" + mount_point: "/mnt/{{ item.pfs }}" device: "{{ item.device }}" mounted_owner: root mounted_group: root diff --git a/roles/nfs_client/tasks/main.yml b/roles/nfs_client/tasks/main.yml index c36625e67..c62311633 100644 --- a/roles/nfs_client/tasks/main.yml +++ b/roles/nfs_client/tasks/main.yml @@ -4,4 +4,4 @@ name: nfs-utils when: pfs_mounts | selectattr('type', 'search', 'nfs') | list | length >= 1 become: true -... \ No newline at end of file +... diff --git a/roles/nfs_server/handlers/main.yml b/roles/nfs_server/handlers/main.yml index 159ac07d8..b6b9f8ce2 100644 --- a/roles/nfs_server/handlers/main.yml +++ b/roles/nfs_server/handlers/main.yml @@ -27,4 +27,4 @@ command: 'exportfs -a' become: true listen: export_nfs_shares -... \ No newline at end of file +... diff --git a/roles/nfs_server/tasks/main.yml b/roles/nfs_server/tasks/main.yml index 3a3544d91..297be7d93 100644 --- a/roles/nfs_server/tasks/main.yml +++ b/roles/nfs_server/tasks/main.yml @@ -29,5 +29,6 @@ - export_nfs_shares become: true -- meta: flush_handlers -... \ No newline at end of file +- name: Flush handlers + meta: flush_handlers +...