From c52ec42e4eb3b0a1a94a33ad8de5bc5adefb1085 Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 19:59:49 +0200 Subject: [PATCH 1/8] Bugfix: added missing single_group_playbooks/data_transfer.yml to cluster.yml. --- cluster.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cluster.yml b/cluster.yml index 78cd122fb..f414f321c 100644 --- a/cluster.yml +++ b/cluster.yml @@ -28,7 +28,7 @@ - name: Run playbook for NFS servers. ansible.builtin.import_playbook: single_group_playbooks/nfs_server.yml -- name: Run playbook for all cluster machines part 2. +- name: Run playbook for all cluster machines part 2. ansible.builtin.import_playbook: single_group_playbooks/cluster_part2.yml - name: Run playbook for sys admin interfaces. @@ -46,6 +46,9 @@ - name: Run playbook for documentation servers. ansible.builtin.import_playbook: single_group_playbooks/docs.yml +- name: Run playbook for data transfer servers. + ansible.builtin.import_playbook: single_group_playbooks/data_transfer.yml + - name: Run playbook for rsyslog servers. ansible.builtin.import_playbook: single_group_playbooks/rsyslog.yml ... From 777f89703f8ec34648c65207880fc427ed388b2d Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 20:01:29 +0200 Subject: [PATCH 2/8] Bugfix: fixed typo in inventory group name when creating security group for data_transfer servers. --- roles/openstack_networking/tasks/security_groups.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/openstack_networking/tasks/security_groups.yml b/roles/openstack_networking/tasks/security_groups.yml index eebb8c704..c97e110a8 100644 --- a/roles/openstack_networking/tasks/security_groups.yml +++ b/roles/openstack_networking/tasks/security_groups.yml @@ -41,9 +41,9 @@ # # Data staging security groups. # -- name: "Create security groups for machines in 'data_staging' inventory group." +- name: "Create security groups for machines in 'data_transfer' inventory group." block: - - name: "Create security group for {{ stack_prefix }} data staging servers." + - name: "Create security group for {{ stack_prefix }} data transfer servers." openstack.cloud.security_group: state: present name: "{{ stack_prefix }}_ds" @@ -73,7 +73,7 @@ port: 636 # LDAPS; ToDo: restrict to {{ ldap_uri }} - protocol: icmp port: -1 # ICMP protocol does not have any ports. - when: "'data_staging' in inventory_groups_with_hosts_created_in_openstack" + when: "'data_transfer' in inventory_groups_with_hosts_created_in_openstack" # # Cluster security groups. # From 2c150195e2ca0fa3d96009a23874099f3673eff9 Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 20:05:39 +0200 Subject: [PATCH 3/8] Added data transfer server for BetaBarrel. --- .../betabarrel_cluster/ip_addresses.yml | 9 ++++++++ .../betabarrel_cluster/ip_addresses.yml.new | 21 +++++++++++++++++++ group_vars/betabarrel_cluster/vars.yml | 17 +++++++++++++++ static_inventories/betabarrel_cluster.yml | 10 +++++++++ 4 files changed, 57 insertions(+) create mode 100644 group_vars/betabarrel_cluster/ip_addresses.yml.new diff --git a/group_vars/betabarrel_cluster/ip_addresses.yml b/group_vars/betabarrel_cluster/ip_addresses.yml index d6d1aea91..6725b77d1 100644 --- a/group_vars/betabarrel_cluster/ip_addresses.yml +++ b/group_vars/betabarrel_cluster/ip_addresses.yml @@ -9,6 +9,15 @@ ip_addresses: address: 195.169.22.140 netmask: /32 fqdn: 'NXDOMAIN' + bb-transfer: + bb_internal_management: + address: 10.10.1.29 + netmask: /32 + publicly_exposed: true # This internal IP is linked to a public (floating) IP. + vlan16: + address: 195.169.22.143 + netmask: /32 + fqdn: 'NXDOMAIN' betabarrel: vlan983: address: 172.23.41.225 diff --git a/group_vars/betabarrel_cluster/ip_addresses.yml.new b/group_vars/betabarrel_cluster/ip_addresses.yml.new new file mode 100644 index 000000000..76b724a00 --- /dev/null +++ b/group_vars/betabarrel_cluster/ip_addresses.yml.new @@ -0,0 +1,21 @@ +--- +ip_addresses: + bb-porch: + bb_internal_management: + address: 10.10.1.164 + netmask: /32 + publicly_exposed: true # This internal IP is linked to a public (floating) IP. + vlan16: + address: 195.169.22.140 + netmask: /32 + fqdn: 'WARNING: Check manually for DNS CNAME records a.k.a. aliases: we cannot reverse lookup an alias automagically.' + bb-transfer: + bb_internal_management: + address: 10.10.1.29 + netmask: /32 + publicly_exposed: true # This internal IP is linked to a public (floating) IP. + vlan16: + address: 195.169.22.143 + netmask: /32 + fqdn: 'WARNING: Check manually for DNS CNAME records a.k.a. aliases: we cannot reverse lookup an alias automagically.' +... diff --git a/group_vars/betabarrel_cluster/vars.yml b/group_vars/betabarrel_cluster/vars.yml index 65b36f3c4..e7cb11a3c 100644 --- a/group_vars/betabarrel_cluster/vars.yml +++ b/group_vars/betabarrel_cluster/vars.yml @@ -350,6 +350,23 @@ remote_users_in_local_groups: 'umcg-labgnkbh', 'umcg-patho', 'umcg-vipt', ] # +# Local storage variables. +# +volume_group_folders: [ + { mount_point: '/groups', + machines: "{{ groups['data_transfer'] }}", + mode: '2750', + groups: [ + "{{ data_transfer_only_group }}" + ]}, + { mount_point: '/groups', + machines: "{{ groups['data_transfer'] }}", + mode: '2770', + groups: [ + 'umcg-genomescan', 'umcg-gst' + ]}, +] +# # Shared storage related variables # pfs_mounts: diff --git a/static_inventories/betabarrel_cluster.yml b/static_inventories/betabarrel_cluster.yml index c7353daf0..cd15e7153 100644 --- a/static_inventories/betabarrel_cluster.yml +++ b/static_inventories/betabarrel_cluster.yml @@ -33,6 +33,15 @@ all: - "{{ all.ip_addresses['umcg']['net4'] }}" - "{{ all.ip_addresses['rug']['bwp_net'] }}" - "{{ all.ip_addresses['rug']['operator'] }}" + data_transfer: + hosts: + bb-transfer: + cloud_flavor: m1.small + host_networks: + - name: "{{ stack_prefix }}_internal_management" + security_group: "{{ stack_prefix }}_ds" + assign_floating_ip: true + local_volume_size_extra: 2000 docs: hosts: docs_on_merlin: @@ -138,6 +147,7 @@ betabarrel_cluster: jumphost: cluster: chaperone: + data_transfer: docs: smb_server: ... From 71b13a1f7fe467adfa723bd33e9264620ce7ce78 Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 20:07:14 +0200 Subject: [PATCH 4/8] Added data transfer server for CopperFist. --- group_vars/copperfist_cluster/ip_addresses.yml | 9 +++++++++ group_vars/copperfist_cluster/vars.yml | 17 +++++++++++++++++ static_inventories/copperfist_cluster.yml | 10 ++++++++++ 3 files changed, 36 insertions(+) diff --git a/group_vars/copperfist_cluster/ip_addresses.yml b/group_vars/copperfist_cluster/ip_addresses.yml index d2efe2c6c..0a7902ba3 100644 --- a/group_vars/copperfist_cluster/ip_addresses.yml +++ b/group_vars/copperfist_cluster/ip_addresses.yml @@ -9,6 +9,15 @@ ip_addresses: address: 195.169.22.177 netmask: /32 fqdn: 'NXDOMAIN' + cf-transfer: + cf_internal_management: + address: 10.10.1.24 + netmask: /32 + publicly_exposed: true # This internal IP is linked to a public (floating) IP. + vlan16: + address: 195.169.22.155 + netmask: /32 + fqdn: 'NXDOMAIN' copperfist: vlan983: address: 172.23.41.226 diff --git a/group_vars/copperfist_cluster/vars.yml b/group_vars/copperfist_cluster/vars.yml index cbbf0bbb2..beab42f72 100644 --- a/group_vars/copperfist_cluster/vars.yml +++ b/group_vars/copperfist_cluster/vars.yml @@ -352,6 +352,23 @@ remote_users_in_local_groups: 'umcg-labgnkbh', 'umcg-patho', 'umcg-vipt', ] # +# Local storage variables. +# +volume_group_folders: [ + { mount_point: '/groups', + machines: "{{ groups['data_transfer'] }}", + mode: '2750', + groups: [ + "{{ data_transfer_only_group }}" + ]}, + { mount_point: '/groups', + machines: "{{ groups['data_transfer'] }}", + mode: '2770', + groups: [ + 'umcg-genomescan', 'umcg-gst' + ]}, +] +# # Shared storage related variables # pfs_mounts: diff --git a/static_inventories/copperfist_cluster.yml b/static_inventories/copperfist_cluster.yml index 8797e310d..c084ccd0a 100644 --- a/static_inventories/copperfist_cluster.yml +++ b/static_inventories/copperfist_cluster.yml @@ -21,6 +21,15 @@ all: - "{{ all.ip_addresses['umcg']['net4'] }}" - "{{ all.ip_addresses['rug']['bwp_net'] }}" - "{{ all.ip_addresses['rug']['operator'] }}" + data_transfer: + hosts: + cf-transfer: + cloud_flavor: m1.small + host_networks: + - name: "{{ stack_prefix }}_internal_management" + security_group: "{{ stack_prefix }}_ds" + assign_floating_ip: true + local_volume_size_extra: 2000 docs: hosts: docs_on_merlin: @@ -126,6 +135,7 @@ copperfist_cluster: jumphost: cluster: chaperone: + data_transfer: docs: smb_server: ... From c7313adaf71dcb75cdbc5f21900edb9bd4c5b134 Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 20:08:20 +0200 Subject: [PATCH 5/8] Removed unused temp file. --- .../betabarrel_cluster/ip_addresses.yml.new | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 group_vars/betabarrel_cluster/ip_addresses.yml.new diff --git a/group_vars/betabarrel_cluster/ip_addresses.yml.new b/group_vars/betabarrel_cluster/ip_addresses.yml.new deleted file mode 100644 index 76b724a00..000000000 --- a/group_vars/betabarrel_cluster/ip_addresses.yml.new +++ /dev/null @@ -1,21 +0,0 @@ ---- -ip_addresses: - bb-porch: - bb_internal_management: - address: 10.10.1.164 - netmask: /32 - publicly_exposed: true # This internal IP is linked to a public (floating) IP. - vlan16: - address: 195.169.22.140 - netmask: /32 - fqdn: 'WARNING: Check manually for DNS CNAME records a.k.a. aliases: we cannot reverse lookup an alias automagically.' - bb-transfer: - bb_internal_management: - address: 10.10.1.29 - netmask: /32 - publicly_exposed: true # This internal IP is linked to a public (floating) IP. - vlan16: - address: 195.169.22.143 - netmask: /32 - fqdn: 'WARNING: Check manually for DNS CNAME records a.k.a. aliases: we cannot reverse lookup an alias automagically.' -... From aa3acaabaaa6df66717a8edb7717b04a9e64b82c Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 20:35:29 +0200 Subject: [PATCH 6/8] Added data transfer server for WingedHelix. --- .../wingedhelix_cluster/ip_addresses.yml | 23 +++++++++++++------ group_vars/wingedhelix_cluster/vars.yml | 17 ++++++++++++++ static_inventories/wingedhelix_cluster.yml | 10 ++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/group_vars/wingedhelix_cluster/ip_addresses.yml b/group_vars/wingedhelix_cluster/ip_addresses.yml index 0fc01acad..8c393aa45 100644 --- a/group_vars/wingedhelix_cluster/ip_addresses.yml +++ b/group_vars/wingedhelix_cluster/ip_addresses.yml @@ -1,5 +1,12 @@ --- ip_addresses: + wh-dai: + wh_internal_management: + address: 10.10.1.222 + netmask: /32 + wh_internal_storage: + address: 10.10.2.218 + netmask: /32 wh-porch: wh_internal_management: address: 10.10.1.148 @@ -9,13 +16,6 @@ ip_addresses: address: 195.169.22.170 netmask: /32 fqdn: 'NXDOMAIN' - wh-dai: - wh_internal_management: - address: 10.10.1.222 - netmask: /32 - wh_internal_storage: - address: 10.10.2.218 - netmask: /32 wh-repo: wh_internal_management: address: 10.10.1.153 @@ -30,6 +30,15 @@ ip_addresses: wh_internal_storage: address: 10.10.2.99 netmask: /32 + wh-transfer: + wh_internal_management: + address: 10.10.1.96 + netmask: /32 + publicly_exposed: true # This internal IP is linked to a public (floating) IP. + vlan16: + address: 195.169.22.152 + netmask: /32 + fqdn: 'NXDOMAIN' wh-vcompute01: wh_internal_management: address: 10.10.1.65 diff --git a/group_vars/wingedhelix_cluster/vars.yml b/group_vars/wingedhelix_cluster/vars.yml index be43f2043..f6c6e0a13 100644 --- a/group_vars/wingedhelix_cluster/vars.yml +++ b/group_vars/wingedhelix_cluster/vars.yml @@ -359,6 +359,23 @@ remote_users_in_local_groups: 'umcg-labgnkbh', 'umcg-patho', 'umcg-vipt', ] # +# Local storage variables. +# +volume_group_folders: [ + { mount_point: '/groups', + machines: "{{ groups['data_transfer'] }}", + mode: '2750', + groups: [ + "{{ data_transfer_only_group }}" + ]}, + { mount_point: '/groups', + machines: "{{ groups['data_transfer'] }}", + mode: '2770', + groups: [ + 'umcg-genomescan', 'umcg-gst' + ]}, +] +# # Shared storage related variables # pfs_mounts: diff --git a/static_inventories/wingedhelix_cluster.yml b/static_inventories/wingedhelix_cluster.yml index f791cf055..8aaaef71c 100644 --- a/static_inventories/wingedhelix_cluster.yml +++ b/static_inventories/wingedhelix_cluster.yml @@ -29,6 +29,15 @@ all: - name: "{{ stack_prefix }}_internal_management" security_group: "{{ stack_prefix }}_repo" swap_file_size: 2 + data_transfer: + hosts: + wh-transfer: + cloud_flavor: m1.small + host_networks: + - name: "{{ stack_prefix }}_internal_management" + security_group: "{{ stack_prefix }}_ds" + assign_floating_ip: true + local_volume_size_extra: 800 docs: hosts: docs_on_merlin: @@ -149,6 +158,7 @@ wingedhelix_cluster: repo: cluster: chaperone: + data_transfer: docs: rsyslog: nfs_server: From 7e9aac1b4a96a8832303c24e773cb2ad0903bac0 Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 20:50:30 +0200 Subject: [PATCH 7/8] Fixed linter issues. --- roles/rsyncd/meta/main.yml | 2 +- roles/rsyncd/tasks/main.yml | 2 +- roles/shared_storage/tasks/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/rsyncd/meta/main.yml b/roles/rsyncd/meta/main.yml index 1ce5681fa..d6ea1c7aa 100644 --- a/roles/rsyncd/meta/main.yml +++ b/roles/rsyncd/meta/main.yml @@ -5,4 +5,4 @@ --- dependencies: - { role: sshd } -... \ No newline at end of file +... diff --git a/roles/rsyncd/tasks/main.yml b/roles/rsyncd/tasks/main.yml index 296d11dd4..72cf6bc92 100644 --- a/roles/rsyncd/tasks/main.yml +++ b/roles/rsyncd/tasks/main.yml @@ -5,7 +5,7 @@ # * Hence there is no systemd managed rsyncd running constantly # and therefore no handler to (re)start a daemon. # -# This role should not be confused with the rsync role, +# This role should not be confused with the rsync role, # which configures rsync on a managed host for use with the Ansible "ansible.posix.synchronize" task. # --- diff --git a/roles/shared_storage/tasks/main.yml b/roles/shared_storage/tasks/main.yml index e2b006706..b20eac48a 100644 --- a/roles/shared_storage/tasks/main.yml +++ b/roles/shared_storage/tasks/main.yml @@ -180,7 +180,7 @@ {%- endif -%} {%- endfor -%}" configured_lfs_mount_devices: "{{ configured_lfs_mount_devices_string - | regex_replace('#+$','') + | regex_replace('#+$', '') | split('#') }}" with_items: "{{ ansible_mounts | selectattr('mount', 'defined') From 0921280c4839a9ffef06925cdf482b55af4661e7 Mon Sep 17 00:00:00 2001 From: Pieter Neerincx Date: Thu, 6 Apr 2023 20:54:50 +0200 Subject: [PATCH 8/8] Fixed linter issues. --- roles/rsyncd/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rsyncd/tasks/main.yml b/roles/rsyncd/tasks/main.yml index 72cf6bc92..4ed5fdf61 100644 --- a/roles/rsyncd/tasks/main.yml +++ b/roles/rsyncd/tasks/main.yml @@ -27,4 +27,4 @@ - rsyncd_all_groups.conf - rsyncd_data_transfer_only_group.conf become: true -... \ No newline at end of file +...