Feature: datatransfer servers for new GD infra.

cluster.yml

@@ -28,7 +28,7 @@
 - name: Run playbook for NFS servers.
   ansible.builtin.import_playbook: single_group_playbooks/nfs_server.yml
 
-- name: Run playbook for  all cluster machines part 2.
+- name: Run playbook for all cluster machines part 2.
   ansible.builtin.import_playbook: single_group_playbooks/cluster_part2.yml
 
 - name: Run playbook for sys admin interfaces.
@@ -46,6 +46,9 @@
 - name: Run playbook for documentation servers.
   ansible.builtin.import_playbook: single_group_playbooks/docs.yml
 
+- name: Run playbook for data transfer servers.
+  ansible.builtin.import_playbook: single_group_playbooks/data_transfer.yml
+
 - name: Run playbook for rsyslog servers.
   ansible.builtin.import_playbook: single_group_playbooks/rsyslog.yml
 ...

group_vars/betabarrel_cluster/ip_addresses.yml

@@ -9,6 +9,15 @@ ip_addresses:
       address: 195.169.22.140
       netmask: /32
       fqdn: 'NXDOMAIN'
+  bb-transfer:
+    bb_internal_management:
+      address: 10.10.1.29
+      netmask: /32
+      publicly_exposed: true  # This internal IP is linked to a public (floating) IP.
+    vlan16:
+      address: 195.169.22.143
+      netmask: /32
+      fqdn: 'NXDOMAIN'
   betabarrel:
     vlan983:
       address: 172.23.41.225

group_vars/betabarrel_cluster/vars.yml

@@ -350,6 +350,23 @@ remote_users_in_local_groups:
         'umcg-labgnkbh', 'umcg-patho', 'umcg-vipt',
     ]
 #
+# Local storage variables.
+#
+volume_group_folders: [
+  { mount_point: '/groups',
+    machines: "{{ groups['data_transfer'] }}",
+    mode: '2750',
+    groups: [
+        "{{ data_transfer_only_group }}"
+      ]},
+  { mount_point: '/groups',
+    machines: "{{ groups['data_transfer'] }}",
+    mode: '2770',
+    groups: [
+        'umcg-genomescan', 'umcg-gst'
+      ]},
+]
+#
 # Shared storage related variables
 #
 pfs_mounts:

group_vars/copperfist_cluster/ip_addresses.yml

@@ -9,6 +9,15 @@ ip_addresses:
       address: 195.169.22.177
       netmask: /32
       fqdn: 'NXDOMAIN'
+  cf-transfer:
+    cf_internal_management:
+      address: 10.10.1.24
+      netmask: /32
+      publicly_exposed: true  # This internal IP is linked to a public (floating) IP.
+    vlan16:
+      address: 195.169.22.155
+      netmask: /32
+      fqdn: 'NXDOMAIN'
   copperfist:
     vlan983:
       address: 172.23.41.226

group_vars/copperfist_cluster/vars.yml

@@ -352,6 +352,23 @@ remote_users_in_local_groups:
         'umcg-labgnkbh', 'umcg-patho', 'umcg-vipt',
     ]
 #
+# Local storage variables.
+#
+volume_group_folders: [
+  { mount_point: '/groups',
+    machines: "{{ groups['data_transfer'] }}",
+    mode: '2750',
+    groups: [
+        "{{ data_transfer_only_group }}"
+      ]},
+  { mount_point: '/groups',
+    machines: "{{ groups['data_transfer'] }}",
+    mode: '2770',
+    groups: [
+        'umcg-genomescan', 'umcg-gst'
+      ]},
+]
+#
 # Shared storage related variables
 #
 pfs_mounts:

group_vars/wingedhelix_cluster/ip_addresses.yml

@@ -1,5 +1,12 @@
 ---
 ip_addresses:
+  wh-dai:
+    wh_internal_management:
+      address: 10.10.1.222
+      netmask: /32
+    wh_internal_storage:
+      address: 10.10.2.218
+      netmask: /32
   wh-porch:
     wh_internal_management:
       address: 10.10.1.148
@@ -9,13 +16,6 @@ ip_addresses:
       address: 195.169.22.170
       netmask: /32
       fqdn: 'NXDOMAIN'
-  wh-dai:
-    wh_internal_management:
-      address: 10.10.1.222
-      netmask: /32
-    wh_internal_storage:
-      address: 10.10.2.218
-      netmask: /32
   wh-repo:
     wh_internal_management:
       address: 10.10.1.153
@@ -30,6 +30,15 @@ ip_addresses:
     wh_internal_storage:
       address: 10.10.2.99
       netmask: /32
+  wh-transfer:
+    wh_internal_management:
+      address: 10.10.1.96
+      netmask: /32
+      publicly_exposed: true  # This internal IP is linked to a public (floating) IP.
+    vlan16:
+      address: 195.169.22.152
+      netmask: /32
+      fqdn: 'NXDOMAIN'
   wh-vcompute01:
     wh_internal_management:
       address: 10.10.1.65

group_vars/wingedhelix_cluster/vars.yml

@@ -359,6 +359,23 @@ remote_users_in_local_groups:
         'umcg-labgnkbh', 'umcg-patho', 'umcg-vipt',
     ]
 #
+# Local storage variables.
+#
+volume_group_folders: [
+  { mount_point: '/groups',
+    machines: "{{ groups['data_transfer'] }}",
+    mode: '2750',
+    groups: [
+        "{{ data_transfer_only_group }}"
+      ]},
+  { mount_point: '/groups',
+    machines: "{{ groups['data_transfer'] }}",
+    mode: '2770',
+    groups: [
+        'umcg-genomescan', 'umcg-gst'
+      ]},
+]
+#
 # Shared storage related variables
 #
 pfs_mounts:

roles/openstack_networking/tasks/security_groups.yml

@@ -41,9 +41,9 @@
 #
 # Data staging security groups.
 #
-- name: "Create security groups for machines in 'data_staging' inventory group."
+- name: "Create security groups for machines in 'data_transfer' inventory group."
   block:
-    - name: "Create security group for {{ stack_prefix }} data staging servers."
+    - name: "Create security group for {{ stack_prefix }} data transfer servers."
       openstack.cloud.security_group:
         state: present
         name: "{{ stack_prefix }}_ds"
@@ -73,7 +73,7 @@
           port: 636  # LDAPS; ToDo: restrict to {{ ldap_uri }}
         - protocol: icmp
           port: -1  # ICMP protocol does not have any ports.
-  when: "'data_staging' in inventory_groups_with_hosts_created_in_openstack"
+  when: "'data_transfer' in inventory_groups_with_hosts_created_in_openstack"
 #
 # Cluster security groups.
 #

roles/rsyncd/meta/main.yml

@@ -5,4 +5,4 @@
 ---
 dependencies:
     - { role: sshd }
-...
+...

roles/rsyncd/tasks/main.yml

@@ -5,7 +5,7 @@
 # * Hence there is no systemd managed rsyncd running constantly
 #   and therefore no handler to (re)start a daemon.
 #
-# This role should not be confused with the rsync role, 
+# This role should not be confused with the rsync role,
 # which configures rsync on a managed host for use with the Ansible "ansible.posix.synchronize" task.
 #
 ---
@@ -27,4 +27,4 @@
     - rsyncd_all_groups.conf
     - rsyncd_data_transfer_only_group.conf
   become: true
-...
+...

roles/shared_storage/tasks/main.yml

@@ -180,7 +180,7 @@
         {%- endif -%}
       {%- endfor -%}"
     configured_lfs_mount_devices: "{{ configured_lfs_mount_devices_string
-      | regex_replace('#+$','')
+      | regex_replace('#+$', '')
       | split('#') }}"
   with_items: "{{ ansible_mounts
     | selectattr('mount', 'defined')

static_inventories/betabarrel_cluster.yml

@@ -33,6 +33,15 @@ all:
             - "{{ all.ip_addresses['umcg']['net4'] }}"
             - "{{ all.ip_addresses['rug']['bwp_net'] }}"
             - "{{ all.ip_addresses['rug']['operator'] }}"
+    data_transfer:
+      hosts:
+        bb-transfer:
+          cloud_flavor: m1.small
+          host_networks:
+            - name: "{{ stack_prefix }}_internal_management"
+              security_group: "{{ stack_prefix }}_ds"
+              assign_floating_ip: true
+          local_volume_size_extra: 2000
     docs:
       hosts:
         docs_on_merlin:
@@ -138,6 +147,7 @@ betabarrel_cluster:
     jumphost:
     cluster:
     chaperone:
+    data_transfer:
     docs:
     smb_server:
 ...

static_inventories/copperfist_cluster.yml

@@ -21,6 +21,15 @@ all:
             - "{{ all.ip_addresses['umcg']['net4'] }}"
             - "{{ all.ip_addresses['rug']['bwp_net'] }}"
             - "{{ all.ip_addresses['rug']['operator'] }}"
+    data_transfer:
+      hosts:
+        cf-transfer:
+          cloud_flavor: m1.small
+          host_networks:
+            - name: "{{ stack_prefix }}_internal_management"
+              security_group: "{{ stack_prefix }}_ds"
+              assign_floating_ip: true
+          local_volume_size_extra: 2000
     docs:
       hosts:
         docs_on_merlin:
@@ -126,6 +135,7 @@ copperfist_cluster:
     jumphost:
     cluster:
     chaperone:
+    data_transfer:
     docs:
     smb_server:
 ...

static_inventories/wingedhelix_cluster.yml

@@ -29,6 +29,15 @@ all:
             - name: "{{ stack_prefix }}_internal_management"
               security_group: "{{ stack_prefix }}_repo"
           swap_file_size: 2
+    data_transfer:
+      hosts:
+        wh-transfer:
+          cloud_flavor: m1.small
+          host_networks:
+            - name: "{{ stack_prefix }}_internal_management"
+              security_group: "{{ stack_prefix }}_ds"
+              assign_floating_ip: true
+          local_volume_size_extra: 800
     docs:
       hosts:
         docs_on_merlin:
@@ -149,6 +158,7 @@ wingedhelix_cluster:
     repo:
     cluster:
     chaperone:
+    data_transfer:
     docs:
     rsyslog:
     nfs_server: