Feature: new data transfer servers

deploy-os_servers.yml

@@ -108,6 +108,9 @@
           #   portip: 10.10.1.1
         wait: true
         timeout: "{{ openstack_api_timeout }}"
+    #
+    # Jumphosts security group.
+    #
     - name: "Create security group for {{ slurm_cluster_name }} jumphosts."
       openstack.cloud.security_group:
         state: present
@@ -159,6 +162,63 @@
         remote_ip_prefix: 0.0.0.0/0
         wait: true
         timeout: "{{ openstack_api_timeout }}"
+    #
+    # Data staging security group.
+    #
+    - name: "Create security group for {{ slurm_cluster_name }} data staging servers."
+      openstack.cloud.security_group:
+        state: present
+        name: "{{ slurm_cluster_name }}_ds"
+        description: |
+                     Security group for data staging severs without access to the internal network.
+                     Allows SSH inbound on both port 22 and 443.
+                     Allows ICMP inbound.
+                     Allows all outbound traffic.
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+    - name: "Add rule to {{ slurm_cluster_name }}_ds security group: allow SSH inbound on port 22."
+      openstack.cloud.security_group_rule:
+        security_group: "{{ slurm_cluster_name }}_ds"
+        direction: ingress
+        protocol: tcp
+        port_range_min: 22
+        port_range_max: 22
+        remote_ip_prefix: 0.0.0.0/0
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+    - name: "Add rule to {{ slurm_cluster_name }}_ds security group: allow SSH inbound on port 443."
+      openstack.cloud.security_group_rule:
+        security_group: "{{ slurm_cluster_name }}_ds"
+        direction: ingress
+        protocol: tcp
+        port_range_min: 443
+        port_range_max: 443
+        remote_ip_prefix: 0.0.0.0/0
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+    - name: "Add rule to {{ slurm_cluster_name }}_ds security group: allow LDAPS inbound on port 636."
+      openstack.cloud.security_group_rule:
+        security_group: "{{ slurm_cluster_name }}_ds"
+        direction: ingress
+        protocol: tcp
+        port_range_min: 636
+        port_range_max: 636
+        remote_ip_prefix: 0.0.0.0/0  # ToDo restrict to {{ ldap_uri }}
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+    - name: "Add rule to {{ slurm_cluster_name }}_ds security group: allow ICMP inbound."
+      openstack.cloud.security_group_rule:
+        security_group: "{{ slurm_cluster_name }}_ds"
+        direction: ingress
+        protocol: icmp
+        port_range_min: -1  # ICMP protocol does not have any ports.
+        port_range_max: -1  # ICMP protocol does not have any ports.
+        remote_ip_prefix: 0.0.0.0/0
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+    #
+    # Cluster security group.
+    #
     - name: Create security group for {{ slurm_cluster_name }} cluster machines behind jumphost.
       openstack.cloud.security_group:
         state: present
@@ -289,6 +349,94 @@
       when: (jumphost_vm.server.addresses | dict2items(key_name='vlan', value_name='specs') | json_query(query)) != network_private_management_id
       vars:
         query: '[?specs[?"OS-EXT-IPS:type"==`floating`]].vlan | [0]'
+    - name: Make sure jumphosts are started.
+      openstack.cloud.server_action:
+        action: start
+        server: "{{ inventory_hostname }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+##############################################################################
+# Configure data staging servers from inventory using Openstack API.
+##############################################################################
+- name: Create data staging servers.
+  hosts: data_transfer
+  connection: local
+  gather_facts: false
+  vars:
+    #
+    # Disable Ansible's interpretor detection logic,
+    # which fails to use the interpretor from an activated virtual environment
+    # and hence fails to find the OpenStackSDK if it was installed in a virtual environment.
+    #
+    - ansible_python_interpreter: python
+  tasks:
+    - name: Create persistent data volume for data staging servers.
+      openstack.cloud.volume:
+        display_name: "{{ inventory_hostname }}-volume"
+        size: "{{ local_volume_size_ds }}"
+        state: present
+        availability_zone: "{{ storage_availability_zone | default('nova') }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+    - name: Create data staging servers.
+      openstack.cloud.server:
+        state: present
+        name: "{{ inventory_hostname }}"
+        image: "{{ cloud_image }}"
+        volume_size: "{{ local_volume_size_root }}"
+        terminate_volume: true
+        boot_from_volume: true
+        flavor: "{{ flavor_ds }}"
+        security_groups: "{{ slurm_cluster_name }}_ds"
+        auto_floating_ip: false
+        nics:
+          - net-name: "{{ network_private_management_id }}"
+        userdata: |
+          #cloud-config
+          password: "{{ cloud_console_pass }}"
+          chpasswd: { expire: False }
+          #
+          # Add each entry to ~/.ssh/authorized_keys for the configured user
+          # or the first user defined in the user definition directive.
+          #
+          ssh_authorized_keys:
+          {% for public_key in public_keys_of_local_admins %}  - {{ public_key }}
+          {% endfor %}
+        availability_zone: "{{ availability_zone }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+      register: ds_vm
+    - name: Assign floating IPs to data staging servers.
+      openstack.cloud.floating_ip:
+        server: "{{ inventory_hostname }}"
+        state: present
+        reuse: true
+        network: "{{ network_public_external_id }}"
+        nat_destination: "{{ network_private_management_id }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+      #
+      # Known bug https://github.com/ansible/ansible/issues/57451
+      # openstack.cloud.floating_ip is not idempotent:
+      # Succeeds only the first time and throws error on any subsequent calls.
+      # Therefore we use a "when" with a silly complex jinja filter including a JMESpath query
+      # to check is the VM already has a floating IP linked to an interface in the correct VXLAN.
+      #
+      when: (ds_vm.server.addresses | dict2items(key_name='vlan', value_name='specs') | json_query(query)) != network_private_management_id
+      vars:
+        query: '[?specs[?"OS-EXT-IPS:type"==`floating`]].vlan | [0]'
+    - name: Attach local storage volume to data staging servers.
+      openstack.cloud.server_volume:
+        server: "{{ inventory_hostname }}"
+        volume: "{{ inventory_hostname }}-volume"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
+    - name: Make sure data staging servers are started.
+      openstack.cloud.server_action:
+        action: start
+        server: "{{ inventory_hostname }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
 ##############################################################################
 # Configure repo servers from inventory using Openstack API.
 ##############################################################################
@@ -331,7 +479,12 @@
         availability_zone: "{{ availability_zone }}"
         wait: true
         timeout: "{{ openstack_api_timeout }}"
-      register: repo_vm
+    - name: Make sure repo servers are started.
+      openstack.cloud.server_action:
+        action: start
+        server: "{{ inventory_hostname }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
 ##############################################################################
 # Configure UIs from inventory using Openstack API.
 ##############################################################################
@@ -375,6 +528,12 @@
         availability_zone: "{{ availability_zone }}"
         wait: true
         timeout: "{{ openstack_api_timeout }}"
+    - name: Make sure UIs are started.
+      openstack.cloud.server_action:
+        action: start
+        server: "{{ inventory_hostname }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
 ##############################################################################
 # Configure compute nodes from inventory using Openstack API.
 ##############################################################################
@@ -433,6 +592,12 @@
         volume: "{{ inventory_hostname }}-volume"
         wait: true
         timeout: "{{ openstack_api_timeout }}"
+    - name: Make sure compute nodes are started.
+      openstack.cloud.server_action:
+        action: start
+        server: "{{ inventory_hostname }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
 #############################################################################
 # Configure SAI from inventory using Openstack API.
 #############################################################################
@@ -550,6 +715,12 @@
         volume: "{{ inventory_hostname }}-volume"
         wait: true
         timeout: "{{ openstack_api_timeout }}"
+    - name: Make sure admin interface servers are started.
+      openstack.cloud.server_action:
+        action: start
+        server: "{{ inventory_hostname }}"
+        wait: true
+        timeout: "{{ openstack_api_timeout }}"
 ##############################################################################
 # Get IPs addresses from API for static hostname lookup with /etc/hosts.
 ##############################################################################
@@ -578,7 +749,9 @@
         dest: "{{ playbook_dir }}/group_vars/{{ slurm_cluster_name }}_cluster/ip_addresses.yml.new"
         mode: '0644'
       vars:
-        relevant_servers_list: "{{ groups['jumphost'] | default([]) + groups['repo'] | default([]) + groups['cluster'] | default([]) }}"
+        relevant_servers_list: "{{ groups['jumphost'] | default([]) + \
+                                   groups['data_transfer'] | default([]) + \
+                                   groups['repo'] | default([]) + groups['cluster'] | default([]) }}"
         relevant_servers_info: "{{ api_server_info.openstack_servers | selectattr('name', 'in', relevant_servers_list) | list }}"
     - name: "ToDo"
       debug:

group_vars/data_transfer.yml

@@ -0,0 +1,17 @@
+---
+firewall_allowed_tcp_ports:
+  - 22   # SSH.
+  - 443  # SSH.
+ssh_host_signer_hostnames: "{{ ansible_hostname }}\
+                            {% if slurm_cluster_domain | length %},{{ ansible_hostname }}.{{ slurm_cluster_domain }}{% endif %}\
+                            {% if public_ip_addresses is defined and public_ip_addresses[ansible_hostname] | length %},{{ public_ip_addresses[ansible_hostname] }}{% endif %}\
+                            {% for host in groups['jumphost'] %},{{ host }}+{{ ansible_hostname }}{% endfor %}"
+volumes:
+  - mount_point: '/groups'
+    device: '/dev/vdb'
+    mounted_owner: root
+    mounted_group: root
+    mounted_mode: '0755'
+    mount_options: 'rw,relatime'
+    type: ext4
+...

group_vars/deploy_admin_interface.yml

@@ -0,0 +1,10 @@
+---
+volumes:
+  - mount_point: '/apps'
+    device: '/dev/vdb'
+    mounted_owner: root
+    mounted_group: "{{ envsync_group }}"
+    mounted_mode: '2755'
+    mount_options: 'rw,relatime'
+    type: ext4
+...

group_vars/docs.yml

@@ -13,6 +13,10 @@ extra_jumphosts_for_docs_server:
   - 'corridor'    # Fender
   - 'dockingport' # Marvin
 ssh_host_signer_hostnames: "{{ ansible_fqdn }},{{ ansible_hostname }}\
-    {% for jumphost_for_this_cluster in groups['jumphost'] %},{{ jumphost_for_this_cluster }}+{{ ansible_hostname }}{% endfor %}\
-    {% for extra_jumphost in extra_jumphosts_for_docs_server %},{{ extra_jumphost }}+{{ ansible_hostname }}{% endfor %}"
+    {% for jumphost_for_this_cluster in groups['jumphost'] %}\
+      ,{{ jumphost_for_this_cluster }}+{{ ansible_hostname }}\
+    {% endfor %}\
+    {% for extra_jumphost in extra_jumphosts_for_docs_server %}\
+      ,{{ extra_jumphost }}+{{ ansible_hostname }}\
+    {% endfor %}"
 ...

group_vars/gearshift_cluster/vars.yml

@@ -63,6 +63,7 @@ ldap_group_quota_hard_limit_template: 'ruggroupumcgquotaLFS'
 filter_passwd: '(|(rugpersonentitlementvalue=scz)(rugpersonentitlementvalue=umcg))'
 filter_shadow: '(|(rugpersonentitlementvalue=scz)(rugpersonentitlementvalue=umcg))'
 pam_authz_search: '(|(&(objectClass=posixGroup)(cn=co_bbmri_g-GRP_Gearshift)(memberUid=$username))(&(cn=$username)(rugpersonentitlementvalue=umcg)))'
+data_transfer_only_group: umcg-sftp-only
 nameservers: [
   '172.23.40.244', # Order is important: local DNS for Isilon storage first!
   '8.8.4.4',       # Google DNS.

group_vars/jumphost.yml

@@ -1,9 +1,15 @@
 ---
 firewall_allowed_tcp_ports:
-  - "22"    # SSH.
-  - "443"   # SSH fallback when 22 is blocked.
-  - "3000"  # Grafana server.
+  - 22    # SSH.
+  - 443   # SSH fallback when 22 is blocked.
+  - 3000  # Grafana server.
 firewall_additional_rules:
   - "iptables -A INPUT -i eth1 -p tcp -s 129.125.2.233,129.125.2.225,129.125.2.226 --dport 9090 -j ACCEPT -m comment --comment 'prometheus server'"
-ssh_host_signer_hostnames: "{{ ansible_hostname }}{% if slurm_cluster_domain | length %}.{{ slurm_cluster_domain }}{% endif %},{{ ansible_hostname }}{% if public_ip_addresses is defined and public_ip_addresses[ansible_hostname] | length %},{{ public_ip_addresses[ansible_hostname] }}{% endif %}"
+ssh_host_signer_hostnames: "{{ ansible_hostname }}\
+                            {% if slurm_cluster_domain | length %}\
+                              ,{{ ansible_hostname }}.{{ slurm_cluster_domain }}\
+                            {% endif %}\
+                            {% if public_ip_addresses[ansible_hostname] is defined and public_ip_addresses[ansible_hostname] | length %}\
+                              ,{{ public_ip_addresses[ansible_hostname] }}\
+                            {% endif %}"
 ...

group_vars/nibbler_cluster/ip_addresses.yml

@@ -1,5 +1,10 @@
 ---
 ip_addresses:
+  nb-dt:
+    addr: 10.10.1.28
+    mask: /32
+    vlan: internal_management
+    fqdn:
   nb-repo:
     addr: 10.10.1.56
     mask: /32

group_vars/nibbler_cluster/vars.yml

@@ -58,9 +58,11 @@ ldap_binddn: 'cn=clusteradminumcg,o=asds'
 ldap_group_object_class: 'groupofnames'
 ldap_group_quota_soft_limit_template: 'ruggroupumcgquotaLFSsoft'
 ldap_group_quota_hard_limit_template: 'ruggroupumcgquotaLFS'
+data_transfer_only_group: umcg-sftp-only
 cloud_image: CentOS 7
 cloud_user: centos
 flavor_jumphost: m1.small
+flavor_ds: m1.small
 flavor_ui: m1.xlarge
 #flavor_vcompute: htc-node # Quota currently too small for 40 core compute nodes.
 flavor_vcompute: m1.xlarge
@@ -73,12 +75,13 @@ network_private_storage_id: internal_storage
 network_private_storage_cidr: '10.10.2.0/24'
 public_ip_addresses:
   tunnel: '195.169.22.136'
+  nb-ds: '195.169.22.166'
 availability_zone: nova
+local_volume_size_ds: 10000
 local_volume_size_repo: 20
 local_volume_size_vcompute: 1
 local_volume_size_dai: 200 # used for /apps, but test server shouldn't take too much space
 local_volume_size_sai: 1
-
 nameservers: [
   '8.8.4.4',  # Google DNS.
   '8.8.8.8',  # Google DNS.
@@ -109,6 +112,7 @@ regular_groups:
   - 'umcg-depad'
   - 'umcg-gcc'
   - 'umcg-lifelines'
+  - "{{ data_transfer_only_group }}"
 regular_users:
   - user: 'umcg-atd-ateambot'
     groups: ['umcg-atd']

group_vars/talos_cluster/vars.yml

@@ -61,6 +61,7 @@ ldap_binddn: 'cn=clusteradminumcg,o=asds'
 ldap_group_object_class: 'groupofnames'
 ldap_group_quota_soft_limit_template: 'ruggroupumcgquotaLFSsoft'
 ldap_group_quota_hard_limit_template: 'ruggroupumcgquotaLFS'
+data_transfer_only_group: umcg-sftp-only
 nameservers: [
   '/gcc-storage001.stor.hpc.local/172.23.40.244',  # Local DNS lookups for shared storage.
   '8.8.4.4',  # Google DNS.

inventory.py

@@ -16,7 +16,7 @@
 with the hostname of one of our proxy/jumphost servers.
 Note we only use hostnames and not FQDN nor IP addresses as those are managed
 together with usernames and other connection settings in
-our ~/.ssh/config files like this:
+our ~/.ssh/conf.d/ files like this:
 
 ########################################################################################################
 #