Feature/firewall

.ansible-lint

@@ -2,7 +2,8 @@
 exclude_paths:
   - '~/.ansible' # Exclude external playbooks.
 skip_list:
-  # We explicitly use latest combined with other tech to pin versions (e.g. Spacewalk).
+  # We explicitly use latest combined with other tech to pin versions (e.g. Pulp).
   - 'package-latest'  # "Package installs should not use latest (403)."
   - 'meta-no-info'  # "No 'galaxy_info' found in meta/main.yml of a role (701)."
+  - 'experimental'  # All rules tagged as experimental.
 ...

.circleci/config.yml

@@ -21,8 +21,8 @@ jobs:
           command: |
             python -m venv venv
             . venv/bin/activate
-            pip install "ansible-lint[community,yamllint]"
-            ansible-galaxy install -r galaxy-requirements.yml
+            pip install ansible-lint
+            ansible-galaxy install -r requirements.yml
       - run:
           name: run tests
           shell: /bin/bash
@@ -36,7 +36,7 @@ jobs:
             else
                 export ANSIBLE_ROLES_PATH="${HOME}/.ansible/roles/"
             fi
-            if ansible-lint -p --nocolor cluster.yml deploy-os_servers.yml > lint_results 2>&1; then
+            if ansible-lint -p --nocolor *.yml > lint_results 2>&1; then
               lint_errors=0
             else
               cat lint_results

README.md

@@ -144,7 +144,7 @@ pip3 install mitogen
 #### 1. First import the required roles and collections for the playbooks:
 
 ```bash
-ansible-galaxy install -r galaxy-requirements.yml
+ansible-galaxy install -r requirements.yml
 ```
 
 Note: the default location where these dependencies will get installed with the above command is ```${HOME}/.ansible/```.

cluster.yml

@@ -1,39 +1,7 @@
----
 #
-# Order of deployment required to prevent chicken versus the egg issues:
-#  0. For all deployment phases:
-#       export AI_PROXY="${jumphost_name}"
-#       export ANSIBLE_INVENTORY="static_inventories/${stack_name}.yml"
-#       ANSIBLE_VAULT_PASSWORD_FILE=".vault_pass.txt.${stack_name}"
-#  1. Use standard CentOS cloud image user 'centos' or 'root' user and without host key checking:
-#       export ANSIBLE_HOST_KEY_CHECKING=False
-#       ansible-playbook -u centos -l 'jumphost,cluster'  single_role_playbooks/admin_users.yml
-#       ansible-playbook -u root   -l 'docs'              single_role_playbooks/admin_users.yml
-#  2. Use local admin user's account and without host key checking:
-#       export ANSIBLE_HOST_KEY_CHECKING=False
-#       ansible-playbook -u [admin_account] single_role_playbooks/ssh_host_signer.yml
-#  3. Use local admin user's account and with strict host key checking to deploy everything else:
-#       export ANSIBLE_HOST_KEY_CHECKING=True
-#       ansible-playbook  -u [admin_account] cluster.yml
-#     This will configure:
-#       * Jumphost first as it is required to access the other machines.
-#       * Repo management server second as it is required for version control of the packages
-#         installed on all other machines except for the jumphost, which will always get the latest updates for security.
-#       * Basic roles for all cluster machines part 1:
-#           * Roles that do NOT require regular accounts or groups to be present.
-#       * An LDAP with regular user accounts, which may be required for additional roles.
-#             (E.g. a chmod or chgrp for a file/folder requires the corresponding user or group to be present.)
-#       * Basic roles for all cluster machines part 2:
-#           * Roles that DO depend on regular accounts and groups.
-#       * SAI as it is required to:
-#           * Configure layout on shared storage devices used by other machines.
-#           * Configure Slurm control and Slurm database.
-#       * DAI
-#       * UI
-#       * Compute nodes
-#       * Documentation server
+# See README.md for instructins how to use this playbook.
 #
-
+---
 #
 # Dummy play to ping jumphosts and establish a persisting SSH connection
 # before trying to connect to the machines behind the jumphost,

create-docs-server.yml

@@ -1,36 +1,13 @@
 #
-#   1.   $> cd git/league-of-robots
-# Create Python virtual environment (once)
-#   2.   $> python3 -m venv openstacksdk.venv
-# Activate virtual environment.
-#   3.   $> source openstacksdk.venv/bin/activate
-# Install OpenStack SDK (once).
-#   4.   $> pip3 install openstacksdk
-#        $> pip3 install ruamel.yaml
-# NOTE: Openstack RC file must be sourced first to be able to use Openstack API from SDK:
-#   5. Login to OpenStack web interface -> "Identity" -> "Application Credentials" -> click the "Create Application Credential" button.
-#      This will result in a popup window: specify "Name", "Expiration Date", "Expiration Time", leave the rest empty / use defaults and
-#      click the "Create Application Credential" button.
-#      In the new popup window click the "Download openrc file" button and save the generated *-openrc.sh file in the root of the repo.
-#   6. Source the downloaded file. E.g.:
-#        $> source ./[Application_Credential_Name]-openrc.sh
-#   7. Fetch Ansible dependencies
-#        $> ansible-galaxy install -r galaxy-requirements.yml
-# Configure this repo for deployment of a specifc HPC cluster.
-#   8. Source lor-init from this repo. E.g.:
-#        $> source ./lor-init
-#   9. Configure League of Robots for a specific cluster. E.g.:
-#        $> lor-config nb
-#  10. Execute playbook to create VMs. E.g.:
-#        $> ansible-playbook deploy-os_servers.yml
+# See README.md for instructins how to use this playbook.
 #
 ---
 - name: 'Sanity checks before we start.'
   hosts: localhost
   connection: local
   pre_tasks:
     - name: 'Verify Ansible version meets requirements.'
-      assert:
+      ansible.builtin.assert:
         that: "ansible_version.full is version_compare('2.10', '>=')"
         msg: 'You must update Ansible to at least 2.10.x to use this playbook.'
 ##############################################################################
@@ -47,6 +24,8 @@
     # which would fail to use the interpretor from an activated virtual environment.
     #
     - ansible_python_interpreter: python
+  roles:
+    - include_vars_from_other_groups
   tasks:
     - name: "Create {{ network_private_management_id }} network."
       openstack.cloud.network:
@@ -104,47 +83,39 @@
                      Allows all outbound traffic.
         wait: true
         timeout: "{{ openstack_api_timeout }}"
-    - name: "Add rule to {{ stack_prefix }}_webservers security group: allow SSH inbound from external jumphost on port 22."
+    - name: "Add rules to {{ stack_prefix }}_webservers security group: allow inbound SSH from jumphosts."
       openstack.cloud.security_group_rule:
         security_group: "{{ stack_prefix }}_webservers"
         direction: ingress
         protocol: tcp
         port_range_min: 22
         port_range_max: 22
-        remote_ip_prefix: "{{ ip_addresses[item].addr }}{{ ip_addresses[item].mask }}"
+        remote_ip_prefix: "{{ remote_ip_address }}{{ remote_ip_netmask }}"
         wait: true
         timeout: "{{ openstack_api_timeout }}"
-      with_items: "{{ jumphosts }}"
-    - name: "Add rule to {{ stack_prefix }}_webservers security group: allow HTTP inbound on port 80."
-      openstack.cloud.security_group_rule:
-        security_group: "{{ stack_prefix }}_webservers"
-        direction: ingress
-        protocol: tcp
-        port_range_min: 80
-        port_range_max: 80
-        remote_ip_prefix: 0.0.0.0/0
-        wait: true
-        timeout: "{{ openstack_api_timeout }}"
-    - name: "Add rule to {{ stack_prefix }}_webservers security group: allow HTTPS inbound on port 443."
-      openstack.cloud.security_group_rule:
-        security_group: "{{ stack_prefix }}_webservers"
-        direction: ingress
-        protocol: tcp
-        port_range_min: 443
-        port_range_max: 443
-        remote_ip_prefix: 0.0.0.0/0
-        wait: true
-        timeout: "{{ openstack_api_timeout }}"
-    - name: "Add rule to {{ stack_prefix }}_webservers security group: allow ICMP inbound."
+      vars:
+        remote_ip_address: "{{ lookup('vars', item.0.group)['ip_addresses'][item.1.hostname][item.1.network]['address'] }}"
+        remote_ip_netmask: "{{ lookup('vars', item.0.group)['ip_addresses'][item.1.hostname][item.1.network]['netmask'] }}"
+      with_subelements:
+        - "{{ jumphosts }}"
+        - hosts
+    - name: "Add rules to {{ stack_prefix }}_webservers security group: allow inbound HTTP(S) and ICMP."
       openstack.cloud.security_group_rule:
         security_group: "{{ stack_prefix }}_webservers"
         direction: ingress
-        protocol: icmp
-        port_range_min: -1  # ICMP protocol does not have any ports.
-        port_range_max: -1  # ICMP protocol does not have any ports.
+        protocol: "{{ item.protocol }}"
+        port_range_min: "{{ item.port }}"
+        port_range_max: "{{ item.port }}"
         remote_ip_prefix: 0.0.0.0/0
         wait: true
         timeout: "{{ openstack_api_timeout }}"
+      with_items:
+        - protocol: tcp
+          port: 80  # HTTP
+        - protocol: tcp
+          port: 443  # HTTPS
+        - protocol: icmp
+          port: -1  # ICMP protocol does not have any ports.
 ##############################################################################
 # Configure documentation server(s) from inventory using Openstack API.
 ##############################################################################
@@ -165,7 +136,7 @@
         state: present
         name: "{{ inventory_hostname }}"
         image: "{{ cloud_image }}"
-        flavor: "{{ flavor_docs }}"
+        flavor: "{{ cloud_flavor }}"
         security_groups: "{{ stack_prefix }}_webservers"
         auto_floating_ip: false
         nics:
@@ -229,18 +200,33 @@
     # as filtering directly during the API call is problematic.
     # Will filter the results for the relevant servers later on.
     #
+    - name: Get info on floating IPs from OpenStack API.
+      openstack.cloud.floating_ip_info:
+      register: api_fip_info
+    - name: Get info on networks from OpenStack API.
+      openstack.cloud.networks_info:
+      register: api_network_info
     - name: Get server info from OpenStack API.
       openstack.cloud.server_info:
       register: api_server_info
-    - name: "ToDo"
-      debug:
+    - name: "Add addresses to {{ playbook_dir }}/group_vars/{{ stack_name }}/ip_addresses.yml.new"
+      ansible.builtin.template:
+        src: "{{ playbook_dir }}/group_vars/template/ip_addresses.yml.j2"
+        dest: "{{ playbook_dir }}/group_vars/{{ stack_name }}/ip_addresses.yml.new"
+        mode: '0644'
+      vars:
+        relevant_servers_list: "{{ groups['docs'] | default([]) }}"
+        relevant_servers_info: "{{ api_server_info.openstack_servers | selectattr('name', 'in', relevant_servers_list) | list }}"
+    - name: ToDo
+      ansible.builtin.debug:
         msg: |
              ***********************************************************************************************************
              IMPORTANT: Manual work!
+                        Ansible created:
+                            {{ playbook_dir }}/group_vars/{{ stack_name }}/ip_addresses.yml.new
+                        Please inspect this file carefully with:
+                            diff -y {{ playbook_dir }}/group_vars/{{ stack_name }}/ip_addresses.yml{.new,}
+                        and if Ok execute:
+                            mv {{ playbook_dir }}/group_vars/{{ stack_name }}/ip_addresses.yml{.new,}
              ***********************************************************************************************************
-             Check if public IP address "{{ api_server_info.openstack_servers | selectattr('name', 'equalto', item) | map(attribute='accessIPv4') | first }}" for server "{{ item }}"
-             needs to be updated in:
-                 {{ playbook_dir }}/group_vars/all/ip_addresses.yml
-             ***********************************************************************************************************
-      with_items: "{{ groups['docs'] | default([]) }}"
-...
+...