20220118 zrq arcus testing

.github/workflows/ci.yml

@@ -1,3 +1,25 @@
+#
+# <meta:header>
+#   <meta:licence>
+#     Copyright (c) 2022, ROE (http://www.roe.ac.uk/)
+#
+#     This information is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This information is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#   </meta:licence>
+# </meta:header>
+#
+#
+
 name: CI
 
 on: [push]
@@ -16,3 +38,4 @@ jobs:
           config_file: "yamllint.yml"
           file_or_dir: "deployments/hadoop-yarn/ansible"
           strict: false
+

deployments/common/manila/datashares.yaml

@@ -22,6 +22,7 @@
 datashares:
 
   - id: "GDR2"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-gaia-dr2-6514"
     mountpath: "/data/gaia/GDR2_6514"
     mountmode: "ro"
@@ -31,6 +32,7 @@ datashares:
         md5sum: eac3b823f896299905deeb15c22f9c60
 
   - id: "GEDR3-11932"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-gaia-edr3-11932"
     mountpath: "/data/gaia/GEDR3_11932"
     mountmode: "ro"
@@ -40,6 +42,7 @@ datashares:
         md5sum: 4076c289e9bcf0bb7ef95a8af065ccf5
 
   - id: "GEDR3-2048"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-gaia-edr3-2048"
     mountpath: "/data/gaia/GEDR3_2048"
     mountmode: "ro"
@@ -58,6 +61,7 @@ datashares:
         md5sum: 01f469f33798bdad68a028d2749ef3a2
 
   - id: "GEDR3-4096"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-gaia-edr3-4096"
     mountpath: "/data/gaia/GEDR3_4096"
     mountmode: "ro"
@@ -76,6 +80,7 @@ datashares:
         md5sum: d41d8cd98f00b204e9800998ecf8427e
 
   - id: "GEDR3-8192"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-gaia-edr3-8192"
     mountpath: "/data/gaia/GEDR3_8192"
     mountmode: "ro"
@@ -94,6 +99,7 @@ datashares:
         md5sum: d41d8cd98f00b204e9800998ecf8427e
 
   - id: "ALLWISE"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-wise-allwise"
     mountpath: "/data/wise/ALLWISE"
     mountmode: "ro"
@@ -103,6 +109,7 @@ datashares:
         md5sum: 7edcfd58e3c0a0efb24dcc9fa2bdc70d
 
   - id: "PS1"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-panstarrs-ps1"
     mountpath: "/data/panstarrs/PS1"
     mountmode: "ro"
@@ -112,6 +119,7 @@ datashares:
         md5sum: 532b1cb1760869f02a8efb360b95ab01
 
   - id: "2MASS"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-data-twomass-allsky"
     mountpath: "/data/twomass/2MASSPSC"
     mountmode: "ro"

deployments/common/manila/usershares.yaml

@@ -22,30 +22,37 @@
 usershares:
 
   - id: "nch"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-user-nch"
     mountpath: "/user/nch"
 
   - id: "zrq"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-user-zrq"
     mountpath: "/user/zrq"
 
   - id: "stv"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-user-stv"
     mountpath: "/user/stv"
 
   - id: "dcr"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-user-dcr"
     mountpath: "/user/dcr"
 
   - id: "test"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-test-data"
     mountpath: "/user/test"
 
   - id: "aglais-tools"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-tools"
     mountpath: "/opt/software"
 
   - id: "aglais-notebooks"
+    cloudname: "iris-gaia-data"
     sharename: "aglais-notebooks"
     mountpath: "/home/fedora/zeppelin-0.10.0-bin-all/notebook/"
 

deployments/common/openstack/config/arcus.yml

@@ -0,0 +1,35 @@
+#
+# <meta:header>
+#   <meta:licence>
+#     Copyright (c) 2022, ROE (http://www.roe.ac.uk/)
+#
+#     This information is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This information is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#   </meta:licence>
+# </meta:header>
+#
+#
+
+# Arcus cloud configuration
+openstack:
+    cloud:
+        name: arcus
+        networks:
+            internal:
+                cidr: '10.10.0.0/16'
+            external:
+                name: 'CUDN-Internet'
+            cephnet:
+                name: 'cephfs'
+                cidr: '10.9.0.0/16'
+

deployments/common/openstack/config/cumulus.yml

@@ -0,0 +1,36 @@
+#
+# <meta:header>
+#   <meta:licence>
+#     Copyright (c) 2022, ROE (http://www.roe.ac.uk/)
+#
+#     This information is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This information is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#   </meta:licence>
+# </meta:header>
+#
+#
+
+# Cumulus cloud configuration
+openstack:
+    cloud:
+        name: cumulus
+        networks:
+            internal:
+                cidr: '10.10.0.0/16'
+            external:
+                name: 'internet'
+            cephnet:
+                name: 'cumulus-internal'
+                cidr: '10.218.0.0/16'
+
+

deployments/hadoop-yarn/ansible/01-create-keypair.yml

@@ -33,6 +33,6 @@
       os_keypair:
         cloud: "{{ cloudname }}"
         state: present
-        name:  "{{ sshkeys }}"
+        name:  "{{ sshkeyname }}"
         public_key_file: "../../common/ssh/aglais-team-keys"
         wait: true

deployments/hadoop-yarn/ansible/01-create-network.yml

@@ -33,32 +33,31 @@
       os_network:
         cloud: "{{ cloudname }}"
         state: present
-        name: "{{ networks['internal'] }}"
+        name: "{{ openstack.networks.internal.name }}"
         shared:   false
         external: false
       register:
         agnetwork
 
-
     - name: "Create a subnet for our internal network"
       os_subnet:
         cloud: "{{ cloudname }}"
         state: present
-        name: "{{ agnetwork.network.name }}-subnet"
+        name: "{{ deployname }}-internal-subnet"
         network_name: "{{ agnetwork.network.name }}"
-        cidr: 10.10.0.0/16
+        cidr: "{{ openstack.networks.internal.cidr }}"
       register:
         agsubnet
 
-
     - name: "Create a router for our internal network"
       os_router:
         cloud: "{{ cloudname }}"
         state: present
-        name:    "{{ agnetwork.network.name }}-router"
-        network: "{{ networks['external'] }}"
+        name: "{{ deployname }}-internal-router"
+        network: "{{ openstack.networks.external.name }}"
         interfaces:
           - net:    "{{ agnetwork.network.name }}"
             subnet: "{{ agsubnet.subnet.name }}"
       register:
         agrouter
+

deployments/hadoop-yarn/ansible/03-create-masters.yml

@@ -55,8 +55,8 @@
         name:     "{{ deployname }}-{{ item }}"
         image:    "{{ hostvars[item]['image']  }}"
         flavor:   "{{ hostvars[item]['flavor'] }}"
-        network:  "{{ networks['internal'] }}"
-        key_name: "{{ sshkeys }}"
+        network:  "{{ openstack.networks.internal.name }}"
+        key_name: "{{ sshkeyname }}"
         auto_ip: false
         meta:
           "hostname={{ item }}"
@@ -66,3 +66,4 @@
         "{{ groups['masters'] }}"
       register:
         masternodes
+

deployments/hadoop-yarn/ansible/04-create-workers.yml

@@ -55,8 +55,8 @@
         name:     "{{ deployname }}-{{ item }}"
         image:    "{{ hostvars[item]['image']  }}"
         flavor:   "{{ hostvars[item]['flavor'] }}"
-        network:  "{{ networks['internal'] }}"
-        key_name: "{{ sshkeys }}"
+        network:  "{{ openstack.networks.internal.name }}"
+        key_name: "{{ sshkeyname }}"
         auto_ip: false
         meta:
           "hostname={{ item }}"
@@ -66,3 +66,4 @@
         "{{ groups['workers'] }}"
       register:
         workernodes
+

deployments/hadoop-yarn/ansible/04-update-fedora.yml

@@ -50,3 +50,4 @@
           - 'htop'
           - 'wget'
         state: present
+

deployments/hadoop-yarn/ansible/05-config-ssh.yml

@@ -65,3 +65,4 @@
         dest: "{{ sshdir }}/config"
         state: link
       when: filestat.stat.exists == false
+

deployments/hadoop-yarn/ansible/06-config-dns.yml

@@ -96,3 +96,4 @@
         owner: root
         group: root
         mode:  u=rw,g=r,o=r
+

deployments/hadoop-yarn/ansible/07-host-keys.yml

@@ -82,3 +82,4 @@
         dest: "{{ lookup('env','HOME') }}/.ssh/ansible-hosts"
         mode:  u=rw,g=r,o=r
       when: filestat.stat.exists == false
+

deployments/hadoop-yarn/ansible/08-ping-test.yml

@@ -35,3 +35,4 @@
     - name: "Check we can use sudo"
       become: true
       ping:
+

deployments/hadoop-yarn/ansible/09-cinder-volumes.yml

@@ -55,3 +55,4 @@
       loop_control:
         loop_var: disc
       when: ((disc.type == 'cinder') or (disc.type == 'local'))
+

deployments/hadoop-yarn/ansible/10-install-java.yml

@@ -49,3 +49,4 @@
         marker: '# {mark} Ansible managed Java'
         block: |
           export JAVA_HOME=/etc/alternatives/jre
+

deployments/hadoop-yarn/ansible/11-install-hadoop.yml

@@ -91,3 +91,4 @@
           export HADOOP_DATA={{hddatalink}}
           export HADOOP_CONF_DIR={{hdconf}}
           export HADOOP_LOG_DIR={{hdlogslink}}
+

deployments/hadoop-yarn/ansible/12-config-hadoop-core.yml

@@ -70,3 +70,4 @@
                 <name>hadoop.tmp.dir</name>
                 <value>{{hdtemplink}}</value>
             </property>
+

deployments/hadoop-yarn/ansible/12-config-ssh-access.yml

@@ -116,3 +116,4 @@
       template:
         src:  "templates/hadoop-workers.j2"
         dest: "{{hdhome}}/etc/hadoop/workers"
+

deployments/hadoop-yarn/ansible/13-config-hdfs-namenode.yml

@@ -128,3 +128,4 @@
                 <name>dfs.datanode.use.datanode.hostname</name>
                 <value>true</value>
             </property>
+

deployments/hadoop-yarn/ansible/14-config-hdfs-workers.yml

@@ -68,3 +68,4 @@
                 <name>dfs.datanode.data.dir</name>
                 <value>{{hdfsdatalink}}</value>
             </property>
+

deployments/hadoop-yarn/ansible/15-config-hdfs-security.yml

@@ -151,6 +151,7 @@
         port_range_min: 9866
         port_range_max: 9867
         remote_group: "{{ security['zeppelin'] }}"
+
 #
 # https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml
 #   dfs.datanode.http.address	            9864
@@ -191,3 +192,4 @@
 #   dfs.secondary.http.address                     50090 M2
 #   dfs.ha.zkfc.port                                8019
 #
+

deployments/hadoop-yarn/ansible/16-config-yarn-masters.yml

@@ -135,6 +135,7 @@
             </property>
 
             {{yarnconfig}}
+
     #
     # CapacityScheduler config.
     # https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html
@@ -174,3 +175,4 @@
             # https://stackoverflow.com/a/48170409
             export YARN_RESOURCEMANAGER_USER=fedora
             export YARN_NODEMANAGER_USER=fedora
+

deployments/hadoop-yarn/ansible/17-config-yarn-workers.yml

@@ -184,3 +184,4 @@
             # export YARN_NODEMANAGER_OPTS="--add-modules javax.activation"
             # https://stackoverflow.com/a/48170409
             # export YARN_NODEMANAGER_USER=fedora
+

deployments/hadoop-yarn/ansible/18-config-yarn-security.yml

@@ -218,3 +218,4 @@
         protocol:  'tcp'
         port_range_min: 8048
         port_range_max: 8048
+

deployments/hadoop-yarn/ansible/20-install-spark.yml

@@ -59,3 +59,4 @@
         block: |
           export SPARK_HOME={{sphome}}
           export PATH=${PATH}:{{sphome}}/bin
+

deployments/hadoop-yarn/ansible/21-config-spark-security.yml

@@ -101,3 +101,4 @@
         direction: 'ingress'
         protocol:  'tcp'
         remote_group: "{{ security['workers'] }}"
+