Merge pull request #471 from Zarquan/20210428-zrq-spark-conf

20210428 zrq spark conf

deployments/hadoop-yarn/ansible/01-create-keypair.yml

@@ -24,6 +24,8 @@
   gather_facts: false
   hosts: localhost
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 

deployments/hadoop-yarn/ansible/01-create-network.yml

@@ -24,6 +24,8 @@
   gather_facts: false
   hosts: localhost
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 

deployments/hadoop-yarn/ansible/03-create-masters.yml

@@ -24,6 +24,8 @@
   gather_facts: false
   hosts: localhost
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 

deployments/hadoop-yarn/ansible/04-create-workers.yml

@@ -24,6 +24,8 @@
   gather_facts: false
   hosts: localhost
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 

deployments/hadoop-yarn/ansible/04-update-fedora.yml

@@ -25,8 +25,10 @@
 
 - name: "DNF update"
   gather_facts: false
-  hosts: masters:workers:zeppelin
+  hosts: all
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 

deployments/hadoop-yarn/ansible/05-config-ssh.yml

@@ -26,6 +26,8 @@
   vars:
     sshdir: "{{ lookup('env','HOME') }}/.ssh"
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 
@@ -35,18 +37,19 @@
         mode: 'u=rwx,g=rx,o=rx'
         state: directory
 
-    - name: "Discover our zeppelin node"
+    - name: "Discover our Zeppelin node"
       os_server_info:
         cloud:  "{{ cloudname }}"
         server: "{{ deployname }}-zeppelin"
       register:
-        zeppelinnodes
+        zeppelinnode
 
     - name: "Generate Ansible SSH config"
       template:
         src:  "templates/ssh-ansible.j2"
         dest: "{{ sshdir }}/ansible-config"
         mode: 'u=rw,g=,o='
+        force: true
 
     - name: "Check our local SSH config"
       stat:

deployments/hadoop-yarn/ansible/06-config-dns.yml

@@ -23,6 +23,8 @@
 - hosts: localhost
   gather_facts: false
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 
@@ -52,11 +54,10 @@
         src:  'templates/dns-hosts.j2'
         dest: "/tmp/aglais-dns-hosts"
 
-
 - hosts: zeppelin
   gather_facts: false
   tasks:
-    - name: "Deploy [/etc/hosts] to our Zeppelin node"
+    - name: "Deploy [/etc/hosts] file to our gateway node"
       become: true
       copy:
         src:  /tmp/aglais-dns-hosts
@@ -65,11 +66,10 @@
         group: root
         mode:  u=rw,g=r,o=r
 
-
-- hosts: masters:workers
+- hosts: all:!zeppelin
   gather_facts: false
   tasks:
-    - name: "Deploy [/etc/hosts] file to masters and workers"
+    - name: "Deploy [/etc/hosts] file to all our nodes"
       become: true
       copy:
         src:  /tmp/aglais-dns-hosts

deployments/hadoop-yarn/ansible/07-host-keys.yml

@@ -24,6 +24,9 @@
 
 - hosts: zeppelin
   gather_facts: false
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   tasks:
 
     - name: "ssh-keyscan by name"
@@ -50,7 +53,7 @@
         dest: "/tmp/aglais-ssh-hosts"
 
 
-- hosts: masters:workers:zeppelin
+- hosts: all
   gather_facts: false
   tasks:
     - name: "Deploy the known hosts file to [/etc/ssh/ssh_known_hosts]"
@@ -82,7 +85,3 @@
       when: filestat.stat.exists == false
 
 
-
-
-
-

deployments/hadoop-yarn/ansible/08-ping-test.yml

@@ -22,8 +22,11 @@
 
 ---
 - name: "Ping tests"
-  hosts: zeppelin:masters:workers
+  hosts: all
   gather_facts: false
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   tasks:
 
     - name: "Check we can connect"

deployments/hadoop-yarn/ansible/09-worker-volumes.yml → deployments/hadoop-yarn/ansible/09-cinder-volumes.yml

@@ -20,23 +20,25 @@
 #
 #
 
-- name: "Create volumes on workers"
+- name: "Create Cinder volumes"
   gather_facts: false
   hosts: localhost
   vars_files:
+    - config/ansible.yml
+    - config/openstack.yml
     - /tmp/ansible-vars.yml
   tasks:
 
-    - name: "Create the volumes"
+    - name: "Create Cinder volumes"
       include_tasks: "tasks/create-volumes.yml"
       loop:
-        "{{ groups['workers'] }}"
+        "{{ groups['all'] }}"
       loop_control:
         loop_var: vmname
 
-- name: "Mount volumes on workers"
+- name: "Mount Cinder volumes"
   gather_facts: false
-  hosts: workers
+  hosts: all
   vars_files:
     - /tmp/ansible-vars.yml
   tasks:

deployments/hadoop-yarn/ansible/10-install-java.yml

@@ -25,6 +25,9 @@
 - name: "Install Java"
   hosts: masters:workers:zeppelin
   gather_facts: false
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   tasks:
 
     - name: "Install Java"

deployments/hadoop-yarn/ansible/11-install-hadoop.yml

@@ -29,7 +29,9 @@
 - name: "Install Hadoop"
   hosts: masters:workers:zeppelin
   gather_facts: false
-
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   vars:
     # Empty on all hosts
     hddatalink: "{{ hostvars[inventory_hostname].paths.hddatalink }}"

deployments/hadoop-yarn/ansible/12-config-hadoop-core.yml

@@ -23,7 +23,9 @@
 - name: "Configure Hadoop [core-site.xml]"
   hosts: masters:workers:zeppelin
   gather_facts: false
-
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   vars:
     hdtempdest: "{{ hostvars[inventory_hostname].paths.hdtempdest }}"
     hdtemplink: "{{ hostvars[inventory_hostname].paths.hdtemplink }}"

deployments/hadoop-yarn/ansible/12-config-ssh-access.yml

@@ -30,6 +30,9 @@
 - name: "Create a SSH key pair for each master node"
   hosts: masters
   gather_facts: true
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   vars:
     username: "{{hostvars[inventory_hostname].login}}"
     keyname:  "{{inventory_hostname}}-{{username}}-rsa"
@@ -69,8 +72,8 @@
         owner: "{{ userinfo.name }}"
         group: "{{ userinfo.name }}"
 
-- name: "Install the public keys on all our nodes"
-  hosts: masters:workers
+- name: "Install the master keys on all our nodes"
+  hosts: all
   gather_facts: true
   vars:
     username: "{{hostvars[inventory_hostname].login}}"
@@ -99,7 +102,7 @@
         "{{ groups['masters'] }}"
 
 
-- name: "Configure Hadoop [workers] on master nodes"
+- name: "Install list of [workers] on [master] nodes"
   hosts: masters:zeppelin
   gather_facts: false
 

deployments/hadoop-yarn/ansible/13-config-hdfs-namenode.yml

@@ -23,6 +23,9 @@
 - name: "Configure HDFS namenode"
   hosts: master01
   gather_facts: false
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   vars:
     hdfsmetalink: "{{ hostvars[inventory_hostname].paths.hdfsmetalink }}"
     hdfsmetadest: "{{ hostvars[inventory_hostname].paths.hdfsmetadest }}"

deployments/hadoop-yarn/ansible/14-config-hdfs-workers.yml

@@ -23,6 +23,9 @@
 - name: "Configure Hadoop workers"
   hosts: workers
   gather_facts: false
+  vars_files:
+    - config/ansible.yml
+    - /tmp/ansible-vars.yml
   vars:
     hdfsdatalink: "{{ hostvars[inventory_hostname].paths.hdfsdatalink }}"
     hdfsdatadest: "{{ hostvars[inventory_hostname].paths.hdfsdatadest }}"