initial twitcher support (#76)

* initial twitcher support

Vagrantfile

@@ -20,8 +20,8 @@ Vagrant.configure("2") do |config|
     # wps.vm.box = "bento/ubuntu-18.04"
     # wps.vm.box = "bento/ubuntu-16.04"
     # wps.vm.box = "bento/debian-9"
-    # wps.vm.box = "bento/centos-7"
-    wps.vm.box = "bento/centos-6"
+    wps.vm.box = "bento/centos-7"
+    # wps.vm.box = "bento/centos-6"
     # wps.vm.box = "bento/fedora-27"
     wps.vm.hostname = "wps.local"
     wps.vm.network "private_network", ip: "192.168.128.100"

etc/sample-emu-with-twitcher.yml

@@ -0,0 +1,24 @@
+---
+server_name: 192.168.128.100
+service_enable_https: true
+db_install_postgresql: true
+db_install_sqlite: false
+wps_enable_https: false
+# Enable separate Fileserver for WPS outputs
+# fs_enabled: false
+# fs_host: "{{ server_name }}"
+# fs_port: 5001
+# Conda spec
+# conda_env_use_spec: false
+# Configuration for Emu WPS
+wps_services:
+  - name: emu
+    hostname: "{{ server_name }}"
+    fs_hostname: "{{ server_name }}"
+    port: 5000
+    extra_config: |
+      [data]
+      cache_path = /tmp/cache
+# twitcher
+twitcher_enabled: true
+twitcher_enable_https: true

etc/sample-vagrant.yml

@@ -1,8 +1,11 @@
 ---
 server_name: 192.168.128.100
-wps_enable_https: false
+service_enable_https: false
+#db_install_postgresql: true
+#db_install_sqlite: false
+# wps_enable_https: false
 # Enable separate Fileserver for WPS outputs
-fs_enabled: false
+# fs_enabled: false
 # fs_host: "{{ server_name }}"
 # fs_port: 5001
 # Conda spec
@@ -16,3 +19,6 @@ wps_services:
     extra_config: |
       [data]
       cache_path = /tmp/cache
+# twitcher
+twitcher_enabled: false
+#twitcher_enable_https: false

group_vars/all

@@ -4,31 +4,30 @@
 prefix: /usr/local
 src_dir: "{{ prefix }}/src"
 server_name: localhost
+service_add_user: true
+service_user: wps
+# service_uid: 500
+service_group: "{{ service_user }}"
+# service_gid: 100
+service_user_home: /var/lib/pywps
+# https
+service_enable_https: False
 
 # postgres
 db_install_postgresql: true
 db_install_sqlite: false
-db_name: pywps
 db_host: localhost
 db_port: 5432
-db_user: pywps
-db_password: pywps
-db_connect: "postgresql+psycopg2://{{ db_user }}:{{ db_password }}@{{ db_host }}:{{ db_port }}/{{ db_name }}"
+db_user: dbuser
+db_password: dbuser
 
 # nginx fileserver
 fs_enabled: false
 fs_host: localhost
 fs_port: 6000
 
 # WPS
-wps_add_user: true
-wps_user: wps
-# wps_uid: 1000
-wps_user_home: /var/lib/pywps
-wps_group: "{{ wps_user }}"
-# wps_gid: 1000
-wps_database: "{{ db_connect }}"
-wps_enable_https: false
+wps_database: "postgresql+psycopg2://{{ db_user }}:{{ db_password }}@{{ db_host }}:{{ db_port }}/pywps"
 wps_services: []
   # - name: emu
   #   repo: https://github.com/bird-house/emu.git
@@ -46,3 +45,7 @@ wps_services: []
   #   extra_config: |
   #     [data]
   #     cache_path = /tmp
+
+# twitcher
+twitcher_enabled: false
+twitcher_database: "postgresql+psycopg2://{{ db_user }}:{{ db_password }}@{{ db_host }}:{{ db_port }}/twitcher"

playbook.yml

@@ -26,7 +26,7 @@
     - role: jdauphant.ssl-certs
       tags:
         nginx
-      when: wps_enable_https
+      when: service_enable_https
     - certs
     - role: geerlingguy.nginx
       tags:
@@ -36,3 +36,4 @@
       tags:
         db
     - pywps
+    - twitcher

requirements.yml

@@ -2,20 +2,20 @@
 
 # miniconda
 # ---------
-# - src: andrewrothstein.miniconda
-- src: https://github.com/andrewrothstein/ansible-miniconda/archive/v5.2.1.tar.gz
+#- src: andrewrothstein.miniconda
+- src: https://github.com/andrewrothstein/ansible-miniconda/archive/v5.2.4.tar.gz
   name: andrewrothstein.miniconda
-# - src: andrewrothstein.bash
-- src: https://github.com/andrewrothstein/ansible-bash/archive/v1.1.1.tar.gz
+#- src: andrewrothstein.bash
+- src: https://github.com/andrewrothstein/ansible-bash/archive/v1.1.2.tar.gz
   name: andrewrothstein.bash
-# - src: andrewrothstein.unarchive-deps
-- src: https://github.com/andrewrothstein/ansible-unarchive-deps/archive/v1.0.10.tar.gz
+#- src: andrewrothstein.unarchive-deps
+- src: https://github.com/andrewrothstein/ansible-unarchive-deps/archive/v1.0.11.tar.gz
   name: andrewrothstein.unarchive-deps
 
 # epel
 # ----
 # Installs the EPEL repository (Extra Packages for Enterprise Linux) for RHEL/CentOS.
-# - src: geerlingguy.repo-epel
+#- src: geerlingguy.repo-epel
 - src: https://github.com/geerlingguy/ansible-role-repo-epel/archive/1.2.3.tar.gz
   name: geerlingguy.repo-epel
 
@@ -27,24 +27,24 @@
 
 # supervisor
 # ----------
-# - src: geerlingguy.supervisor
+#- src: geerlingguy.supervisor
 - src: https://github.com/geerlingguy/ansible-role-supervisor/archive/2.0.2.tar.gz
   name: geerlingguy.supervisor
 
 # nginx
 # -----
-# - src: geerlingguy.nginx
+#- src: geerlingguy.nginx
 - src: https://github.com/geerlingguy/ansible-role-nginx/archive/2.6.2.tar.gz
   name: geerlingguy.nginx
 
 # postgresql
 # ----------
-# - src: ANXS.postgresql
+#- src: ANXS.postgresql
 - src: https://github.com/ANXS/postgresql/archive/v1.10.1.tar.gz
   name: ANXS.postgresql
 
 # ssl-certs
 # ---------
-# - src: jdauphant.ssl-certs
-- src: https://github.com/jdauphant/ansible-role-ssl-certs/archive/v1.7.tar.gz
+#- src: jdauphant.ssl-certs
+- src: https://github.com/jdauphant/ansible-role-ssl-certs/archive/v1.7.1.tar.gz
   name: jdauphant.ssl-certs

roles/certs/tasks/main.yml

@@ -5,4 +5,4 @@
     mode: 0440
   tags:
     nginx
-  when: wps_enable_https
+  when: service_enable_https

roles/common/tasks/main.yml

@@ -14,19 +14,19 @@
   tags:
     - common
 
-- name: Add WPS group
+- name: Add service group
   group:
-    name: "{{ wps_group }}"
-    gid: "{{ wps_gid | default(omit) }}"
+    name: "{{ service_group }}"
+    gid: "{{ service_gid | default(omit) }}"
     state: present
 
-- name: Add WPS user
+- name: Add service user
   user:
-    name: "{{ wps_user }}"
-    uid: "{{ wps_uid | default(omit) }}"
-    groups: "{{ wps_group }}"
+    name: "{{ service_user }}"
+    uid: "{{ service_uid | default(omit) }}"
+    groups: "{{ service_group }}"
     system: yes
     shell: /sbin/nologin
     createhome: no
-    home: "{{ wps_user_home }}"
-  when: wps_add_user
+    home: "{{ service_user_home }}"
+  when: service_add_user

roles/pywps/tasks/conda.yml

@@ -1,6 +1,6 @@
 ---
 - name: Create Conda environment from environment file.
-  command: "{{ conda_location }}/bin/conda env update -p {{ conda_envs_dir}}/{{ item.name }}"
+  command: "{{ conda_bin }} env update -p {{ conda_envs_dir}}/{{ item.name }}"
   args:
     chdir: "{{ src_dir }}/{{ item.name }}"
   with_items: "{{ wps_services }}"
@@ -11,7 +11,7 @@
     - conda
 
 - name: Install additional Conda packages.
-  command: "{{ conda_location }}/bin/conda install -y -p {{ conda_envs_dir}}/{{ item.name }} gunicorn psycopg2"
+  command: "{{ conda_bin }} install -y -p {{ conda_envs_dir}}/{{ item.name }} gunicorn psycopg2"
   with_items: "{{ wps_services }}"
   when: conda_env.changed
   tags:

roles/twitcher/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: restart twitcher
+  supervisorctl:
+    name: "twitcher"
+    state: restarted
+    # config: "{{ supervisor_config_path}}/supervisord.conf"
+    username: "{{ supervisor_user }}"
+    password: "{{ supervisor_password }}"
+
+- name: restart nginx
+  service:
+    name: nginx
+    state: restarted

roles/twitcher/tasks/conda.yml

@@ -0,0 +1,23 @@
+---
+- name: Create Conda environment from environment file.
+  command: "{{ conda_bin }} env update -p {{ conda_envs_dir}}/twitcher"
+  args:
+    chdir: "{{ src_dir }}/twitcher"
+  register: conda_env_twitcher
+  tags:
+    - twitcher
+    - conda
+
+- name: Install additional Conda packages.
+  command: "{{ conda_bin }} install -y -p {{ conda_envs_dir}}/twitcher gunicorn psycopg2"
+  when: conda_env_twitcher.changed
+  tags:
+    - twitcher
+    - conda
+
+- name: Install twitcher with pip
+  command: "{{ conda_envs_dir }}/twitcher/bin/pip install -e {{ src_dir }}/twitcher"
+  register: conda
+  tags:
+    - twitcher
+    - conda

roles/twitcher/tasks/config.yml

@@ -0,0 +1,8 @@
+---
+- name: Copy twitcher config
+  template:
+    src: ./templates/twitcher.ini.j2
+    dest: "{{ twitcher_ini }}"
+  tags:
+    - twitcher
+    - conf

roles/twitcher/tasks/db.yml

@@ -0,0 +1,8 @@
+---
+- name: Migrate database.
+  command: "{{ conda_envs_dir }}/twitcher/bin/alembic -c {{ twitcher_ini }} upgrade head"
+  args:
+    chdir: "{{ src_dir }}/twitcher"
+  register: twitcher_db
+  tags:
+    - twitcher

roles/twitcher/tasks/folders.yml

@@ -0,0 +1,9 @@
+---
+- name: Create folders used by twitcher and set owner
+  file: path={{ item }} state=directory owner={{ twitcher_user }} group={{ twitcher_group }} mode=0755
+  with_items:
+    - /var/lib/twitcher
+    - /var/run
+  tags:
+    - twitcher
+    - conf

roles/twitcher/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+- include: source.yml
+  when: twitcher_enabled
+
+- include: conda.yml
+  when: twitcher_enabled
+
+- include: folders.yml
+  when: twitcher_enabled
+
+- include: config.yml
+  when: twitcher_enabled
+
+- include: db.yml
+  when: twitcher_enabled
+
+- include: sqlite.yml
+  when: twitcher_enabled and db_install_sqlite
+
+- include: supervisor.yml
+  when: twitcher_enabled
+
+- include: nginx.yml
+  when: twitcher_enabled
+
+- name: Assure Twitcher running at end of playbook
+  command: /bin/true
+  notify:
+    - restart twitcher
+  tags:
+    - twitcher
+    - conda
+    - conf
+  when: twitcher_enabled
+
+- name: Assure nginx running at end of playbook
+  command: /bin/true
+  notify:
+    - restart nginx
+  tags:
+    - nginx
+    - conf
+  when: twitcher_enabled

roles/twitcher/tasks/nginx.yml

@@ -0,0 +1,8 @@
+---
+- name: Copy nginx config to remote
+  template:
+    src: ./templates/nginx.conf.j2
+    dest: "{{ nginx_conf_path }}/twitcher.conf"
+  tags:
+    - twitcher
+    - conf

roles/twitcher/tasks/source.yml

@@ -0,0 +1,12 @@
+---
+- name: Clone Twitcher.
+  git:
+    repo: "https://github.com/bird-house/twitcher"
+    version: "{{ twitcher_version }}"
+    dest: "{{ src_dir }}/twitcher"
+    update: yes
+    force: yes
+  register: git
+  tags:
+    - twitcher
+    - git

roles/twitcher/tasks/sqlite.yml

@@ -0,0 +1,6 @@
+---
+- name: Update sqlite db file permissions
+  file: path="/var/lib/twitcher/twitcher.sqlite" state=touch owner={{ twitcher_user }} group={{ twitcher_group }} mode=0644
+  tags:
+    - twitcher
+    - conf

roles/twitcher/tasks/supervisor.yml

@@ -0,0 +1,9 @@
+---
+- name: Copy supervisord job file
+  template:
+    src: ./templates/supervisor.conf.j2
+    dest: "{{ supervisor_config_path }}/conf.d/twitcher.conf"
+    owner: root
+  tags:
+    - twitcher
+    - conf