Kubectl namespaces test (#1)

Co-authored-by: przemyslavic <>

ansible/playbooks/roles/common/tasks/main.yml

@@ -81,6 +81,14 @@
   changed_when: false
   when: ansible_selinux is defined and ansible_selinux != False and ansible_selinux.status == 'enabled'
 
+# Before version 1.3 Epiphany updated /etc/motd file, so this task is necessary for upgrades
+# Can be removed after 1.2 deprecation
+- name: Truncate /etc/motd file
+  copy:
+    dest: /etc/motd
+    content: ""
+    force: true
+
 # motd is not used as Ubuntu has its own update-motd framework for dynamic motd generation
 # while for RedHat there is only /etc/motd that doesn't support simple configuration for colored output
 - name: Configure login message

ansible/playbooks/roles/postgresql/defaults/main.yml

@@ -15,6 +15,18 @@ pg:
   instantiated_service_name:
     Debian: postgresql@13-main
     RedHat: null
+  packages:
+    common:
+      Debian:
+        - postgresql-client-common
+        - postgresql-common
+      RedHat: []
+    version_specific:
+      Debian:
+        - postgresql-13
+        - postgresql-client-13
+      RedHat:
+        - postgresql13-server
   service_name:
     Debian: postgresql
     RedHat: postgresql-13

ansible/playbooks/roles/postgresql/defaults/upgrade.yml

@@ -20,6 +20,10 @@ pg:
   service_name:
     Debian: postgresql
     RedHat: postgresql-10
+  upgrade:
+    custom_tablespaces_fallback_file_path: /var/lib/epiphany/upgrade/state/custom-tablespaces.yml
+    postgresql_info_fallback_file_path: /var/lib/epiphany/upgrade/state/postgresql-info.yml
+    state_file_path: /var/lib/epiphany/upgrade/state/postgresql.uncompleted
   version: 10
 
 pgaudit:

ansible/playbooks/roles/postgresql/tasks/preflight/upgrade.yml

@@ -16,17 +16,31 @@
     name: pg_manifest
 
 - name: Check if upgrade was already run
-  stat:
-    path: "{{ pg_new.pg.data_dir[ansible_os_family] }}/PG_VERSION"
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
   become: true
-  register: stat_file_in_pg_new_data_dir
+  block:
+    - name: Check if PostgreSQL {{ pg_new.pg.version }} data directory exists
+      stat:
+        path: "{{ pg_new.pg.data_dir[ansible_os_family] }}"
+      register: new_pg_data_dir
+
+    - name: Check if PostgreSQL {{ pg_new.pg.version }} data directory contains files
+      when: new_pg_data_dir.stat.exists
+      find:
+        paths: "{{ pg_new.pg.data_dir[ansible_os_family] }}"
+      register: new_pg_data_dir_files
+
+    - name: Check if upgrade was run but failed
+      stat:
+        path: "{{ pg_old.pg.upgrade.state_file_path }}"
+        get_attributes: false
+        get_checksum: false
+        get_mime: false
+      register: stat_pg_upgrade_state_file
 
-- name: Set fact about PostgreSQL upgrade status
+- name: Set facts on PostgreSQL upgrade status
   set_fact:
-    pg_upgrade_already_run: "{{ stat_file_in_pg_new_data_dir.stat.exists }}"
+    pg_upgrade_already_run: "{{ new_pg_data_dir.stat.exists and new_pg_data_dir_files.matched > 0 }}"
+    pg_upgrade_failed: "{{ stat_pg_upgrade_state_file.stat.exists }}"
 
 - name: Check available disk space for new PostgreSQL data dir
   when: not pg_upgrade_already_run
@@ -104,19 +118,20 @@
 
 - name: Assert old PostgreSQL cluster is healthy
   when:
-    - not pg_upgrade_already_run
     - pg_manifest.specification.extensions.replication.enabled
+    - not pg_upgrade_already_run
+    - not pg_upgrade_failed
   become: true
   become_user: postgres
   vars:
     _repmgr_config_file: "{{ pg_old.repmgr.config_dir[ansible_os_family] }}/repmgr.conf"
   block:
     - name: Check if PostgreSQL {{ pg_old.pg.version }} cluster is healthy
       # when there are two primary nodes (after failover) rc is non-zero
+      # run on all nodes by purpose (to check exit code)
       command: >-
         {{ pg_old.repmgr.bin_dir[ansible_os_family] }}/repmgr -f {{ _repmgr_config_file }}
         cluster show
-      run_once: true
       changed_when: false
 
     - name: Check if PostgreSQL {{ pg_old.pg.version }} cluster node is healthy

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/pgaudit/packages.yml

@@ -1,5 +1,5 @@
 ---
-- name: PostgreSQL | Extensions | PgAudit | Install package(s) for PostgreSQL {{ new_version.pg.version }}
+- name: Extensions | PgAudit | Install package(s) for PostgreSQL {{ new_version.pg.version }}
   package:
     name: "{{ _packages[ansible_os_family] }}"
     state: present
@@ -8,4 +8,4 @@
       Debian: "{{ new_version.pgaudit.package_name.Debian }}={{ new_version.pgaudit.version.Debian + '-*' }}"
       RedHat: "{{ new_version.pgaudit.package_name.RedHat }}-{{ new_version.pgaudit.version.RedHat }}"
   module_defaults:
-    yum: { lock_timeout: "{{ yum_lock_timeout }}" }
+    yum: {lock_timeout: "{{ yum_lock_timeout }}"}

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/pgbouncer/packages.yml

@@ -8,4 +8,4 @@
       Debian: "pgbouncer={{ new_version.pgbouncer.version.Debian + '-*' }}"
       RedHat: "pgbouncer-{{ new_version.pgbouncer.version.RedHat }}"
   module_defaults:
-    yum: { lock_timeout: "{{ yum_lock_timeout }}" }
+    yum: {lock_timeout: "{{ yum_lock_timeout }}"}

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/packages.yml

@@ -1,14 +1,34 @@
 ---
-- name: PostgreSQL | Extensions | repmgr | Install package(s) for PostgreSQL {{ new_version.pg.version }}
+# On Ubuntu there is dependent 'repmgr-common' package. Till Epiphany 1.2.0 it was installed in the latest version.
+# apt module doesn't support --allow-downgrades (see https://github.com/ansible/ansible/issues/29451)
+# so we keep installed version if it's newer.
+- name: Set target version for repmgr-common package
+  when: ansible_os_family == 'Debian'
+  block:
+    # Fresh info is needed since repmgr-common migth be re-installed from different repository
+    - name: Extensions | repmgr | Refresh information on installed packages
+      package_facts:
+        manager: auto
+
+    - name: Extensions | repmgr | Set target version for repmgr-common package
+      set_fact:
+        repmgr_common_target_version: >-
+          {{ _installed_version is version(_target_version, '>') | ternary(_installed_version, _target_version + '-*') }}
+      vars:
+        _installed_version: "{{ ansible_facts.packages['repmgr-common'][0].version }}"
+        _target_version: "{{ new_version.repmgr.version.Debian }}"
+      when: ansible_facts.packages['repmgr-common'] is defined
+
+- name: Extensions | repmgr | Install package(s) for PostgreSQL {{ new_version.pg.version }}
   package:
     name: "{{ _packages[ansible_os_family] }}"
     state: present
   vars:
     _packages:
       Debian:
         - "{{ new_version.repmgr.package_name.Debian }}={{ new_version.repmgr.version.Debian + '-*' }}"
-        - "repmgr-common={{ new_version.repmgr.version.Debian + '-*' }}"
+        - repmgr-common={{ repmgr_common_target_version | default(new_version.repmgr.version.Debian + '-*') }}
       RedHat:
         - "{{ new_version.repmgr.package_name.RedHat }}-{{ new_version.repmgr.version.RedHat }}"
   module_defaults:
-    yum: { lock_timeout: "{{ yum_lock_timeout }}" }
+    yum: {lock_timeout: "{{ yum_lock_timeout }}"}

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/repmgr-register-primary.yml → ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/primary-register.yml

@@ -1,6 +1,6 @@
 ---
 # Done so as after an upgrade there is no possibility to log in
-- name: PostgreSQL | Extensions | repmgr | Alter super user and replication user in PostgreSQL
+- name: primary | extensions | repmgr | Alter replication users in PostgreSQL
   become_user: postgres
   vars:
     _users:
@@ -14,7 +14,9 @@
   run_once: true
   no_log: true
 
-- name: PostgreSQL | Extensions | repmgr | Register primary node
+# This command needs to be executed before any standby nodes are registered
+# '--force' overwrites existing node record
+- name: primary | extensions | repmgr | Register primary node
   become_user: postgres
   command: >-
     {{ new_version.repmgr.bin_dir[ansible_os_family] }}/repmgr primary register

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/repmgr-conf.yml

@@ -1,13 +1,13 @@
 ---
-- name: PostgreSQL | Extensions | repmgr | Get node id
+- name: Extensions | repmgr | Get node id
   command: >-
     grep -Po '(?<=^node_id=)\d+' "{{ old_version.repmgr.config_dir[ansible_os_family] }}/repmgr.conf"
   register: pg_node_id
   changed_when: false
 
-- name: PostgreSQL | Extensions | repmgr | Copy configuration file
+- name: Extensions | repmgr | Copy configuration file
   template:
-    src: roles/postgresql/templates/repmgr.conf.j2
+    src: repmgr.conf.j2
     dest: "{{ new_version.repmgr.config_dir[ansible_os_family] }}/repmgr.conf"
     owner: postgres
     group: postgres
@@ -20,10 +20,3 @@
     _replication_user_name: "{{ manifest.specification.extensions.replication.replication_user_name }}"
     _repmgr_database: "{{ manifest.specification.extensions.replication.repmgr_database }}"
     _repmgr_service_name: "{{ new_version.repmgr.service_name[ansible_os_family] }}"
-
-- name: PostgreSQL | Extensions | repmgr | Debian | Update symlink to config file
-  file:
-    src: "{{ new_version.repmgr.config_dir[ansible_os_family] }}/repmgr.conf"
-    dest: /etc/repmgr.conf
-    state: link
-  when: ansible_os_family == 'Debian'

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/standby-clone.yml

@@ -0,0 +1,85 @@
+---
+- name: standby | Stop {{ _service[ansible_os_family] }} service
+  systemd:
+    name: "{{ _service[ansible_os_family] }}"
+    state: stopped
+  vars:
+    _service:
+      # On Debian stop only instantiated service
+      Debian: "{{ new_version.pg.instantiated_service_name[ansible_os_family] }}"
+      RedHat: "{{ new_version.pg.service_name[ansible_os_family] }}"
+
+# 'repmgr standby clone' command uses 'pg_basebackup' that requires target directories to be empty
+# tablespace directories on a primary node are untouched
+- name: standby | Clean up tablespace directories
+  vars:
+    pattern: '*'
+  include_tasks: helpers/rm-dir-content.yml
+  loop_control:
+    loop_var: directory_path
+  loop: "{{ pg_custom_tablespace_locations }}"
+
+# Target data directory cannot contain running PostgreSQL instance
+# '--force' overwrites existing data directory
+- name: standby | extensions | repmgr | Clone standby node
+  become_user: postgres
+  command: >-
+    {{ new_version.repmgr.bin_dir[ansible_os_family] }}/repmgr standby clone
+      -h {{ pg_primary_node }}
+      -U {{ manifest.specification.extensions.replication.privileged_user_name }}
+      -d {{ manifest.specification.extensions.replication.repmgr_database }} -p 5432 --force
+  register: repmgr_standby_clone
+  changed_when: repmgr_standby_clone.rc == 0
+
+- name: Stop old PostgreSQL service
+  block:
+    - name: standby | Stop PostgreSQL {{ old_version.pg.version }} service
+      systemd:
+        name: "{{ service_name }}"
+        state: stopped
+      loop_control:
+        loop_var: service_name
+      loop: "{{ _services | reject('==', '') }}"
+      vars:
+        _services:
+          - "{{ old_version.pg.service_name[ansible_os_family] }}"
+          - "{{ old_version.pg.instantiated_service_name[ansible_os_family] }}"
+
+    - name: standby | Debian | Mask {{ old_version.pg.instantiated_service_name.Debian }} service to avoid port conflict
+      when: ansible_os_family == 'Debian'
+      systemd:
+        name: "{{ old_version.pg.instantiated_service_name[ansible_os_family] }}"
+        masked: true
+
+- name: Start new PostgreSQL service
+  block:
+    - name: standby | Debian | Unmask {{ new_version.pg.instantiated_service_name.Debian }} service
+      when: ansible_os_family == 'Debian'
+      systemd:
+        name: "{{ new_version.pg.instantiated_service_name[ansible_os_family] }}"
+        masked: false
+
+    - name: standby | Start PostgreSQL {{ new_version.pg.version }} service
+      systemd:
+        name: "{{ service_name }}"
+        state: started
+      loop_control:
+        loop_var: service_name
+      loop: "{{ _services | reject('==', '') }}"
+      vars:
+        _services:
+          - "{{ new_version.pg.service_name[ansible_os_family] }}"
+          - "{{ new_version.pg.instantiated_service_name[ansible_os_family] }}"
+
+# '--force' overwrites existing node record
+- name: standby | extensions | repmgr | Register standby node
+  become_user: postgres
+  command: >-
+    {{ new_version.repmgr.bin_dir[ansible_os_family] }}/repmgr standby register --force
+      --upstream-conninfo='host={{ pg_primary_node }},
+        user={{ manifest.specification.extensions.replication.replication_user_name }},
+        dbname={{ manifest.specification.extensions.replication.repmgr_database }},
+        connect_timeout=2'
+      --upstream-node-id 1
+  register: repmgr_standby_register
+  changed_when: repmgr_standby_register.rc == 0

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/repmgr-start-and-enable-service.yml → ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/start-and-enable-service.yml

@@ -3,12 +3,12 @@
   vars:
     _service_name: "{{ new_version.repmgr.service_name[ansible_os_family] }}"
   block:
-    - name: PostgreSQL | Extensions | repmgr | Start {{ _service_name }} service
+    - name: "{{ _node_role }} | extensions | repmgr | Start {{ _service_name }} service"
       systemd:
         name: "{{ _service_name }}"
         state: started
 
-    - name: PostgreSQL | Extensions | repmgr | Enable {{ _service_name }} service
+    - name: "{{ _node_role }} | extensions | repmgr | Enable {{ _service_name }} service"
       systemd:
         name: "{{ _service_name }}"
         enabled: true

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/update-repmgrd-config.yml

@@ -0,0 +1,7 @@
+---
+- name: "{{ _node_role }} | extensions | repmgr | Debian | Update REPMGRD_CONF in /etc/default/repmgrd"
+  when: ansible_os_family == 'Debian'
+  replace:
+    path: /etc/default/repmgrd
+    regexp: ^REPMGRD_CONF=\"/etc/postgresql/\d+/main/repmgr.conf\"
+    replace: REPMGRD_CONF="/etc/repmgr.conf"

ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/replication/pg-new/update-symlink-repmgr-bin.yml

@@ -0,0 +1,7 @@
+---
+- name: "{{ _node_role }} | extensions | repmgr | RedHat | Update repmgr symlink"
+  file:
+    src: "{{ new_version.repmgr.bin_dir.RedHat }}/repmgr"
+    dest: "/usr/bin/repmgr"
+    state: link
+  when: ansible_os_family == 'RedHat'