postgresql_pg_hba: bulk rule editing

changelogs/fragments/303-postgresql_pg_hba_add_bulk_rule_arguments.yml

@@ -0,0 +1,7 @@
+minor_changes:
+  - "postgresql_pg_hba - add argument ``rules`` to specify a list of rules using the normal rule-specific argument in each item (https://github.com/ansible-collections/community.postgresql/issues/297)."
+  - >-
+    postgresql_pg_hba - add argument ``rules_behavior`` (choices: conflict (default), combine) to fail when ``rules``
+    and normal rule-specific arguments are given or, when ``combine``, use them as defaults for the ``rules`` items
+    (https://github.com/ansible-collections/community.postgresql/issues/297).
+  - "postgresql_pg_hba - add argument ``overwrite`` (bool, default: false) to remove unmanaged rules (https://github.com/ansible-collections/community.postgresql/issues/297)."

plugins/modules/postgresql_pg_hba.py

@@ -90,13 +90,35 @@
     type: str
     default: sdu
     choices: [ sdu, sud, dsu, dus, usd, uds ]
+  overwrite:
+    description:
+      - Remove all existing rules before adding rules. (Like I(state=absent) for all pre-existing rules.)
+    type: bool
+    default: false
   keep_comments_at_rules:
     description:
       - If C(true), comments that stand together with a rule in one line are kept behind that line.
       - If C(false), such comments are moved to the beginning of the file, like all other comments.
     type: bool
     default: false
     version_added: '1.5.0'
+  rules:
+    description:
+      - A list of objects, specifying rules for the pg_hba.conf. Use this to manage multiple rules at once.
+      - "Each object can have the following keys (the 'rule-specific arguments'), which are treated the same as if they were arguments of this module:"
+      - C(address), C(comment), C(contype), C(databases), C(method), C(netmask), C(options), C(state), C(users)
+      - See also C(rules_behavior).
+    type: list
+    elements: dict
+  rules_behavior:
+    description:
+      - "Configure how the I(rules) argument works together with the rule-specific arguments outside the I(rules) argument."
+      - See I(rules) for the complete list of rule-specific arguments.
+      - When set to C(conflict), fail if I(rules) and, for example, I(address) are set.
+      - If C(combine), the normal rule-specific arguments are not defining a rule, but are used as defaults for the arguments in the I(rules) argument.
+    type: str
+    choices: [ conflict, combine ]
+    default: conflict
   state:
     description:
       - The lines will be added/modified when C(state=present) and removed when C(state=absent).
@@ -174,6 +196,29 @@
     source: ::/0
     keep_comments_at_rules: true
     comment: "this rule is an example"
+
+- name: Replace everything with a new set of rules
+  community.postgresql.postgresql_pg_hba:
+    dest: /var/lib/postgres/data/pg_hba.conf
+    overwrite: true # remove preexisting rules
+
+    # custom defaults
+    rules_behavior: combine
+    contype: hostssl
+    address: 2001:db8::/64
+    comment: added in bulk
+
+    rules:
+    - users: user1
+      databases: db1
+      # contype, address and comment come from custom default
+    - users: user2
+      databases: db2
+      comment: added with love # overwrite custom default for this rule
+      # contype and address come from custom default
+    - users: user3
+      databases: db3
+      # contype, address and comment come from custom default
 '''
 
 RETURN = r'''
@@ -221,6 +266,7 @@
 import tempfile
 import shutil
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
+
 # from ansible.module_utils.postgres import postgres_common_argument_spec
 
 PG_HBA_METHODS = ["trust", "reject", "md5", "password", "gss", "sspi", "krb5", "ident", "peer",
@@ -267,6 +313,7 @@ class PgHba(object):
     PgHba object to read/write entries to/from.
     pg_hba_file - the pg_hba file almost always /etc/pg_hba
     """
+
     def __init__(self, pg_hba_file=None, order="sdu", backup=False, create=False, keep_comments_at_rules=False):
         if order not in PG_HBA_ORDERS:
             msg = "invalid order setting {0} (should be one of '{1}')."
@@ -291,6 +338,9 @@ def __init__(self, pg_hba_file=None, order="sdu", backup=False, create=False, ke
 
         self.read()
 
+    def clear_rules(self):
+        self.rules = {}
+
     def unchanged(self):
         '''
         This method resets self.diff to a empty default
@@ -722,7 +772,10 @@ def main():
                    removed_in_version='3.0.0', removed_from_collection='community.postgresql'),
         keep_comments_at_rules=dict(type='bool', default=False),
         state=dict(type='str', default="present", choices=["absent", "present"]),
-        users=dict(type='str', default='all')
+        users=dict(type='str', default='all'),
+        rules=dict(type='list', elements='dict'),
+        rules_behavior=dict(type='str', default='conflict', choices=['combine', 'conflict']),
+        overwrite=dict(type='bool', default=False),
     )
     module = AnsibleModule(
         argument_spec=argument_spec,
@@ -732,61 +785,108 @@ def main():
     if IPADDRESS_IMP_ERR is not None:
         module.fail_json(msg=missing_required_lib('ipaddress'), exception=IPADDRESS_IMP_ERR)
 
-    contype = module.params["contype"]
     create = bool(module.params["create"] or module.check_mode)
     if module.check_mode:
         backup = False
     else:
         backup = module.params['backup']
         backup_file = module.params['backup_file']
-    databases = module.params["databases"]
     dest = module.params["dest"]
-
-    method = module.params["method"]
-    netmask = module.params["netmask"]
-    options = module.params["options"]
     order = module.params["order"]
-    source = module.params["address"]
-    state = module.params["state"]
-    users = module.params["users"]
     keep_comments_at_rules = module.params["keep_comments_at_rules"]
-    comment = module.params["comment"]
+    rules = module.params["rules"]
+    rules_behavior = module.params["rules_behavior"]
+    overwrite = module.params["overwrite"]
 
     ret = {'msgs': []}
     try:
         pg_hba = PgHba(dest, order, backup=backup, create=create, keep_comments_at_rules=keep_comments_at_rules)
     except PgHbaError as error:
         module.fail_json(msg='Error reading file:\n{0}'.format(error))
 
-    if contype:
+    if overwrite:
+        pg_hba.clear_rules()
+
+    rule_keys = [
+        'address',
+        'comment',
+        'contype',
+        'databases',
+        'method',
+        'netmask',
+        'options',
+        'state',
+        'users'
+    ]
+    if rules is None:
+        single_rule = dict()
+        for key in rule_keys:
+            single_rule[key] = module.params[key]
+        rules = [single_rule]
+    else:
+        if rules_behavior == 'conflict':
+            # it's ok if the module default is set
+            used_rule_keys = [key for key in rule_keys if module.params[key] != argument_spec[key].get('default', None)]
+            if len(used_rule_keys) > 0:
+                module.fail_json(msg='conflict: either argument "rules_behavior" needs to be changed or "rules" must'
+                                     ' not be set or {0} must not be set'.format(used_rule_keys))
+
+        new_rules = []
+        for index, rule in enumerate(rules):
+            # alias handling
+            address_keys = [key for key in rule.keys() if key in ('address', 'source', 'src')]
+            if len(address_keys) > 1:
+                module.fail_json(msg='rule number {0} of the "rules" argument ({1}) uses ambiguous settings: '
+                                     '{2} are aliases, only one is allowed'.format(index, address_keys, rule))
+            if len(address_keys) == 1:
+                address = rule[address_keys[0]]
+                del rule[address_keys[0]]
+                rule['address'] = address
+
+            for key in rule_keys:
+                if key not in rule:
+                    if rules_behavior == 'combine':
+                        # use user-supplied defaults or module defaults
+                        rule[key] = module.params[key]
+                    else:
+                        # use module defaults
+                        rule[key] = argument_spec[key].get('default', None)
+            new_rules.append(rule)
+        rules = new_rules
+
+    for rule in rules:
+        if rule.get('contype', None) is None:
+            continue
+
         try:
-            for database in databases.split(','):
-                for user in users.split(','):
-                    rule = PgHbaRule(contype, database, user, source, netmask, method, options, comment=comment)
-                    if state == "present":
-                        ret['msgs'].append('Adding')
-                        pg_hba.add_rule(rule)
+            for database in rule['databases'].split(','):
+                for user in rule['users'].split(','):
+                    pg_hba_rule = PgHbaRule(rule['contype'], database, user, rule['address'], rule['netmask'],
+                                            rule['method'], rule['options'], comment=rule['comment'])
+                    if rule['state'] == "present":
+                        ret['msgs'].append('Adding rule {0}'.format(pg_hba_rule))
+                        pg_hba.add_rule(pg_hba_rule)
                     else:
-                        ret['msgs'].append('Removing')
-                        pg_hba.remove_rule(rule)
+                        ret['msgs'].append('Removing rule {0}'.format(pg_hba_rule))
+                        pg_hba.remove_rule(pg_hba_rule)
         except PgHbaError as error:
             module.fail_json(msg='Error modifying rules:\n{0}'.format(error))
-        file_args = module.load_file_common_arguments(module.params)
-        ret['changed'] = changed = pg_hba.changed()
-        if changed:
-            ret['msgs'].append('Changed')
-            ret['diff'] = pg_hba.diff
-
-            if not module.check_mode:
-                ret['msgs'].append('Writing')
-                try:
-                    if pg_hba.write(backup_file):
-                        module.set_fs_attributes_if_different(file_args, True, pg_hba.diff,
-                                                              expand=False)
-                except PgHbaError as error:
-                    module.fail_json(msg='Error writing file:\n{0}'.format(error))
-                if pg_hba.last_backup:
-                    ret['backup_file'] = pg_hba.last_backup
+    file_args = module.load_file_common_arguments(module.params)
+    ret['changed'] = changed = pg_hba.changed()
+    if changed:
+        ret['msgs'].append('Changed')
+        ret['diff'] = pg_hba.diff
+
+        if not module.check_mode:
+            ret['msgs'].append('Writing')
+            try:
+                if pg_hba.write(backup_file):
+                    module.set_fs_attributes_if_different(file_args, True, pg_hba.diff,
+                                                          expand=False)
+            except PgHbaError as error:
+                module.fail_json(msg='Error writing file:\n{0}'.format(error))
+            if pg_hba.last_backup:
+                ret['backup_file'] = pg_hba.last_backup
 
     ret['pg_hba'] = list(pg_hba.get_rules())
     module.exit_json(**ret)

tests/integration/targets/postgresql_pg_hba/tasks/main.yml

@@ -5,3 +5,4 @@
 
 # Initial CI tests of postgresql_pg_hba module
 - import_tasks: postgresql_pg_hba_initial.yml
+- import_tasks: postgresql_pg_hba_bulk_rules.yml

tests/integration/targets/postgresql_pg_hba/tasks/postgresql_pg_hba_bulk_rules.yml

@@ -0,0 +1,116 @@
+- name: set test variables
+  set_fact:
+    pghba_defaults: &pghba_defaults
+      create: yes
+      dest: "/tmp/pg_hba_bulk_test.conf"
+    test_rule0: &test_rule0
+      contype: host
+      databases: "db0"
+      users: "user0"
+      address: "2001:db8::0/128"
+      method: pam
+    test_rule1: &test_rule1
+      contype: host
+      databases: "db1"
+      users: "user1"
+      address: "2001:db8::1/128"
+      method: pam
+    test_rule2: &test_rule2
+      contype: host
+      databases: "db2"
+      users: "user2"
+      address: "2001:db8::2/128"
+      method: pam
+
+- name: create one rule to clear
+  community.postgresql.postgresql_pg_hba:
+    <<: *pghba_defaults
+    state: "present"
+    <<: *test_rule0
+- name: overwrite with one normal rule
+  community.postgresql.postgresql_pg_hba:
+    <<: *pghba_defaults
+    overwrite: true
+    <<: *test_rule1
+  register: result
+- assert:
+    that:
+      - "result.pg_hba|length == 1"
+      - "result.pg_hba[0].db == test_rule1.databases"
+      - "result.pg_hba[0].src == test_rule1.address"
+      - "result.pg_hba[0].usr == test_rule1.users"
+      - "result.pg_hba[0].type == test_rule1.contype"
+- name: overwrite with one bulk rule
+  community.postgresql.postgresql_pg_hba:
+    <<: *pghba_defaults
+    overwrite: true
+    rules:
+      - "{{ test_rule2 }}"
+  register: result
+- assert:
+    that:
+      - "result.pg_hba|length == 1"
+      - "result.pg_hba[0].db == test_rule2.databases"
+      - "result.pg_hba[0].src == test_rule2.address"
+      - "result.pg_hba[0].usr == test_rule2.users"
+      - "result.pg_hba[0].type == test_rule2.contype"
+
+- name: test rules_behavior conflict
+  community.postgresql.postgresql_pg_hba: "{{ pghba_defaults|combine(item)|combine({'rules': [test_rule2]}) }}"
+  loop:
+    - address: 2001:db8::a/128
+    - comment: 'testcomment'
+    - contype: hostssl
+    - databases: db_a
+    - method: cert
+    - netmask: 255.255.255.0
+      # address: 192.0.2.0
+    - options: "clientcert=verify-full"
+    - state: absent
+    - users: testuser
+  register: result
+  ignore_errors: true
+- name: get jinja2 version
+  shell: '/usr/bin/pip --disable-pip-version-check --no-cache-dir show Jinja2 2>/dev/null | grep -oPm 1 "(?<=^Version: )\d+\.\d+"'
+  register: jinja2_version
+  ignore_errors: true
+- assert:
+    that:
+      - result.failed
+      - not result.changed
+      - "result.results|selectattr('changed')|length == 0"
+      - "result.results|rejectattr('failed')|length == 0"
+      # the 'in' test was added in jinja 2.10
+      - "jinja2_version.rc == 0 and jinja2_version.stdout|trim is version('2.10', '<') or result.results|selectattr('msg', 'in', 'conflict')|length == 0"
+
+- name: test rules with module defaults
+  community.postgresql.postgresql_pg_hba:
+    <<: *pghba_defaults
+    rules:
+      - contype: hostssl
+  register: result
+- assert:
+    that:
+      - result.changed
+      # assert that module defaults are used
+      - "{'db': 'all', 'method': 'md5', 'src': 'samehost', 'type': 'hostssl', 'usr': 'all'} in result.pg_hba"
+
+- name: test rules with custom defaults
+  community.postgresql.postgresql_pg_hba:
+    <<: *pghba_defaults
+    rules_behavior: combine
+    <<: *test_rule1
+    rules:
+      - {} # complete fallback to custom defaults
+      - databases: other_db # partial fallback to custom defaults
+      # no fallback
+      - <<: *test_rule2
+        state: absent
+  register: result
+- assert:
+    that:
+      - result.changed
+      - "{'db': 'all', 'method': 'md5', 'src': 'samehost', 'type': 'hostssl', 'usr': 'all'} in result.pg_hba" # unchanged preexisting from previous task
+      - "{'db': test_rule1.databases, 'method': test_rule1.method, 'src': test_rule1.address, 'type': test_rule1.contype, 'usr': test_rule1.users} in result.pg_hba"
+      - "{'db': test_rule2.databases, 'method': test_rule2.method, 'src': test_rule2.address, 'type': test_rule2.contype, 'usr': test_rule2.users} not in result.pg_hba"
+      - "{'db': 'other_db', 'method': test_rule1.method, 'src': test_rule1.address, 'type': test_rule1.contype, 'usr': test_rule1.users} in result.pg_hba"

tests/integration/targets/postgresql_pg_hba/tasks/postgresql_pg_hba_initial.yml

@@ -182,6 +182,11 @@
     - 'not netmask_sameas_prefix_check is changed'
     - 'pg_hba_options is changed'
 
+- name: ensure test file is empty
+  copy:
+    content: ''
+    dest: /tmp/pg_hba2.conf
+
 - name: Create a rule with the comment 'comment1'
   postgresql_pg_hba:
     contype: host