testing for nftables collection

defaults/main.yml

@@ -54,24 +54,26 @@ tester:
   packages:
     apt: [
       'python3', 'python3-virtualenv', 'python3-pip',
-      'libssl-dev', 'git', 'rsync',
+      'libssl-dev', 'git', 'rsync', 'jq',
     ]
     pip: [
       'setuptools',
       'ansible', 'molecule', 'ansible-core', 'molecule-docker',
       'anybadge', 'jmespath',
       'yamllint', 'ansible-lint', 'pylint', 'flake8',
       # collections
-      'httpx', 'xmltodict',
+      'httpx', 'xmltodict', 'passlib',
     ]
     ansible_collections: ['community.general', 'ansible.utils']
 
   collections:
     opnsense:
-      label: 'OPNSense v23.7.8'  # todo: dynamic lookup of version number
+      label_prefix: 'OPNSense'
+      label_script: "source /home/tester/.secret/opn.key && curl --insecure --silent -d \"\" -k -u \"$key\":\"$secret\" https://opn-test.local/api/core/firmware/info | jq \".product_version\" | tr -d \\\\\""
       ip: '172.17.1.52'
       dns: 'opn-test.local'
       test_args: ['opn-test.local', '/home/tester/.secret/opn.key', '0']
+
     linuxha:
       label: 'SH-4.2.1 | CS-3.1.2 | PM-2.0.5'
       # debian 11; crmsh: 4.2.1; corosync: 3.1.2-2; pacemaker: 2.0.5-2
@@ -83,6 +85,11 @@ tester:
         dns: 'node2.local'
       test_args: ['node1.local', 'node2.local', '/home/tester/.secret/lha.yml', '0']
 
+    nftables:
+      label_prefix: 'NFTables'
+      label_script: "ssh 172.17.1.56 \"apt policy nftables 2>/dev/null | grep Installed | cut -d \\\\\" \\\\\" -f 4 | cut -d \\\\\"-\\\\\" -f1\" 2>/dev/null"
+      test_args: ['tester', '/home/tester/.secret/nftables.pwd', '22', '172.17.1.56', '172.17.1.57']
+
   log:
     lines: 300
 

files/home/tester/.secret/nftables.pwd

@@ -0,0 +1,6 @@
+$ANSIBLE_VAULT;1.1;AES256
+32323233616163376238643734613661323461613235306631336563336239663562356362656237
+3561653566356462666535356536376432373235303261610a343934623435353633353062323639
+31633238333638316264353134333761356135643035346231653930626439623462386237333262
+3837633961353537620a616530616666303137313635613431366366393931353861613866316430
+3963

tasks/jobs_collection.yml

@@ -45,4 +45,4 @@
     creates: "{{ tester.path.web }}/{{ item.file }}.svg"
   loop:
     - {label: 'LINT', file: "{{ collection_name }}.collection.lint"}
-    - {label: "TEST {{ job.value.label }}", file: "{{ collection_name }}.collection.test"}
+    - {label: "TEST {{ job.value.label | default(job.value.label_prefix + ' ?') }}", file: "{{ collection_name }}.collection.test"}

tasks/main.yml

@@ -142,6 +142,7 @@
     - '.ssh/id_ed25519'
     - '.secret/opn.key'
     - '.secret/lha.yml'
+    - '.secret/nftables.pwd'
   tags: collections
 
 - name: CICD | Ansible Tester | Adding docker-cleanup job

templates/etc/systemd/system/ansible-test-collection.service.j2

@@ -10,7 +10,12 @@ User={{ tester.user }}
 Group={{ tester.user }}
 ExecStartPre=mkdir -p {{ tester.path.test }}
 ExecStartPre=rm -rf {{ tester.path.test }}/collection_{{ collection_name }}
-ExecStart=/bin/bash {{ tester.path.script }}/collection_test.sh '{{ collection_name }}' '{{ job.value.label }}'{% for arg in job.value.test_args %} '{{ arg }}'{% endfor %}
+{% if 'label_script' in job.value %}
+ExecStartPre=/bin/bash -c '{{ job.value.label_script }} > /tmp/collection_{{ collection_name }}.label && sleep 1'
+{% else %}
+ExecStartPre=/bin/echo -c "echo '{{ job.value.label }}' > /tmp/collection_{{ collection_name }}.label"
+{% endif %}
+ExecStart=/bin/bash -c '{{ tester.path.script }}/collection_test.sh "{{ collection_name }}" "{{ job.value.label_prefix + ' ' | default('') }}$(cat /tmp/collection_{{ collection_name }}.label)"{% for arg in job.value.test_args %} "{{ arg }}"{% endfor %}'
 
 {% if ansible_distribution_version == '10' %}
 StandardOutput=syslog

templates/usr/local/bin/cicd/ansiblelint.sh.j2

@@ -35,24 +35,24 @@ function update_badge {
   cd "$PATH_BADGE"
   if [[ "$collection" == "1" ]]
   then
-    KEY_BADGE="collection_$ROLE"
+    KEY_BADGE="collection_${ROLE}"
   else
     KEY_BADGE="$ROLE"
   fi
-  rm -f "$KEY_BADGE.{{ test }}.svg"
-  anybadge --label="$LABEL" --value="$status | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="$KEY_BADGE.{{ test }}" --color="${BADGE_COLORS[$status]}"
+  rm -f "${KEY_BADGE}.{{ test }}.svg"
+  anybadge --label="$LABEL" --value="${status} | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="${KEY_BADGE}.{{ test }}" --color="${BADGE_COLORS[$status]}"
 }
 
-source "$PATH_VENV/bin/activate"
+source "${PATH_VENV}/bin/activate"
 collection=0
 
 if echo "$ROLE" | grep -q 'collection'
 then
   ROLE=$(echo "$ROLE" | cut -d '_' -f2)
   collection=1
-  PATH_TEST_FULL="$PATH_TEST/collections/ansible_collections/$GALAXY_NAMESPACE/$ROLE"
+  PATH_TEST_FULL="${PATH_TEST}/collections/ansible_collections/${GALAXY_NAMESPACE}/${ROLE}"
 else
-  PATH_TEST_FULL="$PATH_TEST/$GALAXY_NAMESPACE.$ROLE"
+  PATH_TEST_FULL="${PATH_TEST}/${GALAXY_NAMESPACE}.${ROLE}"
 fi
 
 if [ -n "$2" ]
@@ -70,9 +70,9 @@ rm -rf "$PATH_TEST_FULL"
 
 if [[ "$collection" == "1" ]]
 then
-  ansible-galaxy collection install "git+https://github.com/$GALAXY_NAMESPACE/collection_$ROLE" -p "$PATH_TEST/collections"
+  ansible-galaxy collection install "git+https://github.com/${GALAXY_NAMESPACE}/collection_${ROLE}" -p "${PATH_TEST}/collections"
 else
-  ansible-galaxy install --roles-path "$PATH_TEST" "$GALAXY_NAMESPACE.$ROLE"
+  ansible-galaxy install --roles-path "$PATH_TEST" "${GALAXY_NAMESPACE}.${ROLE}"
 fi
 
 cd "$PATH_TEST_FULL"

templates/usr/local/bin/cicd/collection_lint.sh.j2

@@ -32,10 +32,10 @@ BADGE_COLORS[FAILED-ENVIRONMENT]='#{{ tester.colors.bad }}'
 function update_badge {
   cd "$PATH_BADGE"
   rm -f "$COLLECTION.collection.lint.svg"
-  anybadge --label="$LABEL" --value="$status | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="$COLLECTION.collection.lint" --color="${BADGE_COLORS[$status]}"
+  anybadge --label="$LABEL" --value="${status} | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="$COLLECTION.collection.lint" --color="${BADGE_COLORS[$status]}"
 }
 
-source "$PATH_VENV/bin/activate"
+source "${PATH_VENV}/bin/activate"
 
 update_badge
 
@@ -57,5 +57,5 @@ fi
 
 update_badge
 cd /tmp
-rm -rf "$PATH_TEST/collection_$COLLECTION"
+rm -rf "${PATH_TEST}/collection_$COLLECTION"
 exit "$exit_code"

templates/usr/local/bin/cicd/collection_test.sh.j2

@@ -15,7 +15,7 @@ fi
 
 PATH_TEST='{{ tester.path.test }}'
 PATH_BADGE='{{ tester.path.web }}'
-PATH_LOG="$PATH_BADGE/log"
+PATH_LOG="${PATH_BADGE}/log"
 PATH_VENV='{{ tester.path.venv }}'
 HOME='{{ tester.path.home }}'
 LOG_LINES='{{ tester.log.lines }}'
@@ -25,6 +25,7 @@ LABEL="TEST $2"
 TEST_ARGS="${@:3}"
 status='RUNNING'
 LOG_FILE="${PATH_LOG}/collection_${COLLECTION}_test.log"
+LOG_FILE2="${PATH_LOG}/collection_${COLLECTION}_test_short.log"
 EXIT_CODE_FAILED='{{ tester.rc.soft_fail }}'
 exit_code=0
 
@@ -34,53 +35,61 @@ BADGE_COLORS[RUNNING]='#{{ tester.colors.run }}'
 BADGE_COLORS[PASSED]='#{{ tester.colors.ok }}'
 BADGE_COLORS[FAILED]='#{{ tester.colors.bad }}'
 BADGE_COLORS[FAILED-ENVIRONMENT]='#{{ tester.colors.bad }}'
+BADGE_COLORS[FAILED-CLEANUP]='#{{ tester.colors.bad }}'
 
 function update_badge {
   cd "$PATH_BADGE"
-  rm -f "$COLLECTION.collection.test.svg"
-  anybadge --label="$LABEL" --value="$status | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="$COLLECTION.collection.test" --color="${BADGE_COLORS[$status]}"
+  rm -f "${COLLECTION}.collection.test.svg"
+  anybadge --label="$LABEL" --value="${status} | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="${COLLECTION}.collection.test" --color="${BADGE_COLORS[$status]}"
 }
 
-source "$PATH_VENV/bin/activate"
+function update_logs {
+  journalctl -u "ansible-test-collection-${COLLECTION}.service" --full --no-pager -n "$LOG_LINES" > "$LOG_FILE"
+  journalctl -u "ansible-test-collection-${COLLECTION}.service" -n 10000 --no-pager --full | grep -E 'Starting Service|Finished Service|FAILED|exception|Traceback|RUNNING TESTS' > "$LOG_FILE2"
+}
+
+function set_error {
+  status="$1"
+  echo "$status"
+  exit_code="$EXIT_CODE_FAILED"
+}
+
+source "${PATH_VENV}/bin/activate"
 
 update_badge
-rm -f "$LOG_FILE"
 
 # preparations for tests
 cd "$PATH_TEST"
-git clone "https://github.com/ansibleguy/collection_$COLLECTION.git"
-cd "collection_$COLLECTION"
+git clone "https://github.com/ansibleguy/collection_${COLLECTION}.git"
+cd "collection_${COLLECTION}"
 
 set +e  # soft-handling of failures
 chmod +x ./scripts/test.sh
 chmod +x ./scripts/test_cleanup.sh
 
 if ! ./scripts/test_cleanup.sh $TEST_ARGS "$PATH_VENV"
 then
-  echo 'PRE-CLEANUP FAILED'
-  exit_code="$EXIT_CODE_FAILED"
+  set_error 'FAILED-CLEANUP'
 else
   echo 'PRE-CLEANUP SUCCESSFUL'
 fi
 
 if ! ./scripts/test.sh $TEST_ARGS "$PATH_VENV"
 then
-  status='FAILED'
-  journalctl -u "ansible-test-collection-$COLLECTION.service" --full --no-pager -n "$LOG_LINES" > "$LOG_FILE"
-  exit_code="$EXIT_CODE_FAILED"
+  set_error 'FAILED'
 else
   status='PASSED'
 fi
 
 if ! ./scripts/test_cleanup.sh $TEST_ARGS "$PATH_VENV"
 then
-  echo 'POST-CLEANUP FAILED'
-  exit_code="$EXIT_CODE_FAILED"
+  set_error 'FAILED-CLEANUP'
 else
   echo 'POST-CLEANUP SUCCESSFUL'
 fi
 
+update_logs
 update_badge
 cd /tmp
-rm -rf "$PATH_TEST/collection_$COLLECTION"
+rm -rf "${PATH_TEST}/collection_${COLLECTION}"
 exit "$exit_code"

templates/usr/local/bin/cicd/molecule.sh.j2

@@ -39,17 +39,23 @@ BADGE_COLORS[FAILED-ENVIRONMENT]='#{{ tester.colors.bad }}'
 
 ROLE="$1"
 LOG_FILE="${PATH_LOG}/molecule_${ROLE}_test.log"
+LOG_FILE2="${PATH_LOG}/molecule_${ROLE}_test_short.log"
 
 function update_badge {
   cd "$PATH_BADGE"
   if [[ "$collection" == "1" ]]
   then
-    KEY_BADGE="collection_$ROLE"
+    KEY_BADGE="collection_${ROLE}"
   else
     KEY_BADGE="$ROLE"
   fi
-  rm -f "$KEY_BADGE.{{ test }}.svg"
-  anybadge --label="$LABEL" --value="$status | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="$KEY_BADGE.{{ test }}" --color="${BADGE_COLORS[$status]}"
+  rm -f "${KEY_BADGE}.{{ test }}.svg"
+  anybadge --label="$LABEL" --value="${status} | $(date '+%Y-%m-%d %H:%M') GMT+2" --file="${KEY_BADGE}.{{ test }}" --color="${BADGE_COLORS[$status]}"
+}
+
+function update_logs {
+  journalctl -u "ansible-test-molecule-${ROLE}.service" --full --no-pager -n "$LOG_LINES" > "$LOG_FILE"
+  journalctl -u "ansible-test-molecule-${ROLE}.service" -n 10000 --full --no-pager | grep -E 'Starting Service|Finished Service|completed successful|FAILED' | grep -v 'Wait' > "$LOG_FILE2"
 }
 
 function cleanup {
@@ -63,23 +69,22 @@ function cleanup {
     cd /tmp
     rm -rf "$PATH_TEST_FULL"
   fi
-  rm -rf "$HOME/.cache/molecule/$ROLE"
+  rm -rf "$HOME/.cache/molecule/${ROLE}"
   if [ -d "$HOME/.cache/ansible-compat" ]
   then
     find "$HOME/.cache/ansible-compat" -maxdepth 0 -type d -mmin +30 -exec rm -rf {} \;
   fi
 }
 
-function log_error {
-  journalctl -u "ansible-test-molecule-$ROLE.service" --full --no-pager -n "$LOG_LINES" > "$LOG_FILE"
+function set_error {
+  status="$1"
   exit_code="$EXIT_CODE_FAILED"
 }
 
 mkdir -p "$PATH_TEST"
-source "$PATH_VENV/bin/activate"
+source "${PATH_VENV}/bin/activate"
 collection=0
 cleanup
-rm -f "$LOG_FILE"
 
 if [ -n "$2" ]
 then
@@ -96,13 +101,13 @@ if echo "$ROLE" | grep -q 'collection'
 then
   ROLE=$(echo "$ROLE" | cut -d '_' -f2)
   collection=1
-  PATH_TEST_FULL="$PATH_TEST/collections/ansible_collections/$GALAXY_NAMESPACE/$ROLE"
+  PATH_TEST_FULL="${PATH_TEST}/collections/ansible_collections/${GALAXY_NAMESPACE}/${ROLE}"
   rm -rf "$PATH_TEST_FULL"
-  ansible-galaxy collection install "git+https://github.com/$GALAXY_NAMESPACE/collection_$ROLE" -p "$PATH_TEST/collections"
+  ansible-galaxy collection install "git+https://github.com/${GALAXY_NAMESPACE}/collection_${ROLE}" -p "${PATH_TEST}/collections"
 else
-  PATH_TEST_FULL="$PATH_TEST/$GALAXY_NAMESPACE.$ROLE"
+  PATH_TEST_FULL="${PATH_TEST}/${GALAXY_NAMESPACE}.${ROLE}"
   rm -rf "$PATH_TEST_FULL"
-  ansible-galaxy install --roles-path "$PATH_TEST" "$GALAXY_NAMESPACE.$ROLE"
+  ansible-galaxy install --roles-path "$PATH_TEST" "${GALAXY_NAMESPACE}.${ROLE}"
 fi
 
 cd "$PATH_TEST_FULL"
@@ -122,31 +127,28 @@ molecule prepare
 set +e  # soft-handling of failures
 if ! molecule converge
 then
-  status='FAILED-CONVERGE'
-  log_error
+  set_error 'FAILED-CONVERGE'
 else
   if ! molecule verify
   then
-    status='FAILED-VERIFY'
-    log_error
+    set_error 'FAILED-VERIFY'
   else
     if ! molecule idempotence
     then
-      status='FAILED-IDEMPOTENCE'
-      log_error
+      set_error 'FAILED-IDEMPOTENCE'
     else
       # check-mode can be disabled
       if cat "$MOLECULE_CONFIG" | grep '  - check' -q && ! molecule check
       then
-        status='FAILED-CHECK'
-        log_error
+        set_error 'FAILED-CHECK'
       else
         status='PASSED'
       fi
     fi
   fi
 fi
 
+update_logs
 update_badge
 cleanup
 exit "$exit_code"