Merge pull request #851 from Shrews/backport-integration-test

[release_2.0] Backport integration test

Backports of:

PR #790 - Add custom integration python job
PR #796 - Migrate container jobs to centos-8-stream nodeset
PR #797 - Tune tox.ini
PR #798 - Rework bindep.txt files
PR #801 - Small change to properly test containers

Reviewed-by: Shane McDonald <me@shanemcd.com>
Reviewed-by: None <None>

.zuul.d/jobs.yaml

@@ -9,6 +9,7 @@
     timeout: 5400
     vars:
       zuul_work_dir: "{{ zuul.projects['github.com/ansible/ansible-runner'].src_dir }}"
+    nodeset: centos-8-stream
 
 # =============================================================================
 
@@ -192,3 +193,18 @@
 - job:
     name: ansible-runner-upload-container-image-stable-2.9
     parent: ansible-runner-container-image-base
+
+# =============================================================================
+- job:
+    name: ansible-runner-integration
+    parent: ansible-buildset-registry-consumer
+    requires:
+      - ansible-runner-container-image
+    dependencies:
+      - ansible-runner-build-container-image
+    vars:
+      container_command: docker
+
+- job:
+    name: ansible-runner-integration
+    parent: ansible-tox-py38

.zuul.d/project.yaml

@@ -2,18 +2,20 @@
 - project:
     check:
       jobs: &id001
+        - ansible-buildset-registry
         - ansible-runner-build-container-image
         - ansible-runner-build-container-image-stable-2.9
         - ansible-runner-build-container-image-stable-2.10
         - ansible-runner-build-container-image-stable-2.11
-        - ansible-tox-py38
+        - ansible-runner-integration
         - ansible-tox-docs:
             vars:
               sphinx_build_dir: docs/build
     gate:
       jobs: *id001
     post:
       jobs: &id002
+        - ansible-buildset-registry
         - ansible-runner-upload-container-image:
             vars:
               upload_container_image_promote: false

Dockerfile

@@ -17,7 +17,8 @@ RUN if [ "$ANSIBLE_BRANCH" != "" ] ; then \
       echo "Installing requirements.txt" ; \
       cp /tmp/src/tools/requirements.txt /tmp/src/requirements.txt ; \
     fi \
-    && cp /tmp/src/tools/build-requirements.txt /tmp/src/build-requirements.txt
+    && cp /tmp/src/tools/build-requirements.txt /tmp/src/build-requirements.txt \
+    && cp /tmp/src/tools/bindep.txt /tmp/src/bindep.txt
 
 # NOTE(pabelanger): For downstream builds, we compile everything from source
 # over using existing wheels. Do this upstream too so we can better catch

bindep.txt

@@ -1,29 +1,2 @@
 # This is a cross-platform list tracking distribution packages needed by tests;
 # see https://docs.openstack.org/infra/bindep/ for additional information.
-git
-openssh-clients [!platform:ubuntu-bionic]
-openssh-client [platform:ubuntu-bionic]
-rsync
-sshpass [epel]
-
-# ansible
-python38-cffi [platform:centos-8]
-python38-cryptography [platform:centos-8]
-python38-jinja2 [platform:centos-8]
-python38-pycparser [platform:centos-8]
-python38-six [platform:centos-8]
-python38-yaml [platform:centos-8]
-
-# ncclient
-python38-lxml [platform:centos-8 platform:rhel-8]
-
-# paramiko
-findutils [compile platform:centos-8 platform:rhel-8]
-gcc [compile platform:centos-8 platform:rhel-8]
-make [compile platform:centos-8 platform:rhel-8]
-python38-devel [compile platform:centos-8 platform:rhel-8]
-
-# pypsrp
-krb5-devel [compile platform:centos-8 platform:rhel-8]
-krb5-workstation [platform:centos-8 platform:rhel-8]
-python38-requests [platform:centos-8 platform:rhel-8]

test/integration/containerized/test_cli_containerized.py

@@ -29,45 +29,3 @@ def test_playbook_run(cli, skip_if_no_podman):
 def test_provide_env_var(cli, skip_if_no_podman, test_data_dir):
     r = cli(['run', os.path.join(test_data_dir, 'job_env'), '-p', 'printenv.yml'])
     assert 'gifmyvqok2' in r.stdout, r.stdout
-
-
-@pytest.mark.serial
-def test_adhoc_localhost_setup(cli, skip_if_no_podman, container_runtime_installed):
-    r = cli(
-        [
-            'adhoc',
-            '--private-data-dir', os.path.join(HERE, 'priv_data'),
-            '--container-runtime', container_runtime_installed,
-            'localhost', '-m', 'setup'
-        ]
-    )
-    # TODO: look for some fact that indicates we are in container?
-    assert '"ansible_facts": {' in r.stdout
-
-
-@pytest.mark.serial
-def test_playbook_with_private_data_dir(cli, skip_if_no_podman, container_runtime_installed):
-    # tests using a private_data_dir in conjunction with an absolute path
-    r = cli(
-        [
-            'playbook',
-            '--private-data-dir', os.path.join(HERE, 'priv_data'),
-            '--container-runtime', container_runtime_installed,
-            os.path.join(HERE, 'priv_data/project/test-container.yml')
-        ]
-    )
-    assert 'PLAY RECAP *******' in r.stdout
-    assert 'failed=0' in r.stdout
-
-
-@pytest.mark.serial
-def test_playbook_with_relative_path(cli, skip_if_no_podman, container_runtime_installed):
-    r = cli(
-        [
-            'playbook',
-            '--container-runtime', container_runtime_installed,
-            'test/integration/containerized/priv_data/project/test-container.yml'
-        ]
-    )
-    assert 'PLAY RECAP *******' in r.stdout
-    assert 'failed=0' in r.stdout

test/integration/test_main.py

@@ -18,6 +18,16 @@
 HERE = os.path.abspath(os.path.dirname(__file__))
 
 
+def iterate_timeout(max_seconds, purpose, interval=2):
+    start = time.time()
+    count = 0
+    while (time.time() < start + max_seconds):
+        count += 1
+        yield count
+        time.sleep(interval)
+    raise Exception("Timeout waiting for %s" % purpose)
+
+
 def ensure_directory(directory):
     if not os.path.exists(directory):
         os.makedirs(directory)
@@ -329,21 +339,22 @@ def test_playbook_start(skipif_pre_ansible28):
         )
         p.start()
 
-        time.sleep(5)
-
-        assert os.path.exists(os.path.join(temp_dir, 'pid'))
+        pid_path = os.path.join(temp_dir, 'pid')
+        for _ in iterate_timeout(30, "pid file creation"):
+            if os.path.exists(pid_path):
+                break
 
         rc = main(['is-alive', temp_dir])
         assert rc == 0
         rc = main(['stop', temp_dir])
         assert rc == 0
 
-        time.sleep(1)
+        for _ in iterate_timeout(30, "background process to stop"):
+            rc = main(['is-alive', temp_dir])
+            if rc == 1:
+                break
 
-        rc = main(['is-alive', temp_dir])
-        assert rc == 1
-
-        ensure_removed(os.path.join(temp_dir, 'pid'))
+        ensure_removed(pid_path)
 
         rc = main(['stop', temp_dir])
-    assert rc == 1
+        assert rc == 1

tools/bindep.txt

@@ -0,0 +1,29 @@
+# This is a cross-platform list tracking distribution packages needed by tests;
+# see https://docs.openstack.org/infra/bindep/ for additional information.
+git
+openssh-clients [!platform:ubuntu-bionic]
+openssh-client [platform:ubuntu-bionic]
+rsync
+sshpass [epel]
+
+# ansible
+python38-cffi [platform:centos-8]
+python38-cryptography [platform:centos-8]
+python38-jinja2 [platform:centos-8]
+python38-pycparser [platform:centos-8]
+python38-six [platform:centos-8]
+python38-yaml [platform:centos-8]
+
+# ncclient
+python38-lxml [platform:centos-8 platform:rhel-8]
+
+# paramiko
+findutils [compile platform:centos-8 platform:rhel-8]
+gcc [compile platform:centos-8 platform:rhel-8]
+make [compile platform:centos-8 platform:rhel-8]
+python38-devel [compile platform:centos-8 platform:rhel-8]
+
+# pypsrp
+krb5-devel [compile platform:centos-8 platform:rhel-8]
+krb5-workstation [platform:centos-8 platform:rhel-8]
+python38-requests [platform:centos-8 platform:rhel-8]

tox.ini

@@ -1,5 +1,6 @@
 [tox]
 envlist = linters, ansible{27, 28, 29, -base}
+skipsdist = True
 
 [testenv]
 deps = ansible27: ansible<2.8
@@ -10,6 +11,7 @@ deps = ansible27: ansible<2.8
        -rrequirements.txt
        -rtest/requirements.txt
 passenv = HOME
+usedevelop = True
 commands=
     py.test -v -n 4 -m "not serial" {posargs:test}
     py.test -v -m serial {posargs:test}