diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml
index 53ae10637..ab09dddfe 100644
--- a/.zuul.d/jobs.yaml
+++ b/.zuul.d/jobs.yaml
@@ -194,3 +194,17 @@
 - job:
     name: ansible-runner-upload-container-image-stable-2.9
     parent: ansible-runner-container-image-base
+
+- job:
+    name: ansible-runner-integration
+    parent: ansible-buildset-registry-consumer
+    requires:
+      - ansible-runner-container-image
+    dependencies:
+      - ansible-buildset-registry
+    vars:
+      container_command: docker
+
+- job:
+    name: ansible-runner-integration
+    parent: ansible-tox-py38
diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml
index 0665774a1..e9188a976 100644
--- a/.zuul.d/project.yaml
+++ b/.zuul.d/project.yaml
@@ -7,7 +7,7 @@
         - ansible-runner-build-container-image-stable-2.9
         - ansible-runner-build-container-image-stable-2.10
         - ansible-runner-build-container-image-stable-2.11
-        - ansible-tox-py38
+        - ansible-runner-integration
         - ansible-tox-docs:
             vars:
               sphinx_build_dir: docs/build
diff --git a/test/integration/containerized/test_cli_containerized.py b/test/integration/containerized/test_cli_containerized.py
index 56b159931..fc0a55df8 100644
--- a/test/integration/containerized/test_cli_containerized.py
+++ b/test/integration/containerized/test_cli_containerized.py
@@ -29,45 +29,3 @@ def test_playbook_run(cli, skip_if_no_podman):
 def test_provide_env_var(cli, skip_if_no_podman, test_data_dir):
     r = cli(['run', os.path.join(test_data_dir, 'job_env'), '-p', 'printenv.yml'])
     assert 'gifmyvqok2' in r.stdout, r.stdout
-
-
-@pytest.mark.serial
-def test_adhoc_localhost_setup(cli, skip_if_no_podman, container_runtime_installed):
-    r = cli(
-        [
-            'adhoc',
-            '--private-data-dir', os.path.join(HERE, 'priv_data'),
-            '--container-runtime', container_runtime_installed,
-            'localhost', '-m', 'setup'
-        ]
-    )
-    # TODO: look for some fact that indicates we are in container?
-    assert '"ansible_facts": {' in r.stdout
-
-
-@pytest.mark.serial
-def test_playbook_with_private_data_dir(cli, skip_if_no_podman, container_runtime_installed):
-    # tests using a private_data_dir in conjunction with an absolute path
-    r = cli(
-        [
-            'playbook',
-            '--private-data-dir', os.path.join(HERE, 'priv_data'),
-            '--container-runtime', container_runtime_installed,
-            os.path.join(HERE, 'priv_data/project/test-container.yml')
-        ]
-    )
-    assert 'PLAY RECAP *******' in r.stdout
-    assert 'failed=0' in r.stdout
-
-
-@pytest.mark.serial
-def test_playbook_with_relative_path(cli, skip_if_no_podman, container_runtime_installed):
-    r = cli(
-        [
-            'playbook',
-            '--container-runtime', container_runtime_installed,
-            'test/integration/containerized/priv_data/project/test-container.yml'
-        ]
-    )
-    assert 'PLAY RECAP *******' in r.stdout
-    assert 'failed=0' in r.stdout
diff --git a/test/integration/test_main.py b/test/integration/test_main.py
index 34cd96439..0ad33ef1b 100644
--- a/test/integration/test_main.py
+++ b/test/integration/test_main.py
@@ -18,6 +18,16 @@
 HERE = os.path.abspath(os.path.dirname(__file__))
 
 
+def iterate_timeout(max_seconds, purpose, interval=2):
+    start = time.time()
+    count = 0
+    while (time.time() < start + max_seconds):
+        count += 1
+        yield count
+        time.sleep(interval)
+    raise Exception("Timeout waiting for %s" % purpose)
+
+
 def ensure_directory(directory):
     if not os.path.exists(directory):
         os.makedirs(directory)
@@ -304,21 +314,22 @@ def test_playbook_start(skipif_pre_ansible28):
                                            temp_dir]])
         p.start()
 
-        time.sleep(5)
-
-        assert os.path.exists(os.path.join(temp_dir, 'pid'))
+        pid_path = os.path.join(temp_dir, 'pid')
+        for _ in iterate_timeout(30, "pid file creation"):
+            if os.path.exists(pid_path):
+                break
 
         rc = main(['is-alive', temp_dir])
         assert rc == 0
         rc = main(['stop', temp_dir])
         assert rc == 0
 
-        time.sleep(1)
+        for _ in iterate_timeout(30, "background process to stop"):
+            rc = main(['is-alive', temp_dir])
+            if rc == 1:
+                break
 
-        rc = main(['is-alive', temp_dir])
-        assert rc == 1
-
-        ensure_removed(os.path.join(temp_dir, 'pid'))
+        ensure_removed(pid_path)
 
         rc = main(['stop', temp_dir])
-    assert rc == 1
+        assert rc == 1