Add stability test for private pip install (#1521)

.github/workflows/tests.yml

@@ -133,6 +133,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
           AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
+          PYTHON_STUB_PAT: ${{ secrets.PYTHON_STUB_PAT }}
           SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
           SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
           SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}

tests/stability/test_install_plugins.py

@@ -0,0 +1,29 @@
+import os
+
+import pytest
+from distributed import PipInstall
+
+from ..utils_test import wait
+
+
+@pytest.mark.parametrize("restart_workers", [True, False])
+def test_private_pip_install(small_client, restart_workers):
+    small_client.cluster.send_private_envs(
+        {"PYTHON_STUB_TOKEN": os.environ["PYTHON_STUB_PAT"]}
+    )
+
+    plugin = PipInstall(
+        packages=[
+            "python_stub@git+https://${PYTHON_STUB_TOKEN}@github.com/coiled/python-stub.git"
+        ],
+        restart_workers=restart_workers,
+    )
+    small_client.register_plugin(plugin)
+
+    def test(x):
+        from python_stub import stub
+
+        return stub.echo(x)
+
+    fut = small_client.submit(test, "Hello, world!")
+    wait(fut, small_client, 5 * 60)