TLS integration test (#37)

* Import files

* Add TLS implementation

* Add integration test

* Update tests/unit/test_postgresql_tls.py

Co-authored-by: Will Fitch <WRFitch@outlook.com>

* Add required functions and variable for regex

* Improve TLS check

* Update library

* Update library

* Fix PostgreSQL library

* Add relation broken test

* Fix comment

Co-authored-by: Will Fitch <WRFitch@outlook.com>

.github/workflows/ci.yaml

@@ -99,3 +99,18 @@ jobs:
           bootstrap-options: "--agent-version 2.9.29"
       - name: Run integration tests
         run: tox -e password-rotation-integration
+
+  integration-test-microk8s-tls:
+    name: Integration tests for TLS (microk8s)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e tls-integration

.github/workflows/release.yaml

@@ -117,6 +117,21 @@ jobs:
       - name: Run integration tests
         run: tox -e password-rotation-integration
 
+  integration-test-tls:
+    name: Integration tests for TLS
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          bootstrap-options: "--agent-version 2.9.29"
+      - name: Run integration tests
+        run: tox -e tls-integration
+
   release-to-charmhub:
     name: Release to CharmHub
     needs:
@@ -128,6 +143,7 @@ jobs:
       - integration-test-db-relation
       - integration-test-db-admin-relation
       - integration-test-password-rotation
+      - integration-test-tls
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

lib/charms/postgresql_k8s/v0/postgresql_tls.py

@@ -125,7 +125,9 @@ def _on_certificate_available(self, event: CertificateAvailableEvent) -> None:
             logger.error("An unknown certificate expiring.")
             return
 
-        self.charm.set_secret(SCOPE, "chain", event.chain)
+        self.charm.set_secret(
+            SCOPE, "chain", "\n".join(event.chain) if event.chain is not None else None
+        )
         self.charm.set_secret(SCOPE, "cert", event.certificate)
         self.charm.set_secret(SCOPE, "ca", event.ca)
 

pyproject.toml

@@ -18,6 +18,7 @@ markers = [
     "db_relation_tests",
     "db_admin_relation_tests",
     "password_rotation_tests",
+    "tls_tests",
 ]
 
 # Formatting tools configuration

tests/integration/helpers.py

@@ -15,7 +15,14 @@
 from lightkube.generic_resource import GenericNamespacedResource
 from lightkube.resources.core_v1 import Endpoints, Service
 from pytest_operator.plugin import OpsTest
-from tenacity import retry, retry_if_result, stop_after_attempt, wait_exponential
+from tenacity import (
+    RetryError,
+    Retrying,
+    retry,
+    retry_if_result,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 METADATA = yaml.safe_load(Path("./metadata.yaml").read_text())
 DATABASE_APP_NAME = METADATA["name"]
@@ -131,6 +138,7 @@ async def deploy_and_relate_application_with_postgresql(
     number_of_units: int,
     channel: str = "stable",
     relation: str = "db",
+    status: str = "blocked",
 ) -> int:
     """Helper function to deploy and relate application with PostgreSQL.
 
@@ -142,6 +150,7 @@ async def deploy_and_relate_application_with_postgresql(
         channel: The channel to use for the charm.
         relation: Name of the PostgreSQL relation to relate
             the application to.
+        status: The status to wait for in the application (default: blocked).
 
     Returns:
         the id of the created relation.
@@ -155,7 +164,7 @@ async def deploy_and_relate_application_with_postgresql(
     )
     await ops_test.model.wait_for_idle(
         apps=[application_name],
-        status="blocked",
+        status=status,
         raise_on_blocked=False,
         timeout=1000,
     )
@@ -179,6 +188,7 @@ async def execute_query_on_unit(
     password: str,
     query: str,
     database: str = "postgres",
+    sslmode: str = None,
 ):
     """Execute given PostgreSQL query on a unit.
 
@@ -187,12 +197,15 @@ async def execute_query_on_unit(
         password: The PostgreSQL superuser password.
         query: Query to execute.
         database: Optional database to connect to (defaults to postgres database).
+        sslmode: Optional ssl mode to use (defaults to None).
 
     Returns:
-        A list of rows that were potentially returned from the query.
+        The result of the query.
     """
+    extra_connection_parameters = f" sslmode={sslmode}" if sslmode is not None else ""
     with psycopg2.connect(
-        f"dbname='{database}' user='operator' host='{unit_address}' password='{password}' connect_timeout=10"
+        f"dbname='{database}' user='operator' host='{unit_address}'"
+        f"password='{password}' connect_timeout=10{extra_connection_parameters}"
     ) as connection, connection.cursor() as cursor:
         cursor.execute(query)
         output = list(itertools.chain(*cursor.fetchall()))
@@ -370,6 +383,40 @@ async def get_unit_address(ops_test: OpsTest, unit_name: str) -> str:
     return status["applications"][unit_name.split("/")[0]].units[unit_name]["address"]
 
 
+async def check_tls(ops_test: OpsTest, unit_name: str, enabled: bool) -> bool:
+    """Returns whether TLS is enabled on the specific PostgreSQL instance.
+
+    Args:
+        ops_test: The ops test framework instance.
+        unit_name: The name of the unit of the PostgreSQL instance.
+        enabled: check if TLS is enabled/disabled
+
+    Returns:
+        Whether TLS is enabled/disabled.
+    """
+    unit_address = await get_unit_address(ops_test, unit_name)
+    password = await get_password(ops_test)
+    try:
+        for attempt in Retrying(
+            stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30)
+        ):
+            with attempt:
+                output = await execute_query_on_unit(
+                    unit_address,
+                    password,
+                    "SHOW ssl;",
+                    sslmode="require" if enabled else "disable",
+                )
+                tls_enabled = "on" in output
+                if enabled != tls_enabled:
+                    raise ValueError(
+                        f"TLS is{' not' if not tls_enabled else ''} enabled on {unit_name}"
+                    )
+                return True
+    except RetryError:
+        return False
+
+
 async def restart_patroni(ops_test: OpsTest, unit_name: str) -> None:
     """Restart Patroni on a specific unit.
 

tests/integration/test_tls.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+import pytest as pytest
+from pytest_operator.plugin import OpsTest
+
+from tests.helpers import METADATA
+from tests.integration.helpers import (
+    DATABASE_APP_NAME,
+    check_database_creation,
+    check_database_users_existence,
+    check_tls,
+    deploy_and_relate_application_with_postgresql,
+)
+
+MATTERMOST_APP_NAME = "mattermost"
+TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator"
+APPLICATION_UNITS = 2
+DATABASE_UNITS = 3
+
+
+@pytest.mark.abort_on_fail
+@pytest.mark.tls_tests
+@pytest.mark.skip_if_deployed
+async def test_deploy_active(ops_test: OpsTest):
+    """Build the charm and deploy it."""
+    charm = await ops_test.build_charm(".")
+    async with ops_test.fast_forward():
+        await ops_test.model.deploy(
+            charm,
+            resources={
+                "postgresql-image": METADATA["resources"]["postgresql-image"]["upstream-source"]
+            },
+            application_name=DATABASE_APP_NAME,
+            num_units=DATABASE_UNITS,
+            trust=True,
+        )
+        await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1000)
+
+
+@pytest.mark.tls_tests
+async def test_mattermost_db(ops_test: OpsTest) -> None:
+    """Deploy Mattermost to test the 'db' relation.
+
+    Mattermost needs TLS enabled on PostgreSQL to correctly connect to it.
+
+    Args:
+        ops_test: The ops test framework
+    """
+    async with ops_test.fast_forward():
+        # Deploy TLS Certificates operator.
+        config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"}
+        await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="edge", config=config)
+        await ops_test.model.wait_for_idle(
+            apps=[TLS_CERTIFICATES_APP_NAME], status="active", timeout=1000
+        )
+
+        # Relate it to the PostgreSQL to enable TLS.
+        await ops_test.model.relate(DATABASE_APP_NAME, TLS_CERTIFICATES_APP_NAME)
+        await ops_test.model.wait_for_idle(status="active", timeout=1000)
+
+        # Wait for all units enabling TLS.
+        for unit in ops_test.model.applications[DATABASE_APP_NAME].units:
+            assert await check_tls(ops_test, unit.name, enabled=True)
+
+        # Deploy and check Mattermost user and database existence.
+        relation_id = await deploy_and_relate_application_with_postgresql(
+            ops_test, "mattermost-k8s", MATTERMOST_APP_NAME, APPLICATION_UNITS, status="waiting"
+        )
+        await check_database_creation(ops_test, "mattermost")
+
+        mattermost_users = [f"relation_id_{relation_id}"]
+
+        await check_database_users_existence(ops_test, mattermost_users, [])
+
+        # Remove the relation.
+        await ops_test.model.applications[DATABASE_APP_NAME].remove_relation(
+            f"{DATABASE_APP_NAME}:certificates", f"{TLS_CERTIFICATES_APP_NAME}:certificates"
+        )
+        await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1000)
+
+        # Wait for all units disabling TLS.
+        for unit in ops_test.model.applications[DATABASE_APP_NAME].units:
+            assert await check_tls(ops_test, unit.name, enabled=False)

tests/unit/test_postgresql_tls.py

@@ -27,7 +27,7 @@ def emit_certificate_available_event(self) -> None:
             certificate_signing_request="test-csr",
             certificate="test-cert",
             ca="test-ca",
-            chain="test-chain",
+            chain=["test-chain-ca-certificate", "test-chain-certificate"],
         )
 
     def emit_certificate_expiring_event(self) -> None:
@@ -150,7 +150,10 @@ def test_on_certificate_available(self, _push_tls_files_to_workload):
         self.emit_certificate_available_event()
         self.assertEqual(self.charm.get_secret(SCOPE, "ca"), "test-ca")
         self.assertEqual(self.charm.get_secret(SCOPE, "cert"), "test-cert")
-        self.assertEqual(self.charm.get_secret(SCOPE, "chain"), "test-chain")
+        self.assertEqual(
+            self.charm.get_secret(SCOPE, "chain"),
+            "test-chain-ca-certificate\ntest-chain-certificate",
+        )
         _push_tls_files_to_workload.assert_called_once()
 
     @patch_network_get(private_address="1.1.1.1")

tox.ini

@@ -120,6 +120,17 @@ deps =
 commands =
     pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 -m password_rotation_tests
 
+[testenv:tls-integration]
+description = Run TLS integration tests
+deps =
+    juju
+    pytest
+    pytest-operator
+    psycopg2-binary
+    -r{toxinidir}/requirements.txt
+commands =
+    pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} --durations=0 -m tls_tests
+
 [testenv:integration]
 description = Run all integration tests
 deps =