From 6efa8b615ad36a8dbf64dc073167a60d52d8b9a5 Mon Sep 17 00:00:00 2001 From: Matthew Warren Date: Mon, 4 Mar 2024 17:16:14 -0500 Subject: [PATCH 1/4] sc-12482 first batch of tests and lots of packaging love --- .pre-commit-config.yaml | 6 +- pyproject.toml | 16 +- src/dynamic_importer/__init__.py | 0 src/dynamic_importer/api/__init__.py | 0 .../dynamic_importer}/api/client.py | 45 +++- .../dynamic_importer}/api/exceptions.py | 0 .../dynamic_importer}/main.py | 1 - .../dynamic_importer}/processors/__init__.py | 2 +- .../dynamic_importer}/processors/dotenv.py | 1 - .../dynamic_importer}/processors/json.py | 2 +- .../dynamic_importer}/processors/tf.py | 1 - .../dynamic_importer}/processors/tfvars.py | 1 - .../dynamic_importer}/processors/yaml.py | 1 - .../dynamic_importer}/util.py | 0 {dynamic_importer => src}/samples/.env.sample | 2 +- .../samples/azureTRE.yaml | 2 +- .../samples/dotenvs/.env.default.sample | 0 .../samples/dotenvs/.env.dev.sample | 0 .../samples/dotenvs/.env.prod.sample | 0 .../samples/dotenvs/.env.staging.sample | 0 {dynamic_importer => src}/samples/short.json | 2 +- .../samples/terraform.tfvars | 2 +- .../samples/variables.tf | 2 +- src/tests/__init__.py | 0 src/tests/fixtures/requests.py | 235 ++++++++++++++++ src/tests/test_cli.py | 35 +++ src/tests/test_client.py | 254 ++++++++++++++++++ 27 files changed, 582 insertions(+), 28 deletions(-) create mode 100644 src/dynamic_importer/__init__.py create mode 100644 src/dynamic_importer/api/__init__.py rename {dynamic_importer => src/dynamic_importer}/api/client.py (94%) rename {dynamic_importer => src/dynamic_importer}/api/exceptions.py (100%) rename {dynamic_importer => src/dynamic_importer}/main.py (99%) rename {dynamic_importer => src/dynamic_importer}/processors/__init__.py (99%) rename {dynamic_importer => src/dynamic_importer}/processors/dotenv.py (99%) rename {dynamic_importer => src/dynamic_importer}/processors/json.py (95%) rename {dynamic_importer => src/dynamic_importer}/processors/tf.py (99%) rename {dynamic_importer => src/dynamic_importer}/processors/tfvars.py (99%) rename {dynamic_importer => src/dynamic_importer}/processors/yaml.py (99%) rename {dynamic_importer => src/dynamic_importer}/util.py (100%) rename {dynamic_importer => src}/samples/.env.sample (89%) rename {dynamic_importer => src}/samples/azureTRE.yaml (99%) rename {dynamic_importer => src}/samples/dotenvs/.env.default.sample (100%) rename {dynamic_importer => src}/samples/dotenvs/.env.dev.sample (100%) rename {dynamic_importer => src}/samples/dotenvs/.env.prod.sample (100%) rename {dynamic_importer => src}/samples/dotenvs/.env.staging.sample (100%) rename {dynamic_importer => src}/samples/short.json (99%) rename {dynamic_importer => src}/samples/terraform.tfvars (93%) rename {dynamic_importer => src}/samples/variables.tf (99%) create mode 100644 src/tests/__init__.py create mode 100644 src/tests/fixtures/requests.py create mode 100644 src/tests/test_cli.py create mode 100644 src/tests/test_client.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 710a223..2d782eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -10,6 +10,8 @@ repos: - id: check-added-large-files - id: debug-statements - id: name-tests-test + exclude: ^src/tests/fixtures/ + args: ["--pytest-test-first"] - id: requirements-txt-fixer - repo: https://github.com/asottile/reorder-python-imports rev: v3.12.0 @@ -18,7 +20,7 @@ repos: exclude: ^(pre_commit/resources/|testing/resources/python3_hooks_repo/) args: [--py39-plus, --add-import, 'from __future__ import annotations'] - repo: https://github.com/ambv/black - rev: 22.3.0 + rev: 24.2.0 hooks: - id: black language_version: python3.11 diff --git a/pyproject.toml b/pyproject.toml index 0d41f14..4fb82fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,10 +19,22 @@ dev = [ "pre-commit", "mypy", "pytest", + "pytest-black", "pytest-cov", + "pytest-mypy", + "types-requests", + "types-pyyaml", ] [project.scripts] cloudtruth-dynamic-importer = "dynamic_importer.main:import_config" -[tool.setuptools] -packages = ["dynamic_importer"] +[tool.mypy] +packages = "dynamic_importer" + +[tool.pytest.ini_options] +addopts = ["--import-mode=importlib", "--cov=dynamic_importer", "--cov-report=term-missing", "--cov-report=xml", "--mypy"] +minversion = 6.0 + +[tool.setuptools.packages.find] +where = ["src"] +include = ["dynamic_importer", "tests"] diff --git a/src/dynamic_importer/__init__.py b/src/dynamic_importer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dynamic_importer/api/__init__.py b/src/dynamic_importer/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dynamic_importer/api/client.py b/src/dynamic_importer/api/client.py similarity index 94% rename from dynamic_importer/api/client.py rename to src/dynamic_importer/api/client.py index a7e6fbc..816b646 100644 --- a/dynamic_importer/api/client.py +++ b/src/dynamic_importer/api/client.py @@ -6,7 +6,6 @@ from typing import Optional import requests - from dynamic_importer.api.exceptions import ResourceNotFoundError DEFAULT_API_HOST = "api.cloudtruth.io" @@ -139,14 +138,6 @@ def get_template(self, project_name: str, template_name: str) -> Dict: except KeyError: raise ResourceNotFoundError(f"Template {template_name} not found") - def _populate_type_cache(self) -> None: - types = self._make_request("types", "GET") - for ct_type in types["results"]: - self.cache["types"][ct_type["name"]] = { - "url": ct_type["url"], - "id": ct_type["id"], - } - def get_value( self, project_name: str, parameter_name: str, environment_name: str ) -> Dict: @@ -173,6 +164,14 @@ def get_value( except KeyError: raise ResourceNotFoundError(f"Parameter {parameter_name} not found") + def _populate_type_cache(self) -> None: + types = self._make_request("types", "GET") + for ct_type in types["results"]: + self.cache["types"][ct_type["name"]] = { + "url": ct_type["url"], + "id": ct_type["id"], + } + def get_type_id(self, type_name: str) -> str: if type_name in self.cache["types"].keys(): return self.cache["types"][type_name]["id"] @@ -180,7 +179,7 @@ def get_type_id(self, type_name: str) -> str: try: return self.cache["types"][type_name]["id"] except KeyError: - raise ValueError(f"Type {type_name} not found") + raise ResourceNotFoundError(f"Type {type_name} not found") def get_type_url(self, type_name: str) -> str: if type_name in self.cache["types"].keys(): @@ -189,7 +188,7 @@ def get_type_url(self, type_name: str) -> str: try: return self.cache["types"][type_name]["url"] except KeyError: - raise ValueError(f"Type {type_name} not found") + raise ResourceNotFoundError(f"Type {type_name} not found") def create_project(self, name: str, description: str = "") -> Dict: resp = self._make_request( @@ -320,6 +319,25 @@ def update_value( data={"environment": environment_id, "internal_value": value}, ) + def update_parameter( + self, + project_name: str, + parameter_id: str, + name: str, + description: str = "", + type_name: str = "string", + ) -> Dict: + project_id = self.get_project_id(project_name) + return self._make_request( + f"projects/{project_id}/parameters/{parameter_id}", + "PATCH", + data={ + "name": name, + "description": description, + "type": type_name, + }, + ) + def update_template( self, project_name: str, @@ -352,7 +370,10 @@ def upsert_parameter( self.create_project(project_name) try: - return self.get_parameter(project_name, name) + parameter_id = self.get_parameter_id(project_name, name) + return self.update_parameter( + project_name, parameter_id, name, description, type_name + ) except ResourceNotFoundError: return self.create_parameter( project_name, name, description, type_name, secret, create_dependencies diff --git a/dynamic_importer/api/exceptions.py b/src/dynamic_importer/api/exceptions.py similarity index 100% rename from dynamic_importer/api/exceptions.py rename to src/dynamic_importer/api/exceptions.py diff --git a/dynamic_importer/main.py b/src/dynamic_importer/main.py similarity index 99% rename from dynamic_importer/main.py rename to src/dynamic_importer/main.py index 76b3216..3138a65 100644 --- a/dynamic_importer/main.py +++ b/src/dynamic_importer/main.py @@ -5,7 +5,6 @@ import click import urllib3 - from dynamic_importer.api.client import CTClient from dynamic_importer.processors import BaseProcessor from dynamic_importer.processors import get_processor_class diff --git a/dynamic_importer/processors/__init__.py b/src/dynamic_importer/processors/__init__.py similarity index 99% rename from dynamic_importer/processors/__init__.py rename to src/dynamic_importer/processors/__init__.py index a8fb34a..aea227e 100644 --- a/dynamic_importer/processors/__init__.py +++ b/src/dynamic_importer/processors/__init__.py @@ -52,7 +52,7 @@ class BaseProcessor: values = None template: Dict = {} - def __init__(self, env_values: Dict): + def __init__(self, env_values: Dict) -> None: raise NotImplementedError("Subclasses must implement the __init__ method") def guess_type(self, value): diff --git a/dynamic_importer/processors/dotenv.py b/src/dynamic_importer/processors/dotenv.py similarity index 99% rename from dynamic_importer/processors/dotenv.py rename to src/dynamic_importer/processors/dotenv.py index 0c9c54d..9c3f512 100644 --- a/dynamic_importer/processors/dotenv.py +++ b/src/dynamic_importer/processors/dotenv.py @@ -6,7 +6,6 @@ from dotenv import dotenv_values # type: ignore[import-not-found] from dotenv.main import DotEnv # type: ignore[import-not-found] - from dynamic_importer.processors import BaseProcessor diff --git a/dynamic_importer/processors/json.py b/src/dynamic_importer/processors/json.py similarity index 95% rename from dynamic_importer/processors/json.py rename to src/dynamic_importer/processors/json.py index 06acd0f..654e875 100644 --- a/dynamic_importer/processors/json.py +++ b/src/dynamic_importer/processors/json.py @@ -9,7 +9,7 @@ class JSONProcessor(BaseProcessor): - def __init__(self, env_values: Dict): + def __init__(self, env_values: Dict) -> None: for env, file_path in env_values.items(): with open(file_path, "r") as fp: try: diff --git a/dynamic_importer/processors/tf.py b/src/dynamic_importer/processors/tf.py similarity index 99% rename from dynamic_importer/processors/tf.py rename to src/dynamic_importer/processors/tf.py index de4b483..9296fce 100644 --- a/dynamic_importer/processors/tf.py +++ b/src/dynamic_importer/processors/tf.py @@ -9,7 +9,6 @@ from typing import Union import hcl2 - from dynamic_importer.processors import BaseProcessor diff --git a/dynamic_importer/processors/tfvars.py b/src/dynamic_importer/processors/tfvars.py similarity index 99% rename from dynamic_importer/processors/tfvars.py rename to src/dynamic_importer/processors/tfvars.py index 78961f6..4546d85 100644 --- a/dynamic_importer/processors/tfvars.py +++ b/src/dynamic_importer/processors/tfvars.py @@ -5,7 +5,6 @@ from typing import Optional import hcl2 - from dynamic_importer.processors import BaseProcessor diff --git a/dynamic_importer/processors/yaml.py b/src/dynamic_importer/processors/yaml.py similarity index 99% rename from dynamic_importer/processors/yaml.py rename to src/dynamic_importer/processors/yaml.py index 692ef1b..dd8bf57 100644 --- a/dynamic_importer/processors/yaml.py +++ b/src/dynamic_importer/processors/yaml.py @@ -5,7 +5,6 @@ from typing import Optional import yaml - from dynamic_importer.processors import BaseProcessor diff --git a/dynamic_importer/util.py b/src/dynamic_importer/util.py similarity index 100% rename from dynamic_importer/util.py rename to src/dynamic_importer/util.py diff --git a/dynamic_importer/samples/.env.sample b/src/samples/.env.sample similarity index 89% rename from dynamic_importer/samples/.env.sample rename to src/samples/.env.sample index 9c79c7f..4f3b4db 100644 --- a/dynamic_importer/samples/.env.sample +++ b/src/samples/.env.sample @@ -12,4 +12,4 @@ SECRET_HASH="something-with-a-#-hash" SMTP_SERVER="smtp.example.com" SMTP_PORT=587 SMTP_USER="fedex" -SMTP_PASSWORD="itsasecrettoeverybody" \ No newline at end of file +SMTP_PASSWORD="itsasecrettoeverybody" diff --git a/dynamic_importer/samples/azureTRE.yaml b/src/samples/azureTRE.yaml similarity index 99% rename from dynamic_importer/samples/azureTRE.yaml rename to src/samples/azureTRE.yaml index 640205a..450a8da 100644 --- a/dynamic_importer/samples/azureTRE.yaml +++ b/src/samples/azureTRE.yaml @@ -75,4 +75,4 @@ developer_settings: # Used by the API and Resource processor application to change log level # Can be "ERROR", "WARNING", "INFO", "DEBUG" - logging_level: "INFO" \ No newline at end of file + logging_level: "INFO" diff --git a/dynamic_importer/samples/dotenvs/.env.default.sample b/src/samples/dotenvs/.env.default.sample similarity index 100% rename from dynamic_importer/samples/dotenvs/.env.default.sample rename to src/samples/dotenvs/.env.default.sample diff --git a/dynamic_importer/samples/dotenvs/.env.dev.sample b/src/samples/dotenvs/.env.dev.sample similarity index 100% rename from dynamic_importer/samples/dotenvs/.env.dev.sample rename to src/samples/dotenvs/.env.dev.sample diff --git a/dynamic_importer/samples/dotenvs/.env.prod.sample b/src/samples/dotenvs/.env.prod.sample similarity index 100% rename from dynamic_importer/samples/dotenvs/.env.prod.sample rename to src/samples/dotenvs/.env.prod.sample diff --git a/dynamic_importer/samples/dotenvs/.env.staging.sample b/src/samples/dotenvs/.env.staging.sample similarity index 100% rename from dynamic_importer/samples/dotenvs/.env.staging.sample rename to src/samples/dotenvs/.env.staging.sample diff --git a/dynamic_importer/samples/short.json b/src/samples/short.json similarity index 99% rename from dynamic_importer/samples/short.json rename to src/samples/short.json index a9615d9..a982ebe 100644 --- a/dynamic_importer/samples/short.json +++ b/src/samples/short.json @@ -16,4 +16,4 @@ ] ] } -} \ No newline at end of file +} diff --git a/dynamic_importer/samples/terraform.tfvars b/src/samples/terraform.tfvars similarity index 93% rename from dynamic_importer/samples/terraform.tfvars rename to src/samples/terraform.tfvars index 3387cd7..07011f6 100644 --- a/dynamic_importer/samples/terraform.tfvars +++ b/src/samples/terraform.tfvars @@ -25,4 +25,4 @@ instance_ami = "ami-0123456789abcdef0" instance_type = "t2.micro" # Security group IDs for the EC2 instance (comma-separated list) -security_group_ids = "sg-12345678,sg-87654321" \ No newline at end of file +security_group_ids = "sg-12345678,sg-87654321" diff --git a/dynamic_importer/samples/variables.tf b/src/samples/variables.tf similarity index 99% rename from dynamic_importer/samples/variables.tf rename to src/samples/variables.tf index 42ad20a..855e66a 100644 --- a/dynamic_importer/samples/variables.tf +++ b/src/samples/variables.tf @@ -50,4 +50,4 @@ variable "security_group_ids" { description = "Security group IDs for the EC2 instance (comma-separated list)" type = string default = "sg-12345678,sg-87654321" -} \ No newline at end of file +} diff --git a/src/tests/__init__.py b/src/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests/fixtures/requests.py b/src/tests/fixtures/requests.py new file mode 100644 index 0000000..633144f --- /dev/null +++ b/src/tests/fixtures/requests.py @@ -0,0 +1,235 @@ +from __future__ import annotations + + +def mocked_requests_get(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def text(self): + return self.json_data + + get_url = args[0] + if get_url == "https://api.cloudtruth.io/api/v1/projects/": + return MockResponse( + {"results": [{"id": "1", "url": "/projects/1/", "name": "myproj"}]}, 200 + ) + elif get_url == "https://api.cloudtruth.io/api/v1/types/": + return MockResponse( + {"results": [{"id": "1", "url": "/types/1/", "name": "string"}]}, 200 + ) + elif get_url == "https://api.cloudtruth.io/api/v1/environments/": + return MockResponse( + { + "results": [ + {"id": "1", "url": "/environments/1/", "name": "default"}, + {"id": "2", "url": "/environments/2/", "name": "production"}, + ] + }, + 200, + ) + elif get_url == "https://api.cloudtruth.io/api/v1/projects/1/parameters/": + return MockResponse( + { + "results": [ + {"id": "1", "url": "/projects/1/parameters/1/", "name": "param1"} + ] + }, + 200, + ) + elif get_url == "https://api.cloudtruth.io/api/v1/projects/1/templates/": + return MockResponse( + { + "results": [ + {"id": "1", "url": "/projects/1/templates/1/", "name": "template1"} + ] + }, + 200, + ) + elif get_url == "https://api.cloudtruth.io/api/v1/projects/1/parameters/1/values/": + return MockResponse( + { + "results": [ + { + "id": "1", + "url": "/projects/1/parameters/1/values/1/", + "environment_name": "production", + } + ] + }, + 200, + ) + + return MockResponse(None, 404) + + +def mocked_requests_patch(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def text(self): + return self.json_data + + url = args[0] + if url == "https://api.cloudtruth.io/api/v1/projects/1/": + return MockResponse({"id": "1", "url": "/projects/1/", "name": "myproj"}, 200) + elif url == "https://api.cloudtruth.io/api/v1/environments/2/": + return MockResponse( + {"id": "2", "url": "/environments/2/", "name": "production"}, 200 + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/1/parameters/1/": + return MockResponse( + {"id": "1", "url": "/projects/1/parameters/1/", "name": "param1"}, 200 + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/1/templates/1/": + return MockResponse( + {"id": "1", "url": "/projects/1/templates/1/", "name": "template1"}, 200 + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/1/parameters/1/values/1/": + return MockResponse( + { + "id": "1", + "url": "/projects/1/parameters/1/values/1/", + "environment_name": "production", + }, + 200, + ) + + return MockResponse(None, 404) + + +def mocked_requests_upsert_get(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def text(self): + return self.json_data + + url = args[0] + if url == "https://api.cloudtruth.io/api/v1/projects/": + return MockResponse( + {"results": [{"id": "1", "url": "/projects/1/", "name": "myproj"}]}, 200 + ) + elif url == "https://api.cloudtruth.io/api/v1/types/": + return MockResponse( + {"results": [{"id": "1", "url": "/types/1/", "name": "string"}]}, 200 + ) + elif url == "https://api.cloudtruth.io/api/v1/environments/": + return MockResponse( + { + "results": [ + {"id": "1", "url": "/environments/1/", "name": "default"}, + {"id": "2", "url": "/environments/2/", "name": "production"}, + ] + }, + 200, + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/1/parameters/1/": + return MockResponse( + { + "results": [ + {"id": "1", "url": "/projects/1/parameters/1/", "name": "param1"} + ] + }, + 200, + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/1/templates/1/": + return MockResponse( + { + "results": [ + {"id": "1", "url": "/projects/1/templates/1/", "name": "template1"} + ] + }, + 200, + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/1/parameters/1/values/1/": + return MockResponse( + { + "results": [ + { + "id": "1", + "url": "/projects/1/parameters/1/values/1/", + "environment_name": "production", + } + ] + }, + 200, + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/2/parameters/": + return MockResponse({"results": []}, 200) + elif url == "https://api.cloudtruth.io/api/v1/projects/2/templates/": + return MockResponse({"results": []}, 200) + elif url == "https://api.cloudtruth.io/api/v1/projects/2/parameters/2/values/": + return MockResponse({"results": []}, 200) + + return MockResponse(None, 404) + + +def mocked_requests_post(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def text(self): + return self.json_data + + url = args[0] + if url == "https://api.cloudtruth.io/api/v1/projects/": + return MockResponse( + {"id": "2", "url": "/projects/2/", "name": kwargs["json"]["name"]}, 201 + ) + elif url == "https://api.cloudtruth.io/api/v1/types/": + return MockResponse( + {"id": "2", "url": "/types/2/", "name": kwargs["json"]["name"]}, 201 + ) + elif url == "https://api.cloudtruth.io/api/v1/environments/": + return MockResponse( + {"id": "3", "url": "/environments/3/", "name": kwargs["json"]["name"]}, 201 + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/2/parameters/": + return MockResponse( + { + "id": "2", + "url": "/projects/2/parameters/2/", + "name": kwargs["json"]["name"], + }, + 201, + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/2/templates/": + return MockResponse( + { + "id": "2", + "url": "/projects/2/templates/2/", + "name": kwargs["json"]["name"], + }, + 201, + ) + elif url == "https://api.cloudtruth.io/api/v1/projects/2/parameters/2/values/": + return MockResponse( + { + "id": "2", + "url": "/projects/2/parameters/2/values/2/", + "environment_name": "production", + }, + 201, + ) + + return MockResponse(None, 404) diff --git a/src/tests/test_cli.py b/src/tests/test_cli.py new file mode 100644 index 0000000..783025f --- /dev/null +++ b/src/tests/test_cli.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from unittest import TestCase + +from click.testing import CliRunner +from dynamic_importer.main import import_config + + +class TestCLI(TestCase): + def test_cli_help(self): + runner = CliRunner() + result = runner.invoke(import_config, ["--help"]) + self.assertEqual(0, result.exit_code) + self.assertIn( + """Usage: import-config [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. +""", + result.output, + ) + + def test_cli_process_configs(self): + runner = CliRunner() + result = runner.invoke( + import_config, + [ + "process-configs", + "-t", + "dotenv", + "--default-values", + "samples/.env.sample", + ], + ) + self.assertEqual(0, result.exit_code) diff --git a/src/tests/test_client.py b/src/tests/test_client.py new file mode 100644 index 0000000..13bcd92 --- /dev/null +++ b/src/tests/test_client.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import os +from unittest import mock +from unittest import TestCase + +from dynamic_importer.api.client import CTClient +from dynamic_importer.api.client import DEFAULT_API_HOST +from dynamic_importer.api.exceptions import ResourceNotFoundError +from tests.mocks import mocked_requests_get +from tests.mocks import mocked_requests_patch +from tests.mocks import mocked_requests_post +from tests.mocks import mocked_requests_upsert_get + + +class TestClient(TestCase): + def test_client_init(self): + mock_api_key = "super-secret-api-key11!!" + client = CTClient(mock_api_key) + self.assertEqual(client.base_url, f"https://{DEFAULT_API_HOST}/api/v1") + self.assertEqual(client.headers, {"Authorization": f"Api-Key {mock_api_key}"}) + self.assertEqual(client.cache, {}) + + @mock.patch.dict(os.environ, {"CLOUDTRUTH_API_HOST": "localhost:8000"}) + def test_client_init_with_host_override(self): + client = CTClient("super-secret-api-key11!!") + self.assertEqual(client.base_url, "https://localhost:8000/api/v1") + + @mock.patch( + "dynamic_importer.api.client.requests.get", side_effect=mocked_requests_get + ) + def test_client_get(self, mock_get): + client = CTClient("super-secret-api-key11!!") + + # projects + self.assertEqual(client.get_project_id("myproj"), "1") + self.assertEqual(mock_get.call_count, 1) + self.assertDictEqual( + client.cache["projects"]["myproj"], {"id": "1", "url": "/projects/1/"} + ) + self.assertEqual(client.get_project_id("myproj"), "1") + self.assertEqual(mock_get.call_count, 1) + + # environments + self.assertEqual(client.get_environment_id("production"), "2") + self.assertEqual(mock_get.call_count, 2) + self.assertDictEqual( + client.cache["environments"]["production"], + {"id": "2", "url": "/environments/2/"}, + ) + self.assertEqual(client.get_environment_url("production"), "/environments/2/") + self.assertEqual(mock_get.call_count, 2) + + # parameter + self.assertDictEqual( + client.get_parameter("myproj", "param1"), + {"id": "1", "url": "/projects/1/parameters/1/"}, + ) + self.assertEqual(mock_get.call_count, 3) + self.assertDictEqual( + client.cache["parameters"]["myproj/param1"], + {"id": "1", "url": "/projects/1/parameters/1/"}, + ) + self.assertEqual(client.get_parameter_id("myproj", "param1"), "1") + self.assertEqual(mock_get.call_count, 3) + self.assertEqual( + client.get_parameter("myproj", "param1"), + {"id": "1", "url": "/projects/1/parameters/1/"}, + ) + self.assertEqual(mock_get.call_count, 3) + + # template + self.assertDictEqual( + client.get_template("myproj", "template1"), + {"id": "1", "url": "/projects/1/templates/1/"}, + ) + self.assertEqual(mock_get.call_count, 4) + self.assertDictEqual( + client.cache["templates"]["myproj/template1"], + {"id": "1", "url": "/projects/1/templates/1/"}, + ) + self.assertDictEqual( + client.get_template("myproj", "template1"), + {"id": "1", "url": "/projects/1/templates/1/"}, + ) + self.assertEqual(mock_get.call_count, 4) + + # values + self.assertDictEqual( + client.get_value("myproj", "param1", "production"), + {"id": "1", "url": "/projects/1/parameters/1/values/1/"}, + ) + self.assertEqual(mock_get.call_count, 5) + self.assertDictEqual( + client.cache["values"]["myproj/param1/production"], + {"id": "1", "url": "/projects/1/parameters/1/values/1/"}, + ) + self.assertDictEqual( + client.get_value("myproj", "param1", "production"), + {"id": "1", "url": "/projects/1/parameters/1/values/1/"}, + ) + self.assertEqual(mock_get.call_count, 5) + + # types + self.assertEqual(client.get_type_id("string"), "1") + self.assertEqual(mock_get.call_count, 6) + self.assertDictEqual( + client.cache["types"]["string"], {"id": "1", "url": "/types/1/"} + ) + self.assertEqual(client.get_type_url("string"), "/types/1/") + self.assertEqual(mock_get.call_count, 6) + + # errors + with self.assertRaises(ResourceNotFoundError): + client.get_project_id("invalid") + self.assertEqual(mock_get.call_count, 7) + with self.assertRaises(ResourceNotFoundError): + client.get_environment_id("invalid") + self.assertEqual(mock_get.call_count, 8) + with self.assertRaises(ResourceNotFoundError): + client.get_environment_url("invalid"), None + self.assertEqual(mock_get.call_count, 9) + with self.assertRaises(ResourceNotFoundError): + client.get_parameter("invalid", "invalid") + self.assertEqual(mock_get.call_count, 10) + with self.assertRaises(ResourceNotFoundError): + client.get_parameter_id("invalid", "invalid") + self.assertEqual(mock_get.call_count, 11) + with self.assertRaises(ResourceNotFoundError): + client.get_template("invalid", "invalid") + self.assertEqual(mock_get.call_count, 12) + with self.assertRaises(ResourceNotFoundError): + client.get_value("invalid", "invalid", "invalid") + self.assertEqual(mock_get.call_count, 13) + with self.assertRaises(ResourceNotFoundError): + client.get_type_id("invalid") + self.assertEqual(mock_get.call_count, 14) + with self.assertRaises(ResourceNotFoundError): + client.get_type_url("invalid") + self.assertEqual(mock_get.call_count, 15) + + # generic failure + with self.assertRaises(RuntimeError): + client._make_request("invalid", "GET") + + @mock.patch( + "dynamic_importer.api.client.requests.get", side_effect=mocked_requests_get + ) + @mock.patch( + "dynamic_importer.api.client.requests.post", side_effect=mocked_requests_post + ) + def test_client_create(self, mock_post, mock_get): + client = CTClient("time-to-create-the-things") + client.create_project("myproj") + mock_post.assert_called_once() + client.create_environment("production") + self.assertEqual(mock_post.call_count, 2) + client.create_parameter("myproj", "param1") + self.assertEqual(mock_post.call_count, 3) + client.create_template("myproj", "template1", "wooooooooo template!") + self.assertEqual(mock_post.call_count, 4) + client.create_value("myproj", "param1", "production", "important value") + self.assertEqual(mock_post.call_count, 5) + + @mock.patch( + "dynamic_importer.api.client.requests.get", + side_effect=mocked_requests_upsert_get, + ) + @mock.patch( + "dynamic_importer.api.client.requests.patch", side_effect=mocked_requests_patch + ) + @mock.patch( + "dynamic_importer.api.client.requests.post", side_effect=mocked_requests_post + ) + def test_client_upsert_create_dependencies(self, mock_post, mock_patch, mock_get): + client = CTClient("lets-get-upserting") + client.upsert_template( + "proj2", "template1", "wooooooooo template!", create_dependencies=True + ) + # created project and template + self.assertEqual(mock_post.call_count, 2) + client.upsert_value( + "proj3", + "param1", + "development", + "important value", + create_dependencies=True, + ) + # created project parameter and value + self.assertEqual(mock_post.call_count, 6) + client.upsert_parameter("proj5", "param10", create_dependencies=True) + self.assertEqual(mock_post.call_count, 8) + + @mock.patch( + "dynamic_importer.api.client.requests.get", side_effect=mocked_requests_get + ) + @mock.patch( + "dynamic_importer.api.client.requests.patch", side_effect=mocked_requests_patch + ) + @mock.patch( + "dynamic_importer.api.client.requests.post", side_effect=mocked_requests_post + ) + def test_client_upsert_no_create_dependencies( + self, mock_post, mock_patch, mock_get + ): + client = CTClient("lets-get-updating") + client.upsert_template( + "myproj", "template1", "wooooooooo template!", create_dependencies=False + ) + self.assertEqual(mock_patch.call_count, 1) + client.upsert_value( + "myproj", + "param1", + "production", + "important value", + create_dependencies=False, + ) + self.assertEqual(mock_patch.call_count, 2) + + @mock.patch( + "dynamic_importer.api.client.requests.get", side_effect=mocked_requests_get + ) + def test_client_upsert_raises(self, mock_get): + client = CTClient("time-to-error-out!") + with self.assertRaises(ResourceNotFoundError): + client.upsert_template( + "proj2", "template1", "wooooooooo template!", create_dependencies=False + ) + with self.assertRaises(ResourceNotFoundError): + client.upsert_parameter("proj5", "param10", create_dependencies=False) + with self.assertRaises(ResourceNotFoundError): + client.upsert_value( + "proj3", + "param3", + "development", + "important value", + create_dependencies=False, + ) + with self.assertRaises(ResourceNotFoundError): + client.upsert_value( + "myproj", + "param3", + "production", + "important value", + create_dependencies=False, + ) + with self.assertRaises(ResourceNotFoundError): + client.upsert_value( + "myproj", + "param1", + "development", + "important value", + create_dependencies=False, + ) From ef5adbd225c49ec76c166fcf95f62824e4c23167 Mon Sep 17 00:00:00 2001 From: Matthew Warren Date: Mon, 4 Mar 2024 17:20:56 -0500 Subject: [PATCH 2/4] reference fix --- src/tests/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/test_client.py b/src/tests/test_client.py index 13bcd92..d20e136 100644 --- a/src/tests/test_client.py +++ b/src/tests/test_client.py @@ -7,10 +7,10 @@ from dynamic_importer.api.client import CTClient from dynamic_importer.api.client import DEFAULT_API_HOST from dynamic_importer.api.exceptions import ResourceNotFoundError -from tests.mocks import mocked_requests_get -from tests.mocks import mocked_requests_patch -from tests.mocks import mocked_requests_post -from tests.mocks import mocked_requests_upsert_get +from tests.fixtures.requests import mocked_requests_get +from tests.fixtures.requests import mocked_requests_patch +from tests.fixtures.requests import mocked_requests_post +from tests.fixtures.requests import mocked_requests_upsert_get class TestClient(TestCase): From 00e12260faaa7d557bcc9f8d2abbf9cdfa684fc9 Mon Sep 17 00:00:00 2001 From: Matthew Warren Date: Tue, 5 Mar 2024 10:29:31 -0500 Subject: [PATCH 3/4] finish up first pass of tests --- src/dynamic_importer/processors/dotenv.py | 5 + src/dynamic_importer/processors/json.py | 14 +- src/dynamic_importer/processors/tf.py | 7 +- src/dynamic_importer/processors/tfvars.py | 4 +- src/tests/fixtures/requests.py | 83 ++++++++++ src/tests/test_cli.py | 191 +++++++++++++++++++++- 6 files changed, 293 insertions(+), 11 deletions(-) diff --git a/src/dynamic_importer/processors/dotenv.py b/src/dynamic_importer/processors/dotenv.py index 9c3f512..8c75314 100644 --- a/src/dynamic_importer/processors/dotenv.py +++ b/src/dynamic_importer/processors/dotenv.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from re import sub from typing import Dict from typing import Optional @@ -12,6 +13,10 @@ class DotEnvProcessor(BaseProcessor): def __init__(self, env_values: Dict) -> None: for env, file_path in env_values.items(): + if not os.path.isfile(file_path): + raise ValueError( + f"Path to environment values file {file_path} could not be accessed." + ) self.raw_data[env] = dotenv_values(file_path) def encode_template_references( diff --git a/src/dynamic_importer/processors/json.py b/src/dynamic_importer/processors/json.py index 654e875..5a53349 100644 --- a/src/dynamic_importer/processors/json.py +++ b/src/dynamic_importer/processors/json.py @@ -10,6 +10,9 @@ class JSONProcessor(BaseProcessor): def __init__(self, env_values: Dict) -> None: + # the click test library seems to reuse Processor classes somehow + # so we reset self.parameters_and_values to avoid test pollution + self.parameters_and_values: Dict = {} for env, file_path in env_values.items(): with open(file_path, "r") as fp: try: @@ -25,9 +28,12 @@ def encode_template_references( template_body = json.dumps(template, indent=4) if config_data: for _, data in config_data.items(): - if data["type"] != "string": - # JSON strings use double quotes - reference = rf'"(\{{\{{\s+cloudtruth.parameters.{data["param_name"]}\s+\}}\}})"' - template_body = sub(reference, r"\1", template_body) + try: + if data["type"] != "string": + # JSON strings use double quotes + reference = rf'"(\{{\{{\s+cloudtruth.parameters.{data["param_name"]}\s+\}}\}})"' + template_body = sub(reference, r"\1", template_body) + except KeyError: + raise RuntimeError(f"data: {data}") return template_body diff --git a/src/dynamic_importer/processors/tf.py b/src/dynamic_importer/processors/tf.py index 9296fce..ee70745 100644 --- a/src/dynamic_importer/processors/tf.py +++ b/src/dynamic_importer/processors/tf.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from re import sub from typing import Any from typing import Dict @@ -17,6 +18,10 @@ class TFProcessor(BaseProcessor): def __init__(self, env_values: Dict) -> None: for env, file_path in env_values.items(): + if not os.path.isfile(file_path): + raise ValueError( + f"Path to environment values file {file_path} could not be accessed." + ) try: with open(file_path, "r") as fp: # hcl2 does not support dumping to a string/file, @@ -36,7 +41,7 @@ def encode_template_references( environment = "default" if config_data: for _, data in config_data.items(): - value = data["values"][environment] + value = str(data["values"][environment]) reference = f'{{{{ cloudtruth.parameters.{data["param_name"]} }}}}' template_body = sub(value, reference, template_body) diff --git a/src/dynamic_importer/processors/tfvars.py b/src/dynamic_importer/processors/tfvars.py index 4546d85..825a231 100644 --- a/src/dynamic_importer/processors/tfvars.py +++ b/src/dynamic_importer/processors/tfvars.py @@ -31,8 +31,8 @@ def encode_template_references( environment = "default" if config_data: for _, data in config_data.items(): - value = data["values"][environment] - reference = f'{{{{ cloudtruth.parameters.{data["param_name"]} }}}}' + value = str(data["values"][environment]) + reference = rf'{{{{ cloudtruth.parameters.{data["param_name"]} }}}}' template_body = sub(value, reference, template_body) return template_body diff --git a/src/tests/fixtures/requests.py b/src/tests/fixtures/requests.py index 633144f..147da77 100644 --- a/src/tests/fixtures/requests.py +++ b/src/tests/fixtures/requests.py @@ -233,3 +233,86 @@ def text(self): ) return MockResponse(None, 404) + + +def mocked_requests_localhost_get(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def text(self): + return self.json_data + + url = args[0] + if url == "https://localhost:8000/api/v1/projects/": + return MockResponse( + {"results": [{"id": "1", "url": "/projects/1/", "name": "testproj"}]}, 200 + ) + elif url == "https://localhost:8000/api/v1/environments/": + return MockResponse( + {"results": [{"id": "1", "url": "/environments/1/", "name": "default"}]}, + 200, + ) + elif url == "https://localhost:8000/api/v1/projects/1/parameters/": + return MockResponse( + { + "results": [ + {"id": "1", "url": "/projects/1/parameters/1/", "name": "boop"} + ] + }, + 200, + ) + elif url == "https://localhost:8000/api/v1/projects/1/templates/": + return MockResponse({"results": []}, 200) + elif url == "https://localhost:8000/api/v1/projects/1/parameters/2/values/": + return MockResponse({"results": []}, 200) + + +def mocked_requests_localhost_post(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def text(self): + return self.json_data + + url = args[0] + if url == "https://localhost:8000/api/v1/projects/": + return MockResponse( + {"id": "2", "url": "/projects/2/", "name": kwargs["json"]["name"]}, 201 + ) + elif url == "https://localhost:8000/api/v1/projects/1/parameters/": + return MockResponse( + { + "id": "2", + "url": "/projects/1/parameters/2/", + "name": kwargs["json"]["name"], + }, + 201, + ) + elif url == "https://localhost:8000/api/v1/projects/1/templates/": + return MockResponse( + { + "id": "1", + "url": "/projects/1/templates/1/", + "name": kwargs["json"]["name"], + }, + 201, + ) + elif url == "https://localhost:8000/api/v1/projects/1/parameters/2/values/": + return MockResponse( + { + "id": "1", + "url": "/projects/1/parameters/2/values/1/", + "internal_value": kwargs["json"]["internal_value"], + }, + 201, + ) diff --git a/src/tests/test_cli.py b/src/tests/test_cli.py index 783025f..fe461bd 100644 --- a/src/tests/test_cli.py +++ b/src/tests/test_cli.py @@ -1,16 +1,22 @@ from __future__ import annotations +import pathlib +import traceback +from unittest import mock from unittest import TestCase +import pytest from click.testing import CliRunner from dynamic_importer.main import import_config +from tests.fixtures.requests import mocked_requests_localhost_get +from tests.fixtures.requests import mocked_requests_localhost_post class TestCLI(TestCase): def test_cli_help(self): runner = CliRunner() result = runner.invoke(import_config, ["--help"]) - self.assertEqual(0, result.exit_code) + self.assertEqual(result.exit_code, 0) self.assertIn( """Usage: import-config [OPTIONS] COMMAND [ARGS]... @@ -20,8 +26,32 @@ def test_cli_help(self): result.output, ) - def test_cli_process_configs(self): + def test_process_configs_no_args(self): + runner = CliRunner() + result = runner.invoke(import_config, ["process-configs", "-t", "dotenv"]) + self.assertEqual(result.exit_code, 2) + self.assertIn( + "Error: At least one of --default-values and --env-values must be provided", + result.output, + ) + + def test_regenerate_template_no_args(self): runner = CliRunner() + result = runner.invoke( + import_config, ["regenerate-template", "-t", "dotenv", "-d", "test"] + ) + self.assertEqual(result.exit_code, 2) + self.assertIn( + "Error: At least one of --default-values and --env-values must be provided", + result.output, + ) + + +@pytest.mark.usefixtures("tmp_path") +def test_cli_process_configs_dotenv(tmp_path): + runner = CliRunner() + current_dir = pathlib.Path(__file__).parent.resolve() + with runner.isolated_filesystem(temp_dir=tmp_path) as td: result = runner.invoke( import_config, [ @@ -29,7 +59,160 @@ def test_cli_process_configs(self): "-t", "dotenv", "--default-values", - "samples/.env.sample", + f"{current_dir}/../samples/.env.sample", + "--output-dir", + td, + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + + +@pytest.mark.usefixtures("tmp_path") +def test_cli_process_configs_json(tmp_path): + runner = CliRunner() + current_dir = pathlib.Path(__file__).parent.resolve() + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + result = runner.invoke( + import_config, + [ + "process-configs", + "-t", + "json", + "--default-values", + f"{current_dir}/../samples/short.json", + "--output-dir", + td, + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + + +@pytest.mark.usefixtures("tmp_path") +def test_cli_process_configs_tf(tmp_path): + runner = CliRunner() + current_dir = pathlib.Path(__file__).parent.resolve() + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + result = runner.invoke( + import_config, + [ + "process-configs", + "-t", + "tf", + "--default-values", + f"{current_dir}/../samples/variables.tf", + "--output-dir", + td, + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + + +@pytest.mark.usefixtures("tmp_path") +def test_cli_process_configs_tfvars(tmp_path): + runner = CliRunner() + current_dir = pathlib.Path(__file__).parent.resolve() + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + result = runner.invoke( + import_config, + [ + "process-configs", + "-t", + "tfvars", + "--default-values", + f"{current_dir}/../samples/terraform.tfvars", + "--output-dir", + td, + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + + +@pytest.mark.usefixtures("tmp_path") +def test_cli_process_configs_yaml(tmp_path): + runner = CliRunner() + current_dir = pathlib.Path(__file__).parent.resolve() + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + result = runner.invoke( + import_config, + [ + "process-configs", + "-t", + "yaml", + "--default-values", + f"{current_dir}/../samples/azureTRE.yaml", + "--output-dir", + td, + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + + result = runner.invoke( + import_config, + [ + "regenerate-template", + "-t", + "yaml", + "--default-values", + f"{current_dir}/../samples/azureTRE.yaml", + "--data-file", + f"{td}/azureTRE.ctconfig", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + + +@mock.patch( + "dynamic_importer.api.client.requests.get", + side_effect=mocked_requests_localhost_get, +) +@mock.patch( + "dynamic_importer.api.client.requests.post", + side_effect=mocked_requests_localhost_post, +) +@pytest.mark.usefixtures("tmp_path") +def test_cli_import_data_json(mock_get, mock_post, tmp_path): + runner = CliRunner( + env={"CLOUDTRUTH_API_HOST": "localhost:8000", "CLOUDTRUTH_API_KEY": "test"} + ) + current_dir = pathlib.Path(__file__).parent.resolve() + with runner.isolated_filesystem(temp_dir=tmp_path) as td: + result = runner.invoke( + import_config, + [ + "process-configs", + "-t", + "json", + "--default-values", + f"{current_dir}/../samples/short.json", + "-o", + td, + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + + result = runner.invoke( + import_config, + [ + "create-data", + "-d", + f"{td}/short.ctconfig", + "-m", + f"{td}/short.cttemplate", + "-p", + "testproj", ], + catch_exceptions=False, ) - self.assertEqual(0, result.exit_code) + try: + assert result.exit_code == 0 + except AssertionError: + print(result.output) + print(traceback.format_tb(result.exc_info[2])) + raise From 705b408245162555d1f7d24475e58c6a94598645 Mon Sep 17 00:00:00 2001 From: Matthew Warren Date: Tue, 5 Mar 2024 10:33:02 -0500 Subject: [PATCH 4/4] add ci workflow --- .github/workflows/ci.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..2070c6d --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,26 @@ +name: Run tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.11 + uses: actions/setup-python@v2 + with: + python-version: 3.11 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + - name: Test with pytest + run: | + pytest