sc-12482 add tests and testing framework

.github/workflows/ci.yaml

@@ -0,0 +1,26 @@
+name: Run tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.11
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[dev]
+    - name: Test with pytest
+      run: |
+        pytest

.pre-commit-config.yaml

@@ -2,14 +2,16 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+    rev: v4.5.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: check-added-large-files
     -   id: debug-statements
     -   id: name-tests-test
+        exclude: ^src/tests/fixtures/
+        args: ["--pytest-test-first"]
     -   id: requirements-txt-fixer
 -   repo: https://github.com/asottile/reorder-python-imports
     rev: v3.12.0
@@ -18,7 +20,7 @@ repos:
             exclude: ^(pre_commit/resources/|testing/resources/python3_hooks_repo/)
             args: [--py39-plus, --add-import, 'from __future__ import annotations']
 -   repo: https://github.com/ambv/black
-    rev: 22.3.0
+    rev: 24.2.0
     hooks:
     -   id: black
         language_version: python3.11

pyproject.toml

@@ -19,10 +19,22 @@ dev = [
     "pre-commit",
     "mypy",
     "pytest",
+    "pytest-black",
     "pytest-cov",
+    "pytest-mypy",
+    "types-requests",
+    "types-pyyaml",
 ]
 [project.scripts]
 cloudtruth-dynamic-importer = "dynamic_importer.main:import_config"
 
-[tool.setuptools]
-packages = ["dynamic_importer"]
+[tool.mypy]
+packages = "dynamic_importer"
+
+[tool.pytest.ini_options]
+addopts = ["--import-mode=importlib", "--cov=dynamic_importer", "--cov-report=term-missing", "--cov-report=xml", "--mypy"]
+minversion = 6.0
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["dynamic_importer", "tests"]

dynamic_importer/api/client.py → src/dynamic_importer/api/client.py

@@ -6,7 +6,6 @@
 from typing import Optional
 
 import requests
-
 from dynamic_importer.api.exceptions import ResourceNotFoundError
 
 DEFAULT_API_HOST = "api.cloudtruth.io"
@@ -139,14 +138,6 @@ def get_template(self, project_name: str, template_name: str) -> Dict:
         except KeyError:
             raise ResourceNotFoundError(f"Template {template_name} not found")
 
-    def _populate_type_cache(self) -> None:
-        types = self._make_request("types", "GET")
-        for ct_type in types["results"]:
-            self.cache["types"][ct_type["name"]] = {
-                "url": ct_type["url"],
-                "id": ct_type["id"],
-            }
-
     def get_value(
         self, project_name: str, parameter_name: str, environment_name: str
     ) -> Dict:
@@ -173,14 +164,22 @@ def get_value(
         except KeyError:
             raise ResourceNotFoundError(f"Parameter {parameter_name} not found")
 
+    def _populate_type_cache(self) -> None:
+        types = self._make_request("types", "GET")
+        for ct_type in types["results"]:
+            self.cache["types"][ct_type["name"]] = {
+                "url": ct_type["url"],
+                "id": ct_type["id"],
+            }
+
     def get_type_id(self, type_name: str) -> str:
         if type_name in self.cache["types"].keys():
             return self.cache["types"][type_name]["id"]
         self._populate_type_cache()
         try:
             return self.cache["types"][type_name]["id"]
         except KeyError:
-            raise ValueError(f"Type {type_name} not found")
+            raise ResourceNotFoundError(f"Type {type_name} not found")
 
     def get_type_url(self, type_name: str) -> str:
         if type_name in self.cache["types"].keys():
@@ -189,7 +188,7 @@ def get_type_url(self, type_name: str) -> str:
         try:
             return self.cache["types"][type_name]["url"]
         except KeyError:
-            raise ValueError(f"Type {type_name} not found")
+            raise ResourceNotFoundError(f"Type {type_name} not found")
 
     def create_project(self, name: str, description: str = "") -> Dict:
         resp = self._make_request(
@@ -320,6 +319,25 @@ def update_value(
             data={"environment": environment_id, "internal_value": value},
         )
 
+    def update_parameter(
+        self,
+        project_name: str,
+        parameter_id: str,
+        name: str,
+        description: str = "",
+        type_name: str = "string",
+    ) -> Dict:
+        project_id = self.get_project_id(project_name)
+        return self._make_request(
+            f"projects/{project_id}/parameters/{parameter_id}",
+            "PATCH",
+            data={
+                "name": name,
+                "description": description,
+                "type": type_name,
+            },
+        )
+
     def update_template(
         self,
         project_name: str,
@@ -352,7 +370,10 @@ def upsert_parameter(
             self.create_project(project_name)
 
         try:
-            return self.get_parameter(project_name, name)
+            parameter_id = self.get_parameter_id(project_name, name)
+            return self.update_parameter(
+                project_name, parameter_id, name, description, type_name
+            )
         except ResourceNotFoundError:
             return self.create_parameter(
                 project_name, name, description, type_name, secret, create_dependencies

dynamic_importer/main.py → src/dynamic_importer/main.py

@@ -5,7 +5,6 @@
 
 import click
 import urllib3
-
 from dynamic_importer.api.client import CTClient
 from dynamic_importer.processors import BaseProcessor
 from dynamic_importer.processors import get_processor_class

dynamic_importer/processors/__init__.py → src/dynamic_importer/processors/__init__.py

@@ -52,7 +52,7 @@ class BaseProcessor:
     values = None
     template: Dict = {}
 
-    def __init__(self, env_values: Dict):
+    def __init__(self, env_values: Dict) -> None:
         raise NotImplementedError("Subclasses must implement the __init__ method")
 
     def guess_type(self, value):

dynamic_importer/processors/dotenv.py → src/dynamic_importer/processors/dotenv.py

@@ -1,18 +1,22 @@
 from __future__ import annotations
 
+import os
 from re import sub
 from typing import Dict
 from typing import Optional
 
 from dotenv import dotenv_values  # type: ignore[import-not-found]
 from dotenv.main import DotEnv  # type: ignore[import-not-found]
-
 from dynamic_importer.processors import BaseProcessor
 
 
 class DotEnvProcessor(BaseProcessor):
     def __init__(self, env_values: Dict) -> None:
         for env, file_path in env_values.items():
+            if not os.path.isfile(file_path):
+                raise ValueError(
+                    f"Path to environment values file {file_path} could not be accessed."
+                )
             self.raw_data[env] = dotenv_values(file_path)
 
     def encode_template_references(

dynamic_importer/processors/json.py → src/dynamic_importer/processors/json.py

@@ -9,7 +9,10 @@
 
 
 class JSONProcessor(BaseProcessor):
-    def __init__(self, env_values: Dict):
+    def __init__(self, env_values: Dict) -> None:
+        # the click test library seems to reuse Processor classes somehow
+        # so we reset self.parameters_and_values to avoid test pollution
+        self.parameters_and_values: Dict = {}
         for env, file_path in env_values.items():
             with open(file_path, "r") as fp:
                 try:
@@ -25,9 +28,12 @@ def encode_template_references(
         template_body = json.dumps(template, indent=4)
         if config_data:
             for _, data in config_data.items():
-                if data["type"] != "string":
-                    # JSON strings use double quotes
-                    reference = rf'"(\{{\{{\s+cloudtruth.parameters.{data["param_name"]}\s+\}}\}})"'
-                    template_body = sub(reference, r"\1", template_body)
+                try:
+                    if data["type"] != "string":
+                        # JSON strings use double quotes
+                        reference = rf'"(\{{\{{\s+cloudtruth.parameters.{data["param_name"]}\s+\}}\}})"'
+                        template_body = sub(reference, r"\1", template_body)
+                except KeyError:
+                    raise RuntimeError(f"data: {data}")
 
         return template_body

dynamic_importer/processors/tf.py → src/dynamic_importer/processors/tf.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from re import sub
 from typing import Any
 from typing import Dict
@@ -9,7 +10,6 @@
 from typing import Union
 
 import hcl2
-
 from dynamic_importer.processors import BaseProcessor
 
 
@@ -18,6 +18,10 @@ class TFProcessor(BaseProcessor):
 
     def __init__(self, env_values: Dict) -> None:
         for env, file_path in env_values.items():
+            if not os.path.isfile(file_path):
+                raise ValueError(
+                    f"Path to environment values file {file_path} could not be accessed."
+                )
             try:
                 with open(file_path, "r") as fp:
                     # hcl2 does not support dumping to a string/file,
@@ -37,7 +41,7 @@ def encode_template_references(
         environment = "default"
         if config_data:
             for _, data in config_data.items():
-                value = data["values"][environment]
+                value = str(data["values"][environment])
                 reference = f'{{{{ cloudtruth.parameters.{data["param_name"]} }}}}'
                 template_body = sub(value, reference, template_body)
 

dynamic_importer/processors/tfvars.py → src/dynamic_importer/processors/tfvars.py

@@ -5,7 +5,6 @@
 from typing import Optional
 
 import hcl2
-
 from dynamic_importer.processors import BaseProcessor
 
 
@@ -32,8 +31,8 @@ def encode_template_references(
         environment = "default"
         if config_data:
             for _, data in config_data.items():
-                value = data["values"][environment]
-                reference = f'{{{{ cloudtruth.parameters.{data["param_name"]} }}}}'
+                value = str(data["values"][environment])
+                reference = rf'{{{{ cloudtruth.parameters.{data["param_name"]} }}}}'
                 template_body = sub(value, reference, template_body)
 
         return template_body

dynamic_importer/processors/yaml.py → src/dynamic_importer/processors/yaml.py

@@ -5,7 +5,6 @@
 from typing import Optional
 
 import yaml
-
 from dynamic_importer.processors import BaseProcessor
 
 

dynamic_importer/samples/.env.sample → src/samples/.env.sample

@@ -12,4 +12,4 @@ SECRET_HASH="something-with-a-#-hash"
 SMTP_SERVER="smtp.example.com"
 SMTP_PORT=587
 SMTP_USER="fedex"
-SMTP_PASSWORD="itsasecrettoeverybody"
+SMTP_PASSWORD="itsasecrettoeverybody"

dynamic_importer/samples/azureTRE.yaml → src/samples/azureTRE.yaml

@@ -75,4 +75,4 @@ developer_settings:
 
 # Used by the API and Resource processor application to change log level
 # Can be "ERROR", "WARNING", "INFO", "DEBUG"
-  logging_level: "INFO"
+  logging_level: "INFO"

dynamic_importer/samples/short.json → src/samples/short.json

@@ -16,4 +16,4 @@
             ]
         ]
     }
-}
+}

dynamic_importer/samples/terraform.tfvars → src/samples/terraform.tfvars

@@ -25,4 +25,4 @@ instance_ami = "ami-0123456789abcdef0"
 instance_type = "t2.micro"
 
 # Security group IDs for the EC2 instance (comma-separated list)
-security_group_ids = "sg-12345678,sg-87654321"
+security_group_ids = "sg-12345678,sg-87654321"

dynamic_importer/samples/variables.tf → src/samples/variables.tf

@@ -50,4 +50,4 @@ variable "security_group_ids" {
   description = "Security group IDs for the EC2 instance (comma-separated list)"
   type        = string
   default     = "sg-12345678,sg-87654321"
-}
+}