feat: Move K8s to dbt

.github/workflows/transformations_k8s_complience.yml

@@ -0,0 +1,61 @@
+name: Test K8s Policies
+
+on:
+  pull_request:
+    paths:
+      - "transformations/k8s_compliance/**"
+      - ".github/workflows/transformations_k8s_complience.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "transformations/k8s_compliance/**"
+      - ".github/workflows/transformations_k8s_complience.yml"
+
+jobs:
+  test-policies:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./transformations/k8s_compliance
+    services:
+      postgres:
+        image: postgres:11
+        env:
+          POSTGRES_PASSWORD: pass
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: "./transformations/k8s_compliance/requirements.txt"
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Setup CloudQuery
+        uses: cloudquery/setup-cloudquery@v3
+        with:
+          version: v3.26.1
+      - name: Migrate DB
+        run: cloudquery migrate tests/spec.yml
+        env:
+          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres  
+      - name: DBT dependencies
+        run: |
+          dbt deps --target dev --profiles-dir ./tests 
+      - name: Run Policies
+        run: |
+          dbt run --target dev --profiles-dir ./tests

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

to_migrate/k8s/cis_v1_7_0/controller_manager_1_3.sql

@@ -1,6 +1,6 @@
 \echo "Executing K8S Pod Security Standards CIS v1.7.0"
 
- \echo "Policies - Controller Manager"
+\echo "Policies - Controller Manager"
 
- \echo "Queries - Controller Manager"
+\echo "Queries - Controller Manager"
  \ir ../queries_cis_v1_7_0/controller_manager_1_3_queries.sql

to_migrate/k8s/cis_v1_7_0/general_policies_5_7.sql

@@ -1,6 +1,6 @@
 \echo "Executing K8S General Policies CIS v1.7.0"
 
- \echo "Policies - General Policies"
+\echo "Policies - General Policies"
 
- \echo "Queries - General Policies"
+\echo "Queries - General Policies"
  \ir ../queries_cis_v1_7_0/general_policies_5_7_queries.sql

to_migrate/k8s/cis_v1_7_0/network_policies_and_cni_5_3.sql

@@ -1,6 +1,6 @@
 \echo "Executing K8S Network Policies and CNI CIS v1.7.0"
 
- \echo "Policies - Network Policies and CNI"
+\echo "Policies - Network Policies and CNI"
 
- \echo "Queries - Network Policies and CNI"
+\echo "Queries - Network Policies and CNI"
  \ir ../queries_cis_v1_7_0/network_policies_and_cni_5_3_queries.sql

to_migrate/k8s/cis_v1_7_0/pod_security_standards_5_2.sql

@@ -1,6 +1,6 @@
 \echo "Executing K8S Pod Security Standards CIS v1.7.0"
 
- \echo "Policies - Pod Security Standards"
+\echo "Policies - Pod Security Standards"
 
- \echo "Queries - Pod Security Standards"
+\echo "Queries - Pod Security Standards"
  \ir ../queries_cis_v1_7_0/pod_security_standards_5_2_queries.sql

to_migrate/k8s/cis_v1_7_0/secrets_managment_5_4.sql

@@ -1,6 +1,6 @@
 \echo "Executing K8S Secrets Management CIS v1.7.0"
 
- \echo "Policies - Secrets Management"
+\echo "Policies - Secrets Management"
 
- \echo "Queries - Secrets Management"
+\echo "Queries - Secrets Management"
  \ir ../queries_cis_v1_7_0/secrets_managment_5_4_queries.sql

to_migrate/k8s/queries_cis_v1_7_0/api_server_queries.sql

@@ -1,4 +1,3 @@
--- query 1.2.1
 \echo "api_server_1.2.1"
 
 INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,

to_migrate/k8s/queries_cis_v1_7_0/controller_manager_1_3_queries.sql

@@ -1,5 +1,5 @@
 -- 1.3.1
- \echo "controller_manager_1.3.1"
+\echo "controller_manager_1.3.1"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)

to_migrate/k8s/queries_cis_v1_7_0/general_policies_5_7_queries.sql

@@ -1,7 +1,5 @@
--- 5.7.1 - TODO
-
 -- 5.7.2
- \echo "pod_security_standards_5.7.2"
+\echo "pod_security_standards_5.7.2"
 INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
 SELECT
@@ -26,7 +24,7 @@ FROM
   k8s_core_pods;
 
   -- 5.7.3
- \echo "pod_security_standards_5.7.3"
+\echo "pod_security_standards_5.7.3"
 INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
 WITH
@@ -67,7 +65,7 @@ FROM
   k8s_core_pods;
 
   -- 5.7.4
- \echo "pod_security_standards_5.7.4"
+\echo "pod_security_standards_5.7.4"
 INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
 -- only pods, talk with jason for more ressoures

to_migrate/k8s/queries_cis_v1_7_0/network_policies_and_cni_5_3_queries.sql

@@ -1,9 +1,5 @@
--- 5.3.1 - a simple check there is a network policy
--- SELECT *
--- FROM k8s_networking_network_policies;
-
 -- 5.3.2
- \echo "pod_security_standards_5.3.2"
+\echo "pod_security_standards_5.3.2"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)

to_migrate/k8s/queries_cis_v1_7_0/pod_security_standards_5_2_queries.sql

@@ -1,6 +1,5 @@
--- 5.2.1 TODO
 -- 5.2.2
- \echo "pod_security_standards_5.2.2"
+\echo "pod_security_standards_5.2.2"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -40,7 +39,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.3
- \echo "pod_security_standards_5.2.3"
+\echo "pod_security_standards_5.2.3"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -62,7 +61,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.4
- \echo "pod_security_standards_5.2.4"
+\echo "pod_security_standards_5.2.4"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -84,7 +83,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.5
- \echo "pod_security_standards_5.2.5"
+\echo "pod_security_standards_5.2.5"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -106,7 +105,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.6
- \echo "pod_security_standards_5.2.6"
+\echo "pod_security_standards_5.2.6"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -145,10 +144,9 @@ SELECT
 FROM
   k8s_core_pods;
 
--- 5.2.7 TODO
 
 -- 5.2.8
- \echo "pod_security_standards_5.2.8"
+\echo "pod_security_standards_5.2.8"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -193,7 +191,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.9
- \echo "pod_security_standards_5.2.9"
+\echo "pod_security_standards_5.2.9"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -236,7 +234,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.10
- \echo "pod_security_standards_5.2.10"
+\echo "pod_security_standards_5.2.10"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -279,7 +277,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.11
- \echo "pod_security_standards_5.2.11"
+\echo "pod_security_standards_5.2.11"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -320,7 +318,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.12
- \echo "pod_security_standards_5.2.12"
+\echo "pod_security_standards_5.2.12"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
@@ -368,7 +366,7 @@ FROM
   k8s_core_pods;
 
 -- 5.2.13
- \echo "pod_security_standards_5.2.13"
+\echo "pod_security_standards_5.2.13"
 
  INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)

to_migrate/k8s/queries_cis_v1_7_0/secrets_managment_5_4_queries.sql

@@ -1,5 +1,5 @@
 -- 5.4.1
- \echo "pod_security_standards_5.4.1"
+\echo "pod_security_standards_5.4.1"
 INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
 WITH
@@ -40,9 +40,11 @@ FROM
 
 
 -- 5.4.2
- \echo "pod_security_standards_5.4.2"
+\echo "pod_security_standards_5.4.2"
 INSERT INTO k8s_policy_results (resource_id, execution_time, framework, check_id, title, context, namespace,
                                  resource_name, status)
+
+
 WITH
   pod_containers
     AS (

transformations/k8s_compliance/dbt_packages/utils

@@ -0,0 +1 @@
+/Users/benbernays/Documents/GitHub/policies-premium/transformations/utils

transformations/k8s_compliance/dbt_project.yml

@@ -0,0 +1,40 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'k8s_compliance'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'k8s_compliance'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+models:
+  k8s_compliance:
+    # Config indicated by + and applies to all files under models/example/
+    # example:
+    #   +materialized: view
+
+
+
+