Merge pull request #2 from kgmcquate/develop

Added multiple test generation, more tests, and many bugfixes

.github/workflows/dbt_test.yml

@@ -12,10 +12,10 @@ defaults:
     working-directory: integration_tests
 
 jobs:
-  test:
-    name: Test
+  test-duckdb:
+    name: Test on DuckDB
     runs-on: ubuntu-latest
-    environment: production
+    #environment: production
     container:
       image: python:3.11
 
@@ -27,11 +27,46 @@ jobs:
       run: pip install dbt-core duckdb dbt-duckdb
 
     - name: Install DBT deps
-      run: dbt deps
+      run: dbt deps --target duckdb
 
     - name: load test data
-      run: dbt seed
+      run: dbt seed --target duckdb
 
     - name: dbt test
-      run: dbt test
+      run: dbt test --target duckdb
+
+  test-postgres:
+    name: Test on Postgres
+    runs-on: ubuntu-latest
+    #environment: production
+    container:
+      image: python:3.11
+
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: pip install
+      run: pip install dbt-core duckdb dbt-postgres
+
+    - name: Install DBT deps
+      run: dbt deps --target postgres
+
+    - name: load test data
+      run: dbt seed --target postgres
+
+    - name: dbt test
+      run: dbt test --target postgres
+
+
 

integration_tests/dbt_project.yml

@@ -17,4 +17,4 @@ clean-targets:
 
 seeds:
   +schema: integration_test_data
-  +quote_columns: false
+  +quote_columns: true

integration_tests/profiles.yml

@@ -17,4 +17,11 @@ integration_tests:
     duckdb:
       type: duckdb
       path: 'data/duckdb/integration_test_data.duckdb'
-
+    postgres:
+      type: postgres
+      host: postgres
+      user: postgres
+      password: postgres
+      port: 5432
+      dbname: postgres # or database instead of dbname
+      schema: public

integration_tests/run_docker.sh

@@ -1 +1,4 @@
-docker run -it -v ${PWD}:/run/dbt-testgen dbt-testgen-integration-test bash
+docker run -it -v ${PWD}:/run/dbt-testgen dbt-testgen-integration-test bash
+
+
+docker run -it -v ${PWD}:/run/dbt-testgen python:3.11 bash

integration_tests/seeds/colnames_with_spaces.csv

@@ -0,0 +1,4 @@
+First Name,Age (Years),Current City
+John,25,New York
+Alice,30,San Francisco
+Bob,22,Chicago

integration_tests/tests/generate_accepted_values_tests/accepted_values_colnames_with_spaces.sql

@@ -0,0 +1,40 @@
+
+
+{% set actual_yaml = toyaml(fromjson(tojson(
+        testgen.get_accepted_values_test_suggestions(
+            ref('colnames_with_spaces')
+        )
+    )))
+%}
+
+{% set expected_yaml %}
+models:
+- name: colnames_with_spaces
+  columns:
+  - name: First Name
+    description: Accepted values test generated by dbt-testgen
+    tests:
+    - accepted_values:
+        values:
+        - Alice
+        - Bob
+        - John
+  - name: Age (Years)
+    description: Accepted values test generated by dbt-testgen
+    tests:
+    - accepted_values:
+        values:
+        - '22'
+        - '25'
+        - '30'
+  - name: Current City
+    description: Accepted values test generated by dbt-testgen
+    tests:
+    - accepted_values:
+        values:
+        - Chicago
+        - New York
+        - San Francisco
+{% endset %}
+
+{{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}

integration_tests/tests/generate_range_tests/range_colnames_with_spaces.sql

@@ -0,0 +1,22 @@
+
+
+{% set actual_yaml = toyaml(fromjson(tojson(
+        testgen.get_range_test_suggestions(
+            ref('colnames_with_spaces')
+        )
+    )))
+%}
+
+{% set expected_yaml %}
+models:
+- name: colnames_with_spaces
+  columns:
+  - name: Age (Years)
+    description: Numeric range test generated by dbt-testgen
+    tests:
+    - dbt_utils.accepted_range:
+        min_value: 22
+        max_value: 30
+{% endset %}
+
+{{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}

integration_tests/tests/generate_uniqueness_tests/uniqueness_users.sql

@@ -0,0 +1,32 @@
+
+
+{% set actual_yaml = toyaml(fromjson(tojson(
+        testgen.get_uniqueness_test_suggestions(
+            ref('users'),
+            compound_key_length = 1
+        )
+    )))
+%}
+
+{% set expected_yaml %}
+models:
+- name: users
+  columns:
+  - name: user_id
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+  - name: username
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+  - name: email
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+{% endset %}
+
+{{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}

integration_tests/tests/test_generate_uniqueness_tests.sql → integration_tests/tests/merge_dbt_configs/merge_dbt_configs_1.sql

@@ -1,12 +1,4 @@
-
-
-{% set actual_yaml = testgen.print_uniqueness_test_suggestions(
-        ref('users'),
-        compound_key_length = 1
-    )
-%}
-
-{% set expected_yaml %}
+{% set input_yaml_1 %}
 models:
 - name: users
   tests: []
@@ -20,7 +12,14 @@ models:
     - not_null:
         tags:
         - uniqueness
-  - name: username
+{% endset %}
+
+{% set input_yaml_2 %}
+models:
+- name: users
+  tests: []
+  columns:
+  - name: email
     description: Uniqueness test generated by dbt-testgen
     tests:
     - unique:
@@ -29,8 +28,30 @@ models:
     - not_null:
         tags:
         - uniqueness
+{% endset %}
+
+{% set actual_yaml = toyaml(
+        testgen.merge_dbt_configs(
+            fromyaml(input_yaml_1), 
+            fromyaml(input_yaml_2)
+        ) 
+    )
+%}
+
+
+{% set expected_yaml %}
+models:
+- name: users
+  columns:
+  - name: user_id
+    tests:
+    - unique:
+        tags:
+        - uniqueness
+    - not_null:
+        tags:
+        - uniqueness
   - name: email
-    description: Uniqueness test generated by dbt-testgen
     tests:
     - unique:
         tags:

integration_tests/tests/merge_dbt_configs/merge_dbt_configs_2.sql

@@ -0,0 +1,58 @@
+{% set input_yaml_1 %}
+models:
+- name: users
+  tests: []
+  columns:
+  - name: user_id
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - unique:
+        tags:
+        - uniqueness
+    - not_null:
+        tags:
+        - uniqueness
+{% endset %}
+
+{% set input_yaml_2 %}
+models:
+- name: users
+  tests: []
+  columns:
+  - name: user_id
+    description: Uniqueness test generated by dbt-testgen
+    tests:
+    - accepted_values:
+        values:
+        - active
+        - inactive
+{% endset %}
+
+{% set actual_yaml = toyaml(
+        testgen.merge_dbt_configs(
+            fromyaml(input_yaml_1), 
+            fromyaml(input_yaml_2)
+        ) 
+    )
+%}
+
+
+{% set expected_yaml %}
+models:
+- name: users
+  columns:
+  - name: user_id
+    tests:
+    - unique:
+        tags:
+        - uniqueness
+    - not_null:
+        tags:
+        - uniqueness
+    - accepted_values:
+        values:
+        - active
+        - inactive
+{% endset %}
+
+{{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}

integration_tests/tests/test_test_selection/test_selection_users.sql

@@ -0,0 +1,42 @@
+
+
+{% set actual_yaml = toyaml(fromjson(tojson(
+        testgen.get_test_suggestions(
+            ref('users')
+        )
+    )))
+%}
+
+{% set expected_yaml %}
+models:
+- name: users
+  columns:
+  - name: user_id
+    tests:
+    - unique
+    - not_null
+    - dbt_utils.accepted_range:
+        min_value: 1
+        max_value: 30
+  - name: username
+    tests:
+    - unique
+    - not_null
+  - name: email
+    tests:
+    - unique
+    - not_null
+  - name: user_status
+    tests:
+    - accepted_values:
+        values:
+        - active
+        - inactive
+  - name: age
+    tests:
+    - dbt_utils.accepted_range:
+        min_value: 22
+        max_value: 35
+{% endset %}
+
+{{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}