Improve YAML string creation cross-platform (Issue #3) (#4)

Author: robastel

.github/workflows/test.yml

@@ -25,11 +25,15 @@ jobs:
           black dbt_invoke tests setup.py --check --diff --color -S -l 79
   test:
     needs: lint
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.7, 3.x]
         dbt-version: [0.18.x, 0.19.x]
+        exclude:
+          - python-version: 3.x
+            dbt-version: 0.18.x
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -40,6 +44,18 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements/requirements_dbt_${{ matrix.dbt-version }}.txt
-      - name: Run tests
+      # There is more than one target in profiles.yml because
+      # dbt-sqlite for dbt~=0.18.x uses a different format than other
+      # versions.
+      - name: Run tests for dbt~=0.18.x
+        if: ${{ matrix.dbt-version == '0.18.x' }}
+        env:
+          TARGET: ${{ matrix.dbt-version }}
+        run: |
+          python tests/test.py
+      - name: Run tests for other dbt versions
+        if: ${{ matrix.dbt-version != '0.18.x' }}
+        env:
+          TARGET: default
         run: |
           python tests/test.py

README.md

@@ -189,5 +189,8 @@ dbt-invoke properties.delete <options>
   ephemeral materializations. 
   - This may be partially remedied by increasing the value of the `--threads` 
     option in `dbt-invoke properties.update`.
-- dbt-invoke was developed with Python 3.6 to 3.8 and dbt 0.18 to 0.19 in mind.
-  It has not been tested across different types of data warehouses.
+- dbt-invoke is tested against:
+  - Python 3.7 as well as the latest version Python 3.x at the time of merge
+  - dbt 0.18 and 0.19
+  - macos-latest, windows-latest, ubuntu-latest
+- dbt-invoke has not been tested across different types of data warehouses.

dbt_invoke/internal/_utils.py

@@ -2,6 +2,7 @@
 import logging
 from pathlib import Path
 import sys
+import platform
 
 import yaml
 
@@ -236,9 +237,20 @@ def dbt_run_operation(
     }
     dbt_cli_kwargs = get_cli_kwargs(**dbt_kwargs)
     macro_kwargs = json.dumps(kwargs, sort_keys=False)
+    if platform.system().lower().startswith('win'):
+        # Format YAML string for Windows Command Prompt
+        macro_kwargs = macro_kwargs.replace('"', '\\"')
+        macro_kwargs = macro_kwargs.replace('\\\\"', '"')
+        macro_kwargs = macro_kwargs.replace('>', '^>')
+        macro_kwargs = macro_kwargs.replace('<', '^<')
+        macro_kwargs = f'"{macro_kwargs}"'
+    else:
+        # Format YAML string for Mac/Linux (bash)
+        macro_kwargs = macro_kwargs.replace("'", """'"'"'""")
+        macro_kwargs = f"'{macro_kwargs}'"
     command = (
         f"dbt run-operation {dbt_cli_kwargs}"
-        f" {macro_name} --args '{macro_kwargs}'"
+        f" {macro_name} --args {macro_kwargs}"
     )
     logger.debug(f'Running command: {command}')
     result = ctx.run(command, hide=hide)

requirements/requirements_dbt_0.18.x.txt

@@ -1,6 +1,6 @@
 agate<1.6.2
 cryptography<3
-dbt~=0.18.0
+dbt-core~=0.18.0
 dbt-sqlite~=0.0.4
 invoke>=1.4.1
 PyYAML

requirements/requirements_dbt_0.19.x.txt

@@ -1,5 +1,5 @@
 agate<1.6.2
-dbt~=0.19.0
+dbt-core~=0.19.0
 dbt-sqlite~=0.1.0
 invoke>=1.4.1
 pytz<2021.0

tests/test_config.yml

@@ -69,37 +69,51 @@ expected_properties:
 expected_dbt_ls_results:
   resource_type:
     model:
-      - models/marts/core/customers.sql
-      - models/marts/core/orders.sql
+      - &customers
+        - models
+        - marts
+        - core
+        - customers.sql
+      - &orders
+        - models
+        - marts
+        - core
+        - orders.sql
     seed:
-      - data/items.csv
+      - &items
+        - data
+        - items.csv
     snapshot:
-      - snapshots/items_snapshot.sql
+      - &items_snapshot
+        - snapshots
+        - items_snapshot.sql
     analysis:
-      - analyses/revenue_by_daily_cohort.sql
+      - &revenue_by_daily_cohort
+        - analyses
+        - revenue_by_daily_cohort.sql
   select:
     marts.core:
-      - models/marts/core/customers.sql
-      - models/marts/core/orders.sql
+      - *customers
+      - *orders
   models:
     customers:
-      - models/marts/core/customers.sql
+      - *customers
     customers,orders:
-      - models/marts/core/customers.sql
-      - models/marts/core/orders.sql
+      - *customers
+      - *orders
   exclude:
     customers:
-      - models/marts/core/orders.sql
-      - data/items.csv
-      - snapshots/items_snapshot.sql
+      - *orders
+      - *items
+      - *items_snapshot
   selector:
     test_selector:
-      - models/marts/core/orders.sql
-      - snapshots/items_snapshot.sql
+      - *orders
+      - *items_snapshot
   '':
     '':
-      - models/marts/core/customers.sql
-      - models/marts/core/orders.sql
-      - data/items.csv
-      - snapshots/items_snapshot.sql
-      - analyses/revenue_by_daily_cohort.sql
+      - *customers
+      - *orders
+      - *items
+      - *items_snapshot
+      - *revenue_by_daily_cohort

tests/test_dbt_project/analyses/revenue_by_daily_cohort.sql

@@ -11,5 +11,7 @@ LEFT JOIN
     ON o.item_id = i.item_id
     AND o.order_at >= i.dbt_valid_from
     AND o.order_at < COALESCE(i.dbt_valid_to, CURRENT_TIMESTAMP)
+WHERE
+    o.order_at >= '1970-01-01 00:00:00'
 GROUP BY
     CAST(c.created_at AS DATE)

tests/test_dbt_project/profiles.yml

@@ -3,20 +3,31 @@ config:
 
 # Credit to https://github.com/codeforkjeff/dbt-sqlite
 dbt-sqlite:
-  target: dev
+  target: "{{ env_var('TARGET', 'default') }}"
   outputs:
-    dev:
-      type: sqlite
+    # There is more than one target because dbt-sqlite for dbt~=0.18.x
+    # uses a different format for schemas_and_paths
+    0.18.x:
+      type: &type sqlite
       # sqlite locks the whole db on writes so anything > 1 won't help
-      threads: 1
-      # value is arbitrary
-      database: "database"
-      # value of 'schema' must be defined in schema_paths below. in most cases,
-      # this should be 'main'
-      schema: 'main'
-      # connect schemas to paths: at least one of these must be 'main'
-      schemas_and_paths: 'main=test.db'
-      # directory where all *.db files are attached as schema, using base filename
-      # as schema name, and where new schema are created. this can overlap with the dirs of
-      # files in schemas_and_paths as long as there's no conflicts.
-      schema_directory: '.'
+      threads: &threads 1
+      # Value is arbitrary
+      database: &database database
+      # Value of 'schema' must be defined in schema_paths below.
+      # In most cases, this should be 'main'
+      schema: &schema main
+      # Connect schemas to paths: at least one of these must be 'main'
+      schemas_and_paths: main=test.db
+      # Directory where all *.db files are attached as schema, using
+      # base filename as schema name, and where new schema are created.
+      # This can overlap with the dirs of files in schemas_and_paths as
+      # long as there are no conflicts.
+      schema_directory: &schema_directory .
+    default:
+      type: *type
+      threads: *threads
+      database: *database
+      schema: *schema
+      schemas_and_paths:
+        main: test.db
+      schema_directory: *schema_directory

tests/test_utils.py

@@ -35,7 +35,7 @@ def test_dbt_ls(self):
         :return: None
         """
         for db_ls_kwarg, values in self.expected_dbt_ls_results.items():
-            for value, expected_result_lines in values.items():
+            for value, expected_result_parts in values.items():
                 dbt_ls_kwargs = {db_ls_kwarg: value}
                 result_lines = _utils.dbt_ls(
                     self.ctx,
@@ -45,7 +45,10 @@ def test_dbt_ls(self):
                     logger=self.logger,
                     **dbt_ls_kwargs,
                 )
-                self.assertCountEqual(result_lines, expected_result_lines)
+                result_parts = [
+                    list(Path(line).parts) for line in result_lines
+                ]
+                self.assertCountEqual(result_parts, expected_result_parts)
 
 
 if __name__ == '__main__':