fix for primary keys upper case (#1)

Co-authored-by: Jose Valenzuela <jvalenzuela@indeed.com>
jmriego · Jul 4, 2020 · e53ad01 · e53ad01
1 parent f02b9cd
commit e53ad01
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -122,15 +122,16 @@ data only after the change.
 1. Define environment variables that requires running the tests
 ```
   export GOOGLE_APPLICATION_CREDENTIALS=<credentials-json-file>
+  export TARGET_BIGQUERY_PROJECT=<bigquery project to run your tests on>
+  export TARGET_BIGQUERY_SCHEMA=<temporary schema for running the tests>
 ```
 
 2. Install python dependencies in a virtual env and run nose unit and integration tests
 ```
   python3 -m venv venv
   . venv/bin/activate
   pip install --upgrade pip
-  pip install .
-  pip install nose
+  pip install -e ".[test]"
 ```
 
 3. To run unit tests:

diff --git a/target_bigquery/db_sync.py b/target_bigquery/db_sync.py
@@ -379,7 +379,7 @@ def record_primary_key_string(self, record):
             return None
         flatten = flatten_record(record, max_level=self.data_flattening_max_level)
         try:
-            key_props = [str(flatten[p]) for p in self.stream_schema_message['key_properties']]
+            key_props = [str(flatten[p.lower()]) for p in self.stream_schema_message['key_properties']]
         except Exception as exc:
             logger.info("Cannot find {} primary key(s) in record: {}".format(self.stream_schema_message['key_properties'], flatten))
             raise exc

diff --git a/tests/integration/resources/messages-with-non-db-friendly-columns.json b/tests/integration/resources/messages-with-non-db-friendly-columns.json
@@ -1,11 +1,11 @@
 {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_non_db_friendly_columns"}}
-{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "camelcaseColumn": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "minus-column": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]}
+{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "schema": {"properties": {"C_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "camelcaseColumn": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "minus-column": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["C_pk"]}
 {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "version": 1}
-{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"c_pk": 1, "camelcaseColumn": "Dummy row 1", "minus-column": "Dummy row 1"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
-{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"c_pk": 2, "camelcaseColumn": "Dummy row 2", "minus-column": "Dummy row 2"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
-{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"c_pk": 3, "camelcaseColumn": "Dummy row 3", "minus-column": "Dummy row 3"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
-{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"c_pk": 4, "camelcaseColumn": "Dummy row 4", "minus-column": "Dummy row 4"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
-{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"c_pk": 5, "camelcaseColumn": "Dummy row 5", "minus-column": "Dummy row 5"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
+{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"C_pk": 1, "camelcaseColumn": "Dummy row 1", "minus-column": "Dummy row 1"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
+{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"C_pk": 2, "camelcaseColumn": "Dummy row 2", "minus-column": "Dummy row 2"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
+{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"C_pk": 3, "camelcaseColumn": "Dummy row 3", "minus-column": "Dummy row 3"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
+{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"C_pk": 4, "camelcaseColumn": "Dummy row 4", "minus-column": "Dummy row 4"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
+{"type": "RECORD", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "record": {"C_pk": 5, "camelcaseColumn": "Dummy row 5", "minus-column": "Dummy row 5"}, "version": 1, "time_extracted": "2019-01-31T15:51:50.215998Z"}
 {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_non_db_friendly_columns", "bookmarks": {"tap_mysql_test-test_table_one": {"initial_full_table_complete": true}, "tap_mysql_test-test_table_two": {"initial_full_table_complete": true}}}}
 {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_non_db_friendly_columns", "version": 1}
 {"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"tap_mysql_test-test_table_non_db_friendly_columns": {"initial_full_table_complete": true}}}}