diff --git a/.circleci/config.yml b/.circleci/config.yml index 0560ac2..ae18af8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:tap-tester-v4 + - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester steps: - checkout - run: @@ -12,25 +12,31 @@ jobs: source /usr/local/share/virtualenvs/tap-activecampaign/bin/activate pip install -U 'pip<19.2' setuptools pip install .[dev] + - run: + name: 'JSON Validator' + command: | + source /usr/local/share/virtualenvs/tap-tester/bin/activate + stitch-validate-json tap_activecampaign/schemas/*.json + - run: + name: 'pylint' + command: | + source /usr/local/share/virtualenvs/tap-activecampaign/bin/activate + pylint tap_activecampaign -d C,R,W - add_ssh_keys - run: name: 'Integration Tests' command: | - aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/sandbox dev_env.sh + aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh source dev_env.sh source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-activecampaign \ - --target=target-stitch \ - --orchestrator=stitch-orchestrator \ - --email=harrison+sandboxtest@stitchdata.com \ - --password=$SANDBOX_PASSWORD \ - --client-id=50 \ - tests + run-test --tap=tap-activecampaign tests workflows: version: 2 commit: jobs: - - build + - build: + context: + - circleci-user build_daily: triggers: - schedule: diff --git a/tap_activecampaign/schemas/automation_blocks.json b/tap_activecampaign/schemas/automation_blocks.json index bfb38de..d299d6f 100644 --- a/tap_activecampaign/schemas/automation_blocks.json +++ b/tap_activecampaign/schemas/automation_blocks.json @@ -12,9 +12,21 @@ "type": ["null", "integer"] }, "params": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": {} + "anyOf": [ + { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + } + ] }, "deleted": { "type": ["null", "integer"] diff --git a/tap_activecampaign/schemas/brandings.json b/tap_activecampaign/schemas/brandings.json index cf96f32..3eba1fe 100644 --- a/tap_activecampaign/schemas/brandings.json +++ b/tap_activecampaign/schemas/brandings.json @@ -2,7 +2,7 @@ "type": "object", "additionalProperties": false, "properties": { - "group_id": { + "groupid": { "type": ["null", "string"] }, "site_name": { @@ -38,13 +38,13 @@ "help": { "type": ["null", "integer"] }, - "admin_template_httm": { + "admin_template_htm": { "type": ["null", "string"] }, "admin_template_css": { "type": ["null", "string"] }, - "public_template_httm": { + "public_template_htm": { "type": ["null", "string"] }, "public_template_css": { diff --git a/tap_activecampaign/schemas/campaign_links.json b/tap_activecampaign/schemas/campaign_links.json index 71277d6..8cefd9a 100644 --- a/tap_activecampaign/schemas/campaign_links.json +++ b/tap_activecampaign/schemas/campaign_links.json @@ -20,7 +20,7 @@ "tracked": { "type": ["null", "integer"] }, - "uniqueclicks": { + "uniquelinkclicks": { "type": ["null", "integer"] }, "linkclicks": { diff --git a/tap_activecampaign/schemas/contact_custom_fields.json b/tap_activecampaign/schemas/contact_custom_fields.json index bd48f0f..b4d1f56 100644 --- a/tap_activecampaign/schemas/contact_custom_fields.json +++ b/tap_activecampaign/schemas/contact_custom_fields.json @@ -17,7 +17,7 @@ "perstag": { "type": ["null", "string"] }, - "defvalue": { + "defval": { "type": ["null", "string"] }, "show_in_list": { diff --git a/tap_activecampaign/schemas/contact_lists.json b/tap_activecampaign/schemas/contact_lists.json index 6cf32e1..574af4f 100644 --- a/tap_activecampaign/schemas/contact_lists.json +++ b/tap_activecampaign/schemas/contact_lists.json @@ -46,7 +46,7 @@ "last_name": { "type": ["null", "string"] }, - "ip4_sub": { + "ip_4sub": { "type": ["null", "integer"] }, "sourceid": { @@ -58,7 +58,7 @@ "ip4_last": { "type": ["null", "integer"] }, - "ip4_unsub": { + "ip_4unsub": { "type": ["null", "integer"] }, "created_timestamp": { diff --git a/tap_activecampaign/schemas/contacts.json b/tap_activecampaign/schemas/contacts.json index 627a54d..b1b1c1d 100644 --- a/tap_activecampaign/schemas/contacts.json +++ b/tap_activecampaign/schemas/contacts.json @@ -15,7 +15,7 @@ "last_name": { "type": ["null", "string"] }, - "org_id": { + "orgid": { "type": ["null", "integer"] }, "segmentio_id": { diff --git a/tap_activecampaign/schemas/conversions.json b/tap_activecampaign/schemas/conversions.json index fd7e977..e24afc6 100644 --- a/tap_activecampaign/schemas/conversions.json +++ b/tap_activecampaign/schemas/conversions.json @@ -14,7 +14,7 @@ "limit": { "type": ["null", "integer"] }, - "enforce_limit": { + "enforcelimit": { "type": ["null", "integer"] }, "cdate": { diff --git a/tap_activecampaign/schemas/deal_activities.json b/tap_activecampaign/schemas/deal_activities.json index 30f4e65..9aee44d 100644 --- a/tap_activecampaign/schemas/deal_activities.json +++ b/tap_activecampaign/schemas/deal_activities.json @@ -5,7 +5,7 @@ "d_id": { "type": ["null", "integer"] }, - "d_stage_id": { + "d_stageid": { "type": ["null", "integer"] }, "userid": { @@ -20,7 +20,7 @@ "data_action": { "type": ["null", "string"] }, - "data_old_val": { + "data_oldval": { "type": ["null", "string"] }, "cdate": { diff --git a/tap_activecampaign/schemas/deal_stages.json b/tap_activecampaign/schemas/deal_stages.json index 3adcfea..f13ea37 100644 --- a/tap_activecampaign/schemas/deal_stages.json +++ b/tap_activecampaign/schemas/deal_stages.json @@ -2,19 +2,19 @@ "type": "object", "additionalProperties": false, "properties": { - "card_region_1": { + "card_region1": { "type": ["null", "string"] }, - "card_region_2": { + "card_region2": { "type": ["null", "string"] }, - "card_region_3": { + "card_region3": { "type": ["null", "string"] }, - "card_region_4": { + "card_region4": { "type": ["null", "string"] }, - "card_region_5": { + "card_region5": { "type": ["null", "string"] }, "cdate": { diff --git a/tap_activecampaign/schemas/forms.json b/tap_activecampaign/schemas/forms.json index 7dab581..df741f2 100644 --- a/tap_activecampaign/schemas/forms.json +++ b/tap_activecampaign/schemas/forms.json @@ -171,7 +171,7 @@ "cfields": { "anyOf": [ { - "type": "null" + "type": ["null", "object"] }, { "type": "array", @@ -228,7 +228,7 @@ "defaultscreenshot": { "type": ["null", "string"] }, - "recents": { + "recent": { "anyOf": [ { "type": "array", diff --git a/tap_activecampaign/schemas/tasks.json b/tap_activecampaign/schemas/tasks.json index 09d3b8a..ae6d1ff 100644 --- a/tap_activecampaign/schemas/tasks.json +++ b/tap_activecampaign/schemas/tasks.json @@ -28,7 +28,7 @@ "reltype": { "type": ["null", "string"] }, - "deal_task_type": { + "deal_tasktype": { "type": ["null", "integer"] }, "cdate": { diff --git a/tap_activecampaign/schemas/templates.json b/tap_activecampaign/schemas/templates.json index 30a9844..5ddb425 100644 --- a/tap_activecampaign/schemas/templates.json +++ b/tap_activecampaign/schemas/templates.json @@ -17,7 +17,7 @@ "subject": { "type": ["null", "string"] }, - "contant": { + "content": { "type": ["null", "string"] }, "categoryid": { diff --git a/tap_activecampaign/schemas/users.json b/tap_activecampaign/schemas/users.json index 5b0beab..32eb9c2 100644 --- a/tap_activecampaign/schemas/users.json +++ b/tap_activecampaign/schemas/users.json @@ -20,7 +20,7 @@ "signature": { "type": ["null", "string"] }, - "local_zone_id": { + "local_zoneid": { "type": ["null", "string"] }, "password_updated_utc_timestamp": { diff --git a/tap_activecampaign/transform.py b/tap_activecampaign/transform.py index 6a82653..9641ed8 100644 --- a/tap_activecampaign/transform.py +++ b/tap_activecampaign/transform.py @@ -25,6 +25,6 @@ def transform_json(this_json, stream_name, data_key): else: converted_json = humps.decamelize(this_json) - fixed_records = fix_records(converted_json) + fix_records(converted_json) return converted_json diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 0000000..008c1a3 --- /dev/null +++ b/tests/base.py @@ -0,0 +1,530 @@ +import unittest +import os +import tap_tester.connections as connections +import tap_tester.menagerie as menagerie +import tap_tester.runner as runner +from datetime import datetime as dt +from datetime import timedelta +import dateutil.parser +import pytz + +class ActiveCampaignTest(unittest.TestCase): + start_date = "" + + START_DATE_FORMAT = "%Y-%m-%dT00:00:00Z" + PRIMARY_KEYS = "table-key-properties" + REPLICATION_METHOD = "forced-replication-method" + REPLICATION_KEYS = "valid-replication-keys" + FULL_TABLE = "FULL_TABLE" + INCREMENTAL = "INCREMENTAL" + OBEYS_START_DATE = "obey-start-date" + + def tap_name(self): + return "tap-activecampaign" + + def setUp(self): + required_env = { + "TAP_ACTIVECAMPAIGN_API_TOKEN", + "TAP_ACTIVECAMPAIGN_API_URL", + } + missing_envs = [v for v in required_env if not os.getenv(v)] + if missing_envs: + raise Exception("set " + ", ".join(missing_envs)) + + def get_type(self): + return "platform.activecampaign" + + def get_credentials(self): + return { + 'api_url': os.getenv('TAP_ACTIVECAMPAIGN_API_URL'), + 'api_token': os.getenv('TAP_ACTIVECAMPAIGN_API_TOKEN') + } + + def get_properties(self, original: bool = True): + return_value = { + "start_date" : "2021-11-10T00:00:00Z", + } + if original: + return return_value + + # Reassign start date + return_value["start_date"] = self.start_date + return return_value + + def expected_metadata(self): + return { + 'accounts': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'account_contacts': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'account_custom_fields': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'account_custom_field_values': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'addresses': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'automations': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'brandings': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'calendars': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'campaigns': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'campaign_links': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'contacts': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'contact_automations': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'lastdate'}, + self.OBEYS_START_DATE: True + }, + 'contact_custom_fields': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'contact_custom_field_options': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'contact_custom_field_rels': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'contact_custom_field_values': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'contact_deals': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'deal_stages': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'deal_groups': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'deal_custom_fields': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'deal_custom_field_values': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'deals': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'ecommerce_connections': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'ecommerce_customers': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'tstamp'}, + self.OBEYS_START_DATE: True + }, + 'ecommerce_orders': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_date'}, + self.OBEYS_START_DATE: True + }, + 'ecommerce_order_products': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'forms': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'groups': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'lists': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'messages': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'saved_responses': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'scores': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'segments': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'tags': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'task_types': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'tasks': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'templates': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'users': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'webhooks': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'activities': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'tstamp'}, + self.OBEYS_START_DATE: True + }, + 'automation_blocks': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'mdate'}, + self.OBEYS_START_DATE: True + }, + 'bounce_logs': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'campaign_lists': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'campaign_messages': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'configs': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'contact_data': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'tstamp'}, + self.OBEYS_START_DATE: True + }, + 'contact_emails': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'sdate'}, + self.OBEYS_START_DATE: True + }, + 'contact_lists': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'contact_tags': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_timestamp'}, + self.OBEYS_START_DATE: True + }, + 'contact_conversions': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'cdate'}, + self.OBEYS_START_DATE: True + }, + 'conversions': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'conversions': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'conversion_triggers': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'udate'}, + self.OBEYS_START_DATE: True + }, + 'deal_activities': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'cdate'}, + self.OBEYS_START_DATE: True + }, + 'deal_group_users': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'ecommerce_order_activities': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'updated_date'}, + self.OBEYS_START_DATE: True + }, + 'email_activities': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'tstamp'}, + self.OBEYS_START_DATE: True + }, + 'goals': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False + }, + 'site_messages': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'ldate'}, + self.OBEYS_START_DATE: True + }, + 'sms': { + self.PRIMARY_KEYS: {'id'}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {'tstamp'}, + self.OBEYS_START_DATE: True + } + } + + + def expected_check_streams(self): + return set(self.expected_metadata().keys()) + + def expected_primary_keys(self): + return {table: properties.get(self.PRIMARY_KEYS, set()) for table, properties + in self.expected_metadata().items()} + + def expected_automatic_fields(self): + """return a dictionary with key of table name and set of value of automatic(primary key and bookmark field) fields""" + auto_fields = {} + for k, v in self.expected_metadata().items(): + auto_fields[k] = v.get(self.PRIMARY_KEYS, set()) | v.get(self.REPLICATION_KEYS, set()) + return auto_fields + + def run_and_verify_check_mode(self, conn_id): + """ + Run the tap in check mode and verify it succeeds. + This should be ran prior to field selection and initial sync. + Return the connection id and found catalogs from menagerie. + """ + # run in check mode + check_job_name = runner.run_check_mode(self, conn_id) + + # verify check exit codes + exit_status = menagerie.get_exit_status(conn_id, check_job_name) + menagerie.verify_check_exit_status(self, exit_status, check_job_name) + + found_catalogs = menagerie.get_catalogs(conn_id) + self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id)) + + found_catalog_names = set(map(lambda c: c['stream_name'], found_catalogs)) + self.assertSetEqual(self.expected_check_streams(), found_catalog_names, msg="discovered schemas do not match") + print("discovered schemas are OK") + + return found_catalogs + + def run_and_verify_sync(self, conn_id): + sync_job_name = runner.run_sync_mode(self, conn_id) + + # verify tap and target exit codes + exit_status = menagerie.get_exit_status(conn_id, sync_job_name) + menagerie.verify_sync_exit_status(self, exit_status, sync_job_name) + + sync_record_count = runner.examine_target_output_file(self, + conn_id, + self.expected_check_streams(), + self.expected_primary_keys()) + + self.assertGreater( + sum(sync_record_count.values()), 0, + msg="failed to replicate any data: {}".format(sync_record_count) + ) + print("total replicated row count: {}".format(sum(sync_record_count.values()))) + + return sync_record_count + + def perform_and_verify_table_and_field_selection(self, + conn_id, + test_catalogs, + select_all_fields=True): + """ + Perform table and field selection based off of the streams to select + set and field selection parameters. + Verify this results in the expected streams selected and all or no + fields selected for those streams. + """ + + # Select all available fields or select no fields from all testable streams + self.select_all_streams_and_fields( + conn_id=conn_id, catalogs=test_catalogs, select_all_fields=select_all_fields + ) + + catalogs = menagerie.get_catalogs(conn_id) + + # Ensure our selection affects the catalog + expected_selected = [tc.get('stream_name') for tc in test_catalogs] + for cat in catalogs: + catalog_entry = menagerie.get_annotated_schema(conn_id, cat['stream_id']) + + # Verify all testable streams are selected + selected = catalog_entry.get('annotated-schema').get('selected') + print("Validating selection on {}: {}".format(cat['stream_name'], selected)) + if cat['stream_name'] not in expected_selected: + self.assertFalse(selected, msg="Stream selected, but not testable.") + continue # Skip remaining assertions if we aren't selecting this stream + self.assertTrue(selected, msg="Stream not selected.") + + if select_all_fields: + # Verify all fields within each selected stream are selected + for field, field_props in catalog_entry.get('annotated-schema').get('properties').items(): + field_selected = field_props.get('selected') + print("\tValidating selection on {}.{}: {}".format( + cat['stream_name'], field, field_selected)) + self.assertTrue(field_selected, msg="Field not selected.") + else: + # Verify only automatic fields are selected + expected_automatic_fields = self.expected_automatic_fields().get(cat['stream_name']) + selected_fields = self.get_selected_fields_from_metadata(catalog_entry['metadata']) + self.assertEqual(expected_automatic_fields, selected_fields) + + @staticmethod + def get_selected_fields_from_metadata(metadata): + selected_fields = set() + for field in metadata: + is_field_metadata = len(field['breadcrumb']) > 1 + inclusion_automatic_or_selected = ( + field['metadata']['selected'] is True or \ + field['metadata']['inclusion'] == 'automatic' + ) + if is_field_metadata and inclusion_automatic_or_selected: + selected_fields.add(field['breadcrumb'][1]) + return selected_fields + + + @staticmethod + def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = True): + """Select all streams and all fields within streams""" + for catalog in catalogs: + schema = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) + + non_selected_properties = [] + if not select_all_fields: + # get a list of all properties so that none are selected + non_selected_properties = schema.get('annotated-schema', {}).get( + 'properties', {}).keys() + + connections.select_catalog_and_fields_via_metadata( + conn_id, catalog, schema, [], non_selected_properties) diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py new file mode 100644 index 0000000..91fb281 --- /dev/null +++ b/tests/test_all_fields.py @@ -0,0 +1,84 @@ +import tap_tester.connections as connections +import tap_tester.runner as runner +import tap_tester.menagerie as menagerie +from base import ActiveCampaignTest + +class ActiveCampaignAllFields(ActiveCampaignTest): + """Ensure running the tap with all streams and fields selected results in the replication of all fields.""" + + def name(self): + return "activecampaign_all_fields" + + def test_run(self): + """ + • Verify no unexpected streams were replicated + • Verify that more than just the automatic fields are replicated for each stream. + • verify all fields for each stream are replicated + """ + + + # Streams to verify all fields tests + expected_streams = self.expected_check_streams() + + # We are not able to generate data for `contact_conversions` stream. + # For `sms` stream it requires Professional plan of account. So, removing it from streams_to_test set. + expected_streams = expected_streams - {'contact_conversions', 'sms'} + + expected_automatic_fields = self.expected_automatic_fields() + conn_id = connections.ensure_connection(self) + + found_catalogs = self.run_and_verify_check_mode(conn_id) + + # table and field selection + test_catalogs_all_fields = [catalog for catalog in found_catalogs + if catalog.get('tap_stream_id') in expected_streams] + + self.perform_and_verify_table_and_field_selection( + conn_id, test_catalogs_all_fields) + + # grab metadata after performing table-and-field selection to set expectations + # used for asserting all fields are replicated + stream_to_all_catalog_fields = dict() + for catalog in test_catalogs_all_fields: + stream_id, stream_name = catalog['stream_id'], catalog['stream_name'] + catalog_entry = menagerie.get_annotated_schema(conn_id, stream_id) + fields_from_field_level_md = [md_entry['breadcrumb'][1] + for md_entry in catalog_entry['metadata'] + if md_entry['breadcrumb'] != []] + stream_to_all_catalog_fields[stream_name] = set( + fields_from_field_level_md) + + self.run_and_verify_sync(conn_id) + + synced_records = runner.get_records_from_target_output() + + # Verify no unexpected streams were replicated + synced_stream_names = set(synced_records.keys()) + self.assertSetEqual(expected_streams, synced_stream_names) + + + for stream in expected_streams: + with self.subTest(stream=stream): + + # expected values + expected_all_keys = stream_to_all_catalog_fields[stream] + expected_automatic_keys = expected_automatic_fields.get( + stream, set()) + + # Verify that more than just the automatic fields are replicated for each stream. + self.assertTrue(expected_automatic_keys.issubset( + expected_all_keys), msg='{} is not in "expected_all_keys"'.format(expected_automatic_keys-expected_all_keys)) + + messages = synced_records.get(stream) + # collect actual values + actual_all_keys = set() + for message in messages['messages']: + if message['action'] == 'upsert': + actual_all_keys.update(message['data'].keys()) + + # As we can't generate following field by activecampaign APIs and UI, so removed it form expectation list. + if stream == "ecommerce_orders": + expected_all_keys = expected_all_keys - {'order_products'} + + # verify all fields for each stream are replicated + self.assertSetEqual(expected_all_keys, actual_all_keys) \ No newline at end of file diff --git a/tests/test_configurations.py b/tests/test_configurations.py deleted file mode 100644 index 15375fe..0000000 --- a/tests/test_configurations.py +++ /dev/null @@ -1,106 +0,0 @@ -def config(): - return { - "test_name": "test_sync", - "tap_name": "tap-snapchat-ads", - "type": "platform.snapchat_ads", - "properties": { - "start_date": "TAP_SNAPCHAT_ADS_START_DATE", - "client_id": "TAP_SNAPCHAT_ADS_CLIENT_ID", - "swipe_up_attribution_window": "TAP_SNAPCHAT_ADS_SWIPE_UP_ATTRIBUTION_WINDOW", - "view_attribution_window": "TAP_SNAPCHAT_ADS_VIEW_ATTRIBUTION_WINDOW", - "omit_empty": "TAP_SNAPCHAT_ADS_OMIT_EMPTY", - "targeting_country_codes": "TAP_SNAPCHAT_ADS_TARGETING_COUNTRY_CODES", - }, - "credentials": { - "client_secret": "TAP_SNAPCHAT_ADS_CLIENT_SECRET", - "refresh_token": "TAP_SNAPCHAT_ADS_REFRESH_TOKEN", - }, - "bookmark": { - "bookmark_dict": "organizations", - "bookmark_key": "updated_at", - "bookmark_timestamp": "2020-05-11T21:30:43.303000Z" - }, - "streams": { - "ad_accounts": {"id"}, - "ad_squads": {"id"}, - "ads": {"id"}, - "audience_segments": {"id"}, - "billing_centers": {"id"}, - "campaigns": {"id"}, - "creatives": {"id"}, - "funding_sources": {"id"}, - "media": {"id"}, - "members": {"id"}, - "organizations": {"id"}, - "phone_numbers": {"id"}, - "pixel_domain_stats": {"id"}, - "pixels": {"id"}, - "product_catalogs": {"id"}, - "product_sets": {"id"}, - "phone_numbers": {"id"}, - "product_catalogs": {"id"}, - "product_sets": {"id"}, - "roles": {"id"}, - "ad_account_stats_daily": {"id", "start_time"}, - "ad_account_stats_hourly": {"id", "start_time"}, - "campaign_stats_daily": {"id", "start_time"}, - "campaign_stats_hourly": {"id", "start_time"}, - "ad_squad_stats_daily": {"id", "start_time"}, - "ad_squad_stats_hourly": {"id", "start_time"}, - "ad_stats_daily": {"id", "start_time"}, - "ad_stats_hourly": {"id", "start_time"}, - "targeting_age_groups": {"id"}, - "targeting_genders": {"id"}, - "targeting_languages": {"id"}, - "targeting_advanced_demographics": {"id"}, - "targeting_connection_types": {"id"}, - "targeting_os_types": {"id"}, - "targeting_ios_versions": {"id"}, - "targeting_android_versions": {"id"}, - "targeting_carriers": {"id"}, - "targeting_device_makes": {"id"}, - "targeting_countries": {"id"}, - "targeting_regions": {"id"}, - "targeting_metros": {"id"}, - "targeting_postal_codes": {"id"}, - "targeting_interests_scls": {"id"}, - "targeting_interests_dlxs": {"id"}, - "targeting_interests_dlxc": {"id"}, - "targeting_interests_dlxp": {"id"}, - "targeting_interests_nln": {"id"}, - "targeting_interests_plc": {"id"}, - "targeting_location_categories": {"id"} - }, - "exclude_streams": [ - "pixel_domain_stats", - "ad_account_stats_daily", - "ad_account_stats_hourly", - "campaign_stats_daily", - "campaign_stats_hourly", - "ad_squad_stats_daily", - "ad_squad_stats_hourly", - "ad_stats_daily", - "ad_stats_hourly", - "targeting_age_groups", - "targeting_genders", - "targeting_languages", - "targeting_advanced_demographics", - "targeting_connection_types", - "targeting_os_types", - "targeting_ios_versions", - "targeting_android_versions", - "targeting_carriers", - "targeting_device_makes", - "targeting_countries", - "targeting_regions", - "targeting_metros", - "targeting_postal_codes", - "targeting_interests_scls", - "targeting_interests_dlxs", - "targeting_interests_dlxc", - "targeting_interests_dlxp", - "targeting_interests_nln", - "targeting_interests_plc", - "targeting_location_categories" - ] - } diff --git a/tests/test_sync.py b/tests/test_sync.py deleted file mode 100644 index 42ef170..0000000 --- a/tests/test_sync.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Test tap combined -""" - -import unittest -from datetime import datetime, timedelta -import os -from test_configuration import config -from tap_tester import menagerie -import tap_tester.runner as runner -import tap_tester.connections as connections - -configuration = config() - -class TestSyncNonReportStreams(unittest.TestCase): - """ Test the non-report streams """ - - def name(self): - return configuration['test_name'] - - def tap_name(self): - """The name of the tap""" - return configuration['tap_name'] - - def get_type(self): - """the expected url route ending""" - return configuration['type'] - - def expected_check_streams(self): - return set(configuration['streams'].keys()) - - def expected_sync_streams(self): - return set(configuration['streams'].keys()) - - def expected_pks(self): - return configuration['streams'] - - def get_properties(self): - """Configuration properties required for the tap.""" - properties_dict = {} - props = configuration['properties'] - for prop in props: - properties_dict[prop] = os.getenv(props[prop]) - - return properties_dict - - def get_credentials(self): - """Authentication information for the test account. Username is expected as a property.""" - credentials_dict = {} - creds = configuration['credentials'] - for cred in creds: - credentials_dict[cred] = os.getenv(creds[cred]) - - return credentials_dict - - def setUp(self): - missing_envs = [] - props = configuration['properties'] - creds = configuration['credentials'] - - for prop in props: - if os.getenv(props[prop]) is None: - missing_envs.append(prop) - for cred in creds: - if os.getenv(creds[cred]) is None: - missing_envs.append(cred) - - if len(missing_envs) != 0: - raise Exception("set " + ", ".join(missing_envs)) - - def test_run(self): - - conn_id = connections.ensure_connection(self, payload_hook=None) - - # Run the tap in check mode - check_job_name = runner.run_check_mode(self, conn_id) - - # Verify the check's exit status - exit_status = menagerie.get_exit_status(conn_id, check_job_name) - menagerie.verify_check_exit_status(self, exit_status, check_job_name) - - # Verify that there are catalogs found - found_catalogs = menagerie.get_catalogs(conn_id) - self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id)) - - found_catalog_names = set(map(lambda c: c['tap_stream_id'], found_catalogs)) - subset = self.expected_check_streams().issubset(found_catalog_names) - self.assertTrue(subset, msg="Expected check streams are not subset of discovered catalog") - # - # # Select some catalogs - our_catalogs = [c for c in found_catalogs if c.get('tap_stream_id') in self.expected_sync_streams()] - for catalog in our_catalogs: - schema = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) - connections.select_catalog_and_fields_via_metadata(conn_id, catalog, schema, [], []) - - # # Verify that all streams sync at least one row for initial sync - # # This test is also verifying access token expiration handling. If test fails with - # # authentication error, refresh token was not replaced after expiring. - menagerie.set_state(conn_id, {}) - sync_job_name = runner.run_sync_mode(self, conn_id) - - # # Verify tap and target exit codes - exit_status = menagerie.get_exit_status(conn_id, sync_job_name) - menagerie.verify_sync_exit_status(self, exit_status, sync_job_name) - record_count_by_stream = runner.examine_target_output_file(self, conn_id, self.expected_sync_streams(), - self.expected_pks()) - zero_count_streams = {k for k, v in record_count_by_stream.items() if v == 0} - self.assertFalse(zero_count_streams, - msg="The following streams did not sync any rows {}".format(zero_count_streams)) - - # # Verify that bookmark values are correct after incremental sync - bookmark_props = configuration['bookmark'] - current_state = menagerie.get_state(conn_id) - test_bookmark = current_state['bookmarks'][bookmark_props['bookmark_dict']][bookmark_props['bookmark_key']] - print(test_bookmark) - self.assertTrue(test_bookmark == bookmark_props['bookmark_timestamp'], - msg="The bookmark value does not match the expected result")