From ee54c242ce65cc260bfc3b217d8a78294a405501 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Fri, 9 Dec 2022 19:12:50 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89Source=20Delighted:=20migrate=20to?= =?UTF-8?q?=20lowcode=20+=20certify=20to=20beta=20(#19822)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * #18711 source Delighted: migrate to lowcode + certify to Beta * #18711 source delighted: fix configured catalog * #18711 review fixes * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 4 +- .../src/main/resources/seed/source_specs.yaml | 18 +- .../connectors/source-delighted/Dockerfile | 6 +- .../acceptance-test-config.yml | 38 ++-- .../integration_tests/abnormal_state.json | 36 +++- .../integration_tests/configured_catalog.json | 12 +- .../integration_tests/expected_records.txt | 30 +++ .../integration_tests/sample_state.json | 14 -- .../source_delighted/delighted.yaml | 129 ++++++++++++ .../source_delighted/schemas/people.json | 3 + .../source_delighted/source.py | 183 +----------------- .../source_delighted/spec.json | 18 +- .../source-delighted/unit_tests/unit_test.py | 84 -------- docs/integrations/sources/delighted.md | 55 +++--- 14 files changed, 280 insertions(+), 350 deletions(-) create mode 100644 airbyte-integrations/connectors/source-delighted/integration_tests/expected_records.txt delete mode 100644 airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-delighted/source_delighted/delighted.yaml delete mode 100644 airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a050d0f4cb79..1ba067a2482b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -382,11 +382,11 @@ - name: Delighted sourceDefinitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635 dockerRepository: airbyte/source-delighted - dockerImageTag: 0.1.4 + dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.com/integrations/sources/delighted icon: delighted.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Dixa sourceDefinitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7 dockerRepository: airbyte/source-dixa diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 263569072bb0..19e6688f2f25 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3027,7 +3027,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-delighted:0.1.4" +- dockerImage: "airbyte/source-delighted:0.2.0" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -3037,21 +3037,21 @@ required: - "since" - "api_key" - additionalProperties: false + additionalProperties: true properties: + api_key: + title: "Delighted API Key" + type: "string" + description: "A Delighted API key." + airbyte_secret: true + order: 0 since: - title: "Since" + title: "Date Since" type: "string" description: "The date from which you'd like to replicate the data" examples: - "2022-05-30 04:50:23" pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$" - order: 0 - api_key: - title: "Delighted API Key" - type: "string" - description: "A Delighted API key." - airbyte_secret: true order: 1 supportsNormalization: false supportsDBT: false diff --git a/airbyte-integrations/connectors/source-delighted/Dockerfile b/airbyte-integrations/connectors/source-delighted/Dockerfile index 42986453d08c..f427f2095c9d 100644 --- a/airbyte-integrations/connectors/source-delighted/Dockerfile +++ b/airbyte-integrations/connectors/source-delighted/Dockerfile @@ -4,13 +4,13 @@ FROM python:3.9-slim RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* WORKDIR /airbyte/integration_code -COPY source_delighted ./source_delighted -COPY main.py ./ COPY setup.py ./ RUN pip install . +COPY source_delighted ./source_delighted +COPY main.py ./. ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-delighted diff --git a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml index 5a279ddfcfc3..50d5bffc1fc2 100644 --- a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml @@ -1,24 +1,32 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) # for more information about how to configure these tests connector_image: airbyte/source-delighted:dev -tests: +test_strictness_level: high +acceptance_tests: spec: - - spec_path: "source_delighted/spec.json" + tests: + - spec_path: "source_delighted/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["bounces"] + tests: + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.txt" incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json index 08e8cdd7f6aa..b51b0da38377 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json @@ -1,14 +1,30 @@ -{ - "people": { - "created_at": "4126108288" +[ + { + "type": "STREAM", + "stream": { + "stream_state": {"created_at": 4126108288}, + "stream_descriptor": {"name": "people"} + } }, - "unsubscribes": { - "unsubscribed_at": "4126108288" + { + "type": "STREAM", + "stream": { + "stream_state": {"unsubscribed_at": 4126108288}, + "stream_descriptor": {"name": "unsubscribes"} + } }, - "bounces": { - "bounced_at": "4126108288" + { + "type": "STREAM", + "stream": { + "stream_state": {"bounced_at": 4126108288}, + "stream_descriptor": {"name": "bounces"} + } }, - "survey_responses": { - "updated_at": "4126108288" + { + "type": "STREAM", + "stream": { + "stream_state": {"updated_at": 4126108288}, + "stream_descriptor": {"name": "survey_responses"} + } } -} +] diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json index 95d742759140..c98e32add5ae 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json @@ -11,7 +11,8 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["bounced_at"] + "cursor_field": ["bounced_at"], + "primary_key": [["person_id"]] }, { "stream": { @@ -24,7 +25,8 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["created_at"], + "primary_key": [["id"]] }, { "stream": { @@ -37,7 +39,8 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["updated_at"] + "cursor_field": ["updated_at"], + "primary_key": [["id"]] }, { "stream": { @@ -50,7 +53,8 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["unsubscribed_at"] + "cursor_field": ["unsubscribed_at"], + "primary_key": [["person_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-delighted/integration_tests/expected_records.txt new file mode 100644 index 000000000000..077af90270ff --- /dev/null +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/expected_records.txt @@ -0,0 +1,30 @@ +{"stream": "bounces", "data": {"person_id": "1046789978", "email": "foo_test201@airbyte.io", "name": "Foo Test201", "bounced_at": 1641455285}, "emitted_at": 1669106877664} +{"stream": "bounces", "data": {"person_id": "1046789981", "email": "foo_test202@airbyte.io", "name": "Foo Test202", "bounced_at": 1641455285}, "emitted_at": 1669106877664} +{"stream": "bounces", "data": {"person_id": "1046789983", "email": "foo_test203@airbyte.io", "name": "Foo Test203", "bounced_at": 1641455285}, "emitted_at": 1669106877664} +{"stream": "bounces", "data": {"person_id": "1046789984", "email": "foo_test204@airbyte.io", "name": "Foo Test204", "bounced_at": 1641455286}, "emitted_at": 1669106877664} +{"stream": "bounces", "data": {"person_id": "1046789989", "email": "foo_test205@airbyte.io", "name": "Foo Test205", "bounced_at": 1641455286}, "emitted_at": 1669106877664} +{"stream": "people", "data": {"id": "1046789978", "name": "Foo Test201", "email": "foo_test201@airbyte.io", "created_at": 1641455282, "last_sent_at": 1641455283, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863} +{"stream": "people", "data": {"id": "1046789981", "name": "Foo Test202", "email": "foo_test202@airbyte.io", "created_at": 1641455283, "last_sent_at": 1641455283, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863} +{"stream": "people", "data": {"id": "1046789983", "name": "Foo Test203", "email": "foo_test203@airbyte.io", "created_at": 1641455284, "last_sent_at": 1641455284, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863} +{"stream": "people", "data": {"id": "1046789984", "name": "Foo Test204", "email": "foo_test204@airbyte.io", "created_at": 1641455284, "last_sent_at": 1641455285, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863} +{"stream": "people", "data": {"id": "1046789989", "name": "Foo Test205", "email": "foo_test205@airbyte.io", "created_at": 1641455285, "last_sent_at": 1641455285, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863} +{"stream": "survey_responses", "data": {"id": "210554372", "person": "980285131", "survey_type": "nps", "score": 8, "comment": "Test Comment2", "permalink": "https://app.delighted.com/r/ALmUkRpgBNPkvxcniCKw7gCeS9RjnLw8", "created_at": 1641289688, "updated_at": 1641289688, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880196} +{"stream": "survey_responses", "data": {"id": "210554370", "person": "980285130", "survey_type": "nps", "score": 9, "comment": "Test Comment1", "permalink": "https://app.delighted.com/r/LnPusbUATrBj4MMlwfZY2HkBUJ5qqNWu", "created_at": 1641289688, "updated_at": 1641289688, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880197} +{"stream": "survey_responses", "data": {"id": "210554373", "person": "1040825885", "survey_type": "nps", "score": 3, "comment": "Test Comment3", "permalink": "https://app.delighted.com/r/vKJJDgBrSCGEVIIseELBjuqvBdnwzNH4", "created_at": 1641289689, "updated_at": 1641289689, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880197} +{"stream": "survey_responses", "data": {"id": "210554375", "person": "1040826153", "survey_type": "nps", "score": 7, "comment": "Test Comment5", "permalink": "https://app.delighted.com/r/06b0Iq3r0Y2SJfASbXgRmVFCxW4mukYw", "created_at": 1641289690, "updated_at": 1641289690, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880198} +{"stream": "survey_responses", "data": {"id": "210554374", "person": "1040826151", "survey_type": "nps", "score": 1, "comment": "Test Comment4", "permalink": "https://app.delighted.com/r/5zUFV3BLVnAErsNrQe6AzF66G7K2ffHu", "created_at": 1641289690, "updated_at": 1641289690, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880199} +{"stream": "survey_responses", "data": {"id": "210554379", "person": "1040826160", "survey_type": "nps", "score": 7, "comment": "Test Comment7", "permalink": "https://app.delighted.com/r/3vnf4D0LVvz9VUW7xj5l5pIjdnlSy7bs", "created_at": 1641289691, "updated_at": 1641289691, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880199} +{"stream": "survey_responses", "data": {"id": "210554376", "person": "1040826156", "survey_type": "nps", "score": 0, "comment": "Test Comment6", "permalink": "https://app.delighted.com/r/0nkK32ykcorkSL22EZ6ZRknbu3RLgGOc", "created_at": 1641289691, "updated_at": 1641289691, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880200} +{"stream": "survey_responses", "data": {"id": "210554381", "person": "1040826161", "survey_type": "nps", "score": 3, "comment": "Test Comment8", "permalink": "https://app.delighted.com/r/oTmqsmFvjd5JdBcm6vZXs6XsvsgGTGFw", "created_at": 1641289692, "updated_at": 1641289692, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880201} +{"stream": "survey_responses", "data": {"id": "210554385", "person": "1040826168", "survey_type": "nps", "score": 6, "comment": "Test Comment10", "permalink": "https://app.delighted.com/r/So4x71E7PhBlGKe279I3LQR7zUANgV3O", "created_at": 1641289693, "updated_at": 1641289693, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880201} +{"stream": "survey_responses", "data": {"id": "210554382", "person": "1040826163", "survey_type": "nps", "score": 0, "comment": "Test Comment9", "permalink": "https://app.delighted.com/r/HZhU9lwnwZkrkNao9DTSmJgVjnIQll3k", "created_at": 1641289693, "updated_at": 1641289693, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880202} +{"stream": "unsubscribes", "data": {"person_id": "1040826276", "email": "foo_test50@airbyte.io", "name": "Foo Test50", "unsubscribed_at": 1641289576}, "emitted_at": 1669106882340} +{"stream": "unsubscribes", "data": {"person_id": "1040826278", "email": "foo_test51@airbyte.io", "name": "Foo Test51", "unsubscribed_at": 1641289577}, "emitted_at": 1669106882341} +{"stream": "unsubscribes", "data": {"person_id": "1040826281", "email": "foo_test52@airbyte.io", "name": "Foo Test52", "unsubscribed_at": 1641289577}, "emitted_at": 1669106882341} +{"stream": "unsubscribes", "data": {"person_id": "1040826285", "email": "foo_test53@airbyte.io", "name": "Foo Test53", "unsubscribed_at": 1641289578}, "emitted_at": 1669106882342} +{"stream": "unsubscribes", "data": {"person_id": "1040826288", "email": "foo_test54@airbyte.io", "name": "Foo Test54", "unsubscribed_at": 1641289578}, "emitted_at": 1669106882342} +{"stream": "unsubscribes", "data": {"person_id": "1040826291", "email": "foo_test55@airbyte.io", "name": "Foo Test55", "unsubscribed_at": 1641289579}, "emitted_at": 1669106882342} +{"stream": "unsubscribes", "data": {"person_id": "1040826292", "email": "foo_test56@airbyte.io", "name": "Foo Test56", "unsubscribed_at": 1641289579}, "emitted_at": 1669106882343} +{"stream": "unsubscribes", "data": {"person_id": "1040826293", "email": "foo_test57@airbyte.io", "name": "Foo Test57", "unsubscribed_at": 1641289580}, "emitted_at": 1669106882343} +{"stream": "unsubscribes", "data": {"person_id": "1040826295", "email": "foo_test58@airbyte.io", "name": "Foo Test58", "unsubscribed_at": 1641289580}, "emitted_at": 1669106882344} +{"stream": "unsubscribes", "data": {"person_id": "1040826298", "email": "foo_test59@airbyte.io", "name": "Foo Test59", "unsubscribed_at": 1641289581}, "emitted_at": 1669106882344} diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json deleted file mode 100644 index 80f39260626d..000000000000 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "people": { - "created_at": "1601586688" - }, - "unsubscribes": { - "unsubscribed_at": "1601586688" - }, - "bounces": { - "bounced_at": "1601586688" - }, - "survey_responses": { - "updated_at": "1601586688" - } -} diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/delighted.yaml b/airbyte-integrations/connectors/source-delighted/source_delighted/delighted.yaml new file mode 100644 index 000000000000..b03df349553c --- /dev/null +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/delighted.yaml @@ -0,0 +1,129 @@ +version: "0.1.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_pointer: [] + requester: + type: HttpRequester + name: "{{ options['name'] }}" + url_base: "https://api.delighted.com/v1/" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['api_key'] }}" + error_handler: + type: "DefaultErrorHandler" + backoff_strategies: + - type: WaitTimeFromHeader + header: "Retry-After" + stream_slicer: + cursor_field: "{{ options['stream_cursor_field'] }}" + datetime_format: "%s" + start_datetime: + datetime: "{{ config['since'] }}" + datetime_format: "%Y-%m-%d %H:%M:%S" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "1w" + end_time_option: + field_name: "until" + inject_into: "request_parameter" + start_time_option: + field_name: "since" + inject_into: "request_parameter" + type: DatetimeStreamSlicer + retriever: + type: SimpleRetriever + name: "{{ options['name'] }}" + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: DefaultPaginator + pagination_strategy: + type: "PageIncrement" + page_size: 100 + start_from_page: 1 + page_size_option: + field_name: "per_page" + inject_into: "request_parameter" + page_token_option: + field_name: "page" + inject_into: "request_parameter" + url_base: "*ref(definitions.requester.url_base)" + requester: + $ref: "*ref(definitions.requester)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + base_stream: + primary_key: "id" + retriever: + $ref: "*ref(definitions.retriever)" + people: + $ref: "*ref(definitions.base_stream)" + stream_cursor_field: "created_at" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + type: DefaultPaginator + pagination_strategy: + type: CursorPagination + cursor_value: "{{ headers['link']['next']['url'] }}" + stop_condition: "{{ 'next' not in headers['link'] }}" + page_size: 100 + page_size_option: + field_name: "per_page" + inject_into: "request_parameter" + page_token_option: + inject_into: "path" + url_base: "*ref(definitions.requester.url_base)" + $options: + name: "people" + path: "people.json" + stream_cursor_field: "created_at" + bounces: + $ref: "*ref(definitions.base_stream)" + primary_key: "person_id" + stream_cursor_field: "bounced_at" + $options: + stream_cursor_field: "bounced_at" + name: "bounces" + path: "bounces.json" + unsubscribes: + $ref: "*ref(definitions.base_stream)" + primary_key: "person_id" + stream_cursor_field: "unsubscribed_at" + $options: + stream_cursor_field: "unsubscribed_at" + name: "unsubscribes" + path: "unsubscribes.json" + survey_responses: + $ref: "*ref(definitions.base_stream)" + stream_cursor_field: "updated_at" + retriever: + $ref: "*ref(definitions.retriever)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + end_time_option: + field_name: "updated_until" + inject_into: "request_parameter" + start_time_option: + field_name: "updated_since" + inject_into: "request_parameter" + $options: + stream_cursor_field: "updated_at" + name: "survey_responses" + path: "survey_responses.json" + +streams: + - "*ref(definitions.people)" + - "*ref(definitions.unsubscribes)" + - "*ref(definitions.bounces)" + - "*ref(definitions.survey_responses)" + +check: + type: CheckStream + stream_names: ["survey_responses"] diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json index ca209db77baa..f3464126d15c 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json @@ -11,6 +11,9 @@ "email": { "type": ["null", "string"] }, + "phone_number": { + "type": ["null", "string"] + }, "created_at": { "type": "integer" }, diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index f584d5c98792..2f2e7e7de160 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -2,179 +2,16 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import base64 -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qsl, urlparse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - -# Basic full refresh stream -class DelightedStream(HttpStream, ABC): - url_base = "https://api.delighted.com/v1/" - - # Page size - limit = 100 - page = 1 - - # Define primary key to all streams as primary key - primary_key = "id" - - def __init__(self, since: pendulum.datetime, **kwargs): - super().__init__(**kwargs) - self.since = since - - @property - def since_ts(self) -> int: - return int(self.since.timestamp()) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_data = response.json() - if len(response_data) == self.limit: - self.page += 1 - return {"page": self.page} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"per_page": self.limit, "since": self.since_ts} - if next_page_token: - params.update(**next_page_token) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - -class IncrementalDelightedStream(DelightedStream, ABC): - # Getting page size as 'limit' from parent class - @property - def limit(self): - return super().limit - - state_checkpoint_interval = limit - - @property - def cursor_field(self) -> str: - return "created_at" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - - def request_params(self, stream_state=None, **kwargs): - stream_state = stream_state or {} - params = super().request_params(stream_state=stream_state, **kwargs) - if stream_state: - params["since"] = stream_state.get(self.cursor_field) - return params - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - for record in super().parse_response(response=response, stream_state=stream_state, **kwargs): - if self.cursor_field not in stream_state or record[self.cursor_field] > stream_state[self.cursor_field]: - yield record - - -class People(IncrementalDelightedStream): - """ - API docs: https://app.delighted.com/docs/api/listing-people - """ - - def path(self, **kwargs) -> str: - return "people.json" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - # Getting next page link - next_page = response.links.get("next", None) - if next_page: - return {"page_info": dict(parse_qsl(urlparse(next_page.get("url")).query)).get("page_info")} - - -class Unsubscribes(IncrementalDelightedStream): - """ - API docs: https://app.delighted.com/docs/api/listing-unsubscribed-people - """ - - cursor_field = "unsubscribed_at" - primary_key = "person_id" - - def path(self, **kwargs) -> str: - return "unsubscribes.json" - - -class Bounces(IncrementalDelightedStream): - """ - API docs: https://app.delighted.com/docs/api/listing-bounced-people - """ - - cursor_field = "bounced_at" - primary_key = "person_id" - - def path(self, **kwargs) -> str: - return "bounces.json" - - -class SurveyResponses(IncrementalDelightedStream): - """ - API docs: https://app.delighted.com/docs/api/listing-survey-responses - """ - - cursor_field = "updated_at" - - def path(self, **kwargs) -> str: - return "survey_responses.json" - - def request_params(self, stream_state=None, **kwargs): - stream_state = stream_state or {} - params = super().request_params(stream_state=stream_state, **kwargs) - - if "since" in params: - params["updated_since"] = params.pop("since") - - if stream_state: - params["updated_since"] = stream_state.get(self.cursor_field) - - return params - - -# Source -class SourceDelighted(AbstractSource): - def _get_authenticator(self, config): - token = base64.b64encode(f"{config['api_key']}:".encode("utf-8")).decode("utf-8") - return TokenAuthenticator(token=token, auth_method="Basic") - - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - - Testing connection availability for the connector. - - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - - try: - auth = self._get_authenticator(config) - stream = SurveyResponses(authenticator=auth, since=pendulum.parse(config["since"])) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = self._get_authenticator(config) - stream_kwargs = {"authenticator": auth, "since": pendulum.parse(config["since"])} - return [ - Bounces(**stream_kwargs), - People(**stream_kwargs), - SurveyResponses(**stream_kwargs), - Unsubscribes(**stream_kwargs), - ] +# Declarative Source +class SourceDelighted(YamlDeclarativeSource): + def __init__(self): + super().__init__(path_to_yaml="delighted.yaml") diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json index 0292ddfbb50e..4531c866f8e1 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json @@ -5,21 +5,21 @@ "title": "Delighted Spec", "type": "object", "required": ["since", "api_key"], - "additionalProperties": false, + "additionalProperties": true, "properties": { - "since": { - "title": "Since", - "type": "string", - "description": "The date from which you'd like to replicate the data", - "examples": ["2022-05-30 04:50:23"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$", - "order": 0 - }, "api_key": { "title": "Delighted API Key", "type": "string", "description": "A Delighted API key.", "airbyte_secret": true, + "order": 0 + }, + "since": { + "title": "Date Since", + "type": "string", + "description": "The date from which you'd like to replicate the data", + "examples": ["2022-05-30 04:50:23"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$", "order": 1 } } diff --git a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py deleted file mode 100644 index d3d1c3116ab1..000000000000 --- a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import pendulum -import pytest -import responses -from airbyte_cdk.models import SyncMode -from source_delighted.source import Bounces, People, SourceDelighted, SurveyResponses, Unsubscribes - - -@pytest.fixture(scope="module") -def test_config(): - return { - "api_key": "test_api_key", - "since": "2022-01-01 00:00:00", - } - - -@pytest.fixture(scope="module") -def state(): - return { - "bounces": {"bounced_at": 1641455286}, - "people": {"created_at": 1641455285}, - "survey_responses": {"updated_at": 1641289816}, - "unsubscribes": {"unsubscribed_at": 1641289584}, - } - - -BOUNCES_RESPONSE = """ -[ - {"person_id": "1046789984", "email": "foo_test204@airbyte.io", "name": "Foo Test204", "bounced_at": 1641455286}, - {"person_id": "1046789989", "email": "foo_test205@airbyte.io", "name": "Foo Test205", "bounced_at": 1641455286} -] -""" - - -PEOPLE_RESPONSE = """ -[ - {"id": "1046789989", "name": "Foo Test205", "email": "foo_test205@airbyte.io", "created_at": 1641455285, "last_sent_at": 1641455285, "last_responded_at": null, "next_survey_scheduled_at": null} -] -""" - - -SURVEY_RESPONSES_RESPONSE = """ -[ - {"id": "210554887", "person": "1042205953", "survey_type": "nps", "score": 0, "comment": "Test Comment202", "permalink": "https://app.delighted.com/r/0q7QEdWzosv5G5c3w9gakivDwEIM5Hq0", "created_at": 1641289816, "updated_at": 1641289816, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, - {"id": "210554885", "person": "1042205947", "survey_type": "nps", "score": 5, "comment": "Test Comment201", "permalink": "https://app.delighted.com/r/GhWWrBT2wayswOc0AfT7fxpM3UwSpitN", "created_at": 1641289816, "updated_at": 1641289816, "person_properties": null, "notes": [], "tags": [], "additional_answers": []} -] -""" - - -UNSUBSCRIBES_RESPONSE = """ -[ - {"person_id": "1040826319", "email": "foo_test64@airbyte.io", "name": "Foo Test64", "unsubscribed_at": 1641289584} -] -""" - - -@pytest.mark.parametrize( - ("stream_class", "url", "response_body"), - [ - (Bounces, "https://api.delighted.com/v1/bounces.json", BOUNCES_RESPONSE), - (People, "https://api.delighted.com/v1/people.json", PEOPLE_RESPONSE), - (SurveyResponses, "https://api.delighted.com/v1/survey_responses.json", SURVEY_RESPONSES_RESPONSE), - (Unsubscribes, "https://api.delighted.com/v1/unsubscribes.json", UNSUBSCRIBES_RESPONSE), - ], -) -@responses.activate -def test_not_output_records_where_cursor_field_equals_state(state, test_config, stream_class, url, response_body): - responses.add( - responses.GET, - url, - body=response_body, - status=200, - ) - - stream = stream_class(pendulum.parse(test_config["since"]), authenticator=SourceDelighted()._get_authenticator(config=test_config)) - records = [r for r in stream.read_records(SyncMode.incremental, stream_state=state[stream.name])] - assert not records - - -def test_example_method(): - assert True diff --git a/docs/integrations/sources/delighted.md b/docs/integrations/sources/delighted.md index 3d978520943c..f150911ee13f 100644 --- a/docs/integrations/sources/delighted.md +++ b/docs/integrations/sources/delighted.md @@ -1,20 +1,30 @@ ---- -description: >- - Delighted is a proprietary self-serve experience management platform that allows collecting feedback from customers and employees through surveys. ---- - # Delighted -## Sync overview +This page contains the setup guide and reference information for the Delighted source connector. + +## Prerequisites + +To set up the Delighted source connector, you'll need the [Delighted API key](https://app.delighted.com/docs/api#authentication). + +## Set up the Delighted connector in Airbyte -The Delighted source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, enter the name for the Delighted connector and select **Delighted** from the Source type dropdown. +4. For **Since**, enter the date in a Unix Timestamp format. The data added on and after this date will be replicated. +5. For **API Key**, enter your [Delighted `API Key`](https://delighted.com/account/api). +6. Click **Set up source**. -This source can sync data for the [Delighted API](https://app.delighted.com/docs/api). +## Supported sync modes -This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). +The Delighted source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync) +* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) -### Output schema +## Supported Streams This Source is capable of syncing the following core Streams: @@ -23,22 +33,13 @@ This Source is capable of syncing the following core Streams: * [Bounced People](https://app.delighted.com/docs/api/listing-bounced-people) * [Unsubscribed People](https://app.delighted.com/docs/api/listing-unsubscribed-people) -## Getting started - -This connector supports `API PASSWORD` as the authentication method. - -### Connect using `API PASSWORD` option: -1. Go to `https://delighted.com/account/api` -2. Copy your Delighted API key. -6. You're ready to set up Delighted in Airbyte! - - ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.4 | 2022-06-10 | [13439](https://github.com/airbytehq/airbyte/pull/13439) | Change since parameter input to iso date | -| 0.1.3 | 2022-01-31 | [9550](https://github.com/airbytehq/airbyte/pull/9550) | Output only records in which cursor field is greater than the value in state for incremental streams | -| 0.1.2 | 2022-01-06 | [9333](https://github.com/airbytehq/airbyte/pull/9333) | Add incremental sync mode to streams in `integration_tests/configured_catalog.json` | -| 0.1.1 | 2022-01-04 | [9275](https://github.com/airbytehq/airbyte/pull/9275) | Fix pagination handling for `survey_responses`, `bounces` and `unsubscribes` streams | -| 0.1.0 | 2021-10-27 | [4551](https://github.com/airbytehq/airbyte/pull/4551) | Add Delighted source connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| 0.2.0 | 2022-11-22 | [19822](https://github.com/airbytehq/airbyte/pull/19822) | Migrate to Low code + certify to Beta | +| 0.1.4 | 2022-06-10 | [13439](https://github.com/airbytehq/airbyte/pull/13439) | Change since parameter input to iso date | +| 0.1.3 | 2022-01-31 | [9550](https://github.com/airbytehq/airbyte/pull/9550) | Output only records in which cursor field is greater than the value in state for incremental streams | +| 0.1.2 | 2022-01-06 | [9333](https://github.com/airbytehq/airbyte/pull/9333) | Add incremental sync mode to streams in `integration_tests/configured_catalog.json` | +| 0.1.1 | 2022-01-04 | [9275](https://github.com/airbytehq/airbyte/pull/9275) | Fix pagination handling for `survey_responses`, `bounces` and `unsubscribes` streams | +| 0.1.0 | 2021-10-27 | [4551](https://github.com/airbytehq/airbyte/pull/4551) | Add Delighted source connector |