🎉Source Delighted: migrate to lowcode + certify to beta (#19822)

* #18711 source Delighted: migrate to lowcode + certify to Beta * #18711 source delighted: fix configured catalog * #18711 review fixes * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Dec 9, 2022 · ee54c24 · ee54c24
1 parent 78d866d
commit ee54c24
Show file tree

Hide file tree

Showing 14 changed files with 280 additions and 350 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -382,11 +382,11 @@
 - name: Delighted
   sourceDefinitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635
   dockerRepository: airbyte/source-delighted
-  dockerImageTag: 0.1.4
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/delighted
   icon: delighted.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: Dixa
   sourceDefinitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7
   dockerRepository: airbyte/source-dixa

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3027,7 +3027,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-delighted:0.1.4"
+- dockerImage: "airbyte/source-delighted:0.2.0"
   spec:
     documentationUrl: "https://docsurl.com"
     connectionSpecification:
@@ -3037,21 +3037,21 @@
       required:
       - "since"
       - "api_key"
-      additionalProperties: false
+      additionalProperties: true
       properties:
+        api_key:
+          title: "Delighted API Key"
+          type: "string"
+          description: "A Delighted API key."
+          airbyte_secret: true
+          order: 0
         since:
-          title: "Since"
+          title: "Date Since"
           type: "string"
           description: "The date from which you'd like to replicate the data"
           examples:
           - "2022-05-30 04:50:23"
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:[0-9]{2}:[0-9]{2})?$"
-          order: 0
-        api_key:
-          title: "Delighted API Key"
-          type: "string"
-          description: "A Delighted API key."
-          airbyte_secret: true
           order: 1
     supportsNormalization: false
     supportsDBT: false

diff --git a/airbyte-integrations/connectors/source-delighted/Dockerfile b/airbyte-integrations/connectors/source-delighted/Dockerfile
@@ -4,13 +4,13 @@ FROM python:3.9-slim
 RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /airbyte/integration_code
-COPY source_delighted ./source_delighted
-COPY main.py ./
 COPY setup.py ./
 RUN pip install .
+COPY source_delighted ./source_delighted
+COPY main.py ./.
 
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/source-delighted
diff --git a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml
@@ -1,24 +1,32 @@
 # See [Source Acceptance Tests](https://docs.airbyte.com/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md)
 # for more information about how to configure these tests
 connector_image: airbyte/source-delighted:dev
-tests:
+test_strictness_level: high
+acceptance_tests:
   spec:
-    - spec_path: "source_delighted/spec.json"
+    tests:
+      - spec_path: "source_delighted/spec.json"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: "secrets/config.json"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: ["bounces"]
+    tests:
+      - config_path: "secrets/config.json"
+        expect_records:
+          path: "integration_tests/expected_records.txt"
   incremental:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      future_state_path: "integration_tests/abnormal_state.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json
@@ -1,14 +1,30 @@
-{
-  "people": {
-    "created_at": "4126108288"
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {"created_at": 4126108288},
+      "stream_descriptor": {"name": "people"}
+    }
   },
-  "unsubscribes": {
-    "unsubscribed_at": "4126108288"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {"unsubscribed_at": 4126108288},
+      "stream_descriptor": {"name": "unsubscribes"}
+    }
   },
-  "bounces": {
-    "bounced_at": "4126108288"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {"bounced_at": 4126108288},
+      "stream_descriptor": {"name": "bounces"}
+    }
   },
-  "survey_responses": {
-    "updated_at": "4126108288"
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {"updated_at": 4126108288},
+      "stream_descriptor": {"name": "survey_responses"}
+    }
   }
-}
+]
diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json
@@ -11,7 +11,8 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["bounced_at"]
+      "cursor_field": ["bounced_at"],
+      "primary_key": [["person_id"]]
     },
     {
       "stream": {
@@ -24,7 +25,8 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["created_at"],
+      "primary_key": [["id"]]
     },
     {
       "stream": {
@@ -37,7 +39,8 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["updated_at"]
+      "cursor_field": ["updated_at"],
+      "primary_key": [["id"]]
     },
     {
       "stream": {
@@ -50,7 +53,8 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["unsubscribed_at"]
+      "cursor_field": ["unsubscribed_at"],
+      "primary_key": [["person_id"]]
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-delighted/integration_tests/expected_records.txt
@@ -0,0 +1,30 @@
+{"stream": "bounces", "data": {"person_id": "1046789978", "email": "foo_test201@airbyte.io", "name": "Foo Test201", "bounced_at": 1641455285}, "emitted_at": 1669106877664}
+{"stream": "bounces", "data": {"person_id": "1046789981", "email": "foo_test202@airbyte.io", "name": "Foo Test202", "bounced_at": 1641455285}, "emitted_at": 1669106877664}
+{"stream": "bounces", "data": {"person_id": "1046789983", "email": "foo_test203@airbyte.io", "name": "Foo Test203", "bounced_at": 1641455285}, "emitted_at": 1669106877664}
+{"stream": "bounces", "data": {"person_id": "1046789984", "email": "foo_test204@airbyte.io", "name": "Foo Test204", "bounced_at": 1641455286}, "emitted_at": 1669106877664}
+{"stream": "bounces", "data": {"person_id": "1046789989", "email": "foo_test205@airbyte.io", "name": "Foo Test205", "bounced_at": 1641455286}, "emitted_at": 1669106877664}
+{"stream": "people", "data": {"id": "1046789978", "name": "Foo Test201", "email": "foo_test201@airbyte.io", "created_at": 1641455282, "last_sent_at": 1641455283, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863}
+{"stream": "people", "data": {"id": "1046789981", "name": "Foo Test202", "email": "foo_test202@airbyte.io", "created_at": 1641455283, "last_sent_at": 1641455283, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863}
+{"stream": "people", "data": {"id": "1046789983", "name": "Foo Test203", "email": "foo_test203@airbyte.io", "created_at": 1641455284, "last_sent_at": 1641455284, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863}
+{"stream": "people", "data": {"id": "1046789984", "name": "Foo Test204", "email": "foo_test204@airbyte.io", "created_at": 1641455284, "last_sent_at": 1641455285, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863}
+{"stream": "people", "data": {"id": "1046789989", "name": "Foo Test205", "email": "foo_test205@airbyte.io", "created_at": 1641455285, "last_sent_at": 1641455285, "last_responded_at": null, "next_survey_scheduled_at": null}, "emitted_at": 1669106878863}
+{"stream": "survey_responses", "data": {"id": "210554372", "person": "980285131", "survey_type": "nps", "score": 8, "comment": "Test Comment2", "permalink": "https://app.delighted.com/r/ALmUkRpgBNPkvxcniCKw7gCeS9RjnLw8", "created_at": 1641289688, "updated_at": 1641289688, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880196}
+{"stream": "survey_responses", "data": {"id": "210554370", "person": "980285130", "survey_type": "nps", "score": 9, "comment": "Test Comment1", "permalink": "https://app.delighted.com/r/LnPusbUATrBj4MMlwfZY2HkBUJ5qqNWu", "created_at": 1641289688, "updated_at": 1641289688, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880197}
+{"stream": "survey_responses", "data": {"id": "210554373", "person": "1040825885", "survey_type": "nps", "score": 3, "comment": "Test Comment3", "permalink": "https://app.delighted.com/r/vKJJDgBrSCGEVIIseELBjuqvBdnwzNH4", "created_at": 1641289689, "updated_at": 1641289689, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880197}
+{"stream": "survey_responses", "data": {"id": "210554375", "person": "1040826153", "survey_type": "nps", "score": 7, "comment": "Test Comment5", "permalink": "https://app.delighted.com/r/06b0Iq3r0Y2SJfASbXgRmVFCxW4mukYw", "created_at": 1641289690, "updated_at": 1641289690, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880198}
+{"stream": "survey_responses", "data": {"id": "210554374", "person": "1040826151", "survey_type": "nps", "score": 1, "comment": "Test Comment4", "permalink": "https://app.delighted.com/r/5zUFV3BLVnAErsNrQe6AzF66G7K2ffHu", "created_at": 1641289690, "updated_at": 1641289690, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880199}
+{"stream": "survey_responses", "data": {"id": "210554379", "person": "1040826160", "survey_type": "nps", "score": 7, "comment": "Test Comment7", "permalink": "https://app.delighted.com/r/3vnf4D0LVvz9VUW7xj5l5pIjdnlSy7bs", "created_at": 1641289691, "updated_at": 1641289691, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880199}
+{"stream": "survey_responses", "data": {"id": "210554376", "person": "1040826156", "survey_type": "nps", "score": 0, "comment": "Test Comment6", "permalink": "https://app.delighted.com/r/0nkK32ykcorkSL22EZ6ZRknbu3RLgGOc", "created_at": 1641289691, "updated_at": 1641289691, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880200}
+{"stream": "survey_responses", "data": {"id": "210554381", "person": "1040826161", "survey_type": "nps", "score": 3, "comment": "Test Comment8", "permalink": "https://app.delighted.com/r/oTmqsmFvjd5JdBcm6vZXs6XsvsgGTGFw", "created_at": 1641289692, "updated_at": 1641289692, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880201}
+{"stream": "survey_responses", "data": {"id": "210554385", "person": "1040826168", "survey_type": "nps", "score": 6, "comment": "Test Comment10", "permalink": "https://app.delighted.com/r/So4x71E7PhBlGKe279I3LQR7zUANgV3O", "created_at": 1641289693, "updated_at": 1641289693, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880201}
+{"stream": "survey_responses", "data": {"id": "210554382", "person": "1040826163", "survey_type": "nps", "score": 0, "comment": "Test Comment9", "permalink": "https://app.delighted.com/r/HZhU9lwnwZkrkNao9DTSmJgVjnIQll3k", "created_at": 1641289693, "updated_at": 1641289693, "person_properties": null, "notes": [], "tags": [], "additional_answers": []}, "emitted_at": 1669106880202}
+{"stream": "unsubscribes", "data": {"person_id": "1040826276", "email": "foo_test50@airbyte.io", "name": "Foo Test50", "unsubscribed_at": 1641289576}, "emitted_at": 1669106882340}
+{"stream": "unsubscribes", "data": {"person_id": "1040826278", "email": "foo_test51@airbyte.io", "name": "Foo Test51", "unsubscribed_at": 1641289577}, "emitted_at": 1669106882341}
+{"stream": "unsubscribes", "data": {"person_id": "1040826281", "email": "foo_test52@airbyte.io", "name": "Foo Test52", "unsubscribed_at": 1641289577}, "emitted_at": 1669106882341}
+{"stream": "unsubscribes", "data": {"person_id": "1040826285", "email": "foo_test53@airbyte.io", "name": "Foo Test53", "unsubscribed_at": 1641289578}, "emitted_at": 1669106882342}
+{"stream": "unsubscribes", "data": {"person_id": "1040826288", "email": "foo_test54@airbyte.io", "name": "Foo Test54", "unsubscribed_at": 1641289578}, "emitted_at": 1669106882342}
+{"stream": "unsubscribes", "data": {"person_id": "1040826291", "email": "foo_test55@airbyte.io", "name": "Foo Test55", "unsubscribed_at": 1641289579}, "emitted_at": 1669106882342}
+{"stream": "unsubscribes", "data": {"person_id": "1040826292", "email": "foo_test56@airbyte.io", "name": "Foo Test56", "unsubscribed_at": 1641289579}, "emitted_at": 1669106882343}
+{"stream": "unsubscribes", "data": {"person_id": "1040826293", "email": "foo_test57@airbyte.io", "name": "Foo Test57", "unsubscribed_at": 1641289580}, "emitted_at": 1669106882343}
+{"stream": "unsubscribes", "data": {"person_id": "1040826295", "email": "foo_test58@airbyte.io", "name": "Foo Test58", "unsubscribed_at": 1641289580}, "emitted_at": 1669106882344}
+{"stream": "unsubscribes", "data": {"person_id": "1040826298", "email": "foo_test59@airbyte.io", "name": "Foo Test59", "unsubscribed_at": 1641289581}, "emitted_at": 1669106882344}
diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json
diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/delighted.yaml b/airbyte-integrations/connectors/source-delighted/source_delighted/delighted.yaml
@@ -0,0 +1,129 @@
+version: "0.1.0"
+
+definitions:
+  selector:
+    type: RecordSelector
+    extractor:
+      type: DpathExtractor
+      field_pointer: []
+  requester:
+    type: HttpRequester
+    name: "{{ options['name'] }}"
+    url_base: "https://api.delighted.com/v1/"
+    http_method: "GET"
+    authenticator:
+      type: BasicHttpAuthenticator
+      username: "{{ config['api_key'] }}"
+    error_handler:
+      type: "DefaultErrorHandler"
+      backoff_strategies:
+        - type: WaitTimeFromHeader
+          header: "Retry-After"
+  stream_slicer:
+    cursor_field: "{{ options['stream_cursor_field'] }}"
+    datetime_format: "%s"
+    start_datetime:
+      datetime: "{{ config['since'] }}"
+      datetime_format: "%Y-%m-%d %H:%M:%S"
+    end_datetime:
+      datetime: "{{ today_utc() }}"
+      datetime_format: "%Y-%m-%d"
+    step: "1w"
+    end_time_option:
+      field_name: "until"
+      inject_into: "request_parameter"
+    start_time_option:
+      field_name: "since"
+      inject_into: "request_parameter"
+    type: DatetimeStreamSlicer
+  retriever:
+    type: SimpleRetriever
+    name: "{{ options['name'] }}"
+    record_selector:
+      $ref: "*ref(definitions.selector)"
+    paginator:
+      type: DefaultPaginator
+      pagination_strategy:
+        type: "PageIncrement"
+        page_size: 100
+        start_from_page: 1
+      page_size_option:
+        field_name: "per_page"
+        inject_into: "request_parameter"
+      page_token_option:
+        field_name: "page"
+        inject_into: "request_parameter"
+      url_base: "*ref(definitions.requester.url_base)"
+    requester:
+      $ref: "*ref(definitions.requester)"
+    stream_slicer:
+      $ref: "*ref(definitions.stream_slicer)"
+  base_stream:
+    primary_key: "id"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+  people:
+    $ref: "*ref(definitions.base_stream)"
+    stream_cursor_field: "created_at"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      paginator:
+        type: DefaultPaginator
+        pagination_strategy:
+          type: CursorPagination
+          cursor_value: "{{ headers['link']['next']['url'] }}"
+          stop_condition: "{{ 'next' not in headers['link'] }}"
+          page_size: 100
+        page_size_option:
+          field_name: "per_page"
+          inject_into: "request_parameter"
+        page_token_option:
+          inject_into: "path"
+        url_base: "*ref(definitions.requester.url_base)"
+    $options:
+      name: "people"
+      path: "people.json"
+      stream_cursor_field: "created_at"
+  bounces:
+    $ref: "*ref(definitions.base_stream)"
+    primary_key: "person_id"
+    stream_cursor_field: "bounced_at"
+    $options:
+      stream_cursor_field: "bounced_at"
+      name: "bounces"
+      path: "bounces.json"
+  unsubscribes:
+    $ref: "*ref(definitions.base_stream)"
+    primary_key: "person_id"
+    stream_cursor_field: "unsubscribed_at"
+    $options:
+      stream_cursor_field: "unsubscribed_at"
+      name: "unsubscribes"
+      path: "unsubscribes.json"
+  survey_responses:
+    $ref: "*ref(definitions.base_stream)"
+    stream_cursor_field: "updated_at"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      stream_slicer:
+        $ref: "*ref(definitions.stream_slicer)"
+        end_time_option:
+          field_name: "updated_until"
+          inject_into: "request_parameter"
+        start_time_option:
+          field_name: "updated_since"
+          inject_into: "request_parameter"
+    $options:
+      stream_cursor_field: "updated_at"
+      name: "survey_responses"
+      path: "survey_responses.json"
+
+streams:
+  - "*ref(definitions.people)"
+  - "*ref(definitions.unsubscribes)"
+  - "*ref(definitions.bounces)"
+  - "*ref(definitions.survey_responses)"
+
+check:
+  type: CheckStream
+  stream_names: ["survey_responses"]
diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json
@@ -11,6 +11,9 @@
     "email": {
       "type": ["null", "string"]
     },
+    "phone_number": {
+      "type": ["null", "string"]
+    },
     "created_at": {
       "type": "integer"
     },