diff --git a/connectors/source_hubspot/README.md b/connectors/source_hubspot/README.md index f8751a62..df621bf2 100644 --- a/connectors/source_hubspot/README.md +++ b/connectors/source_hubspot/README.md @@ -1,9 +1,156 @@ -# Airbyte source_hubspot dbt Package +# Hubspot Support Airbyte dbt Package -This package contains dbt models for Airbyte source_hubspot source. +--- -What it includes: +- This package contains dbt models to work with Airbyte Hubspot Support connector. +- The package is compatible with latest version of Airbyte Hubspot Support connector. +- Currently, it is limited to creating transformations compatible with [Fivetran's modeling dbt package](https://github.com/fivetran/dbt_hubspot/tree/main). +- In the future, specific models will be applied directly to Airbyte connector output. If you have an idea or want to propose an analytical model for this source, please refer to the contributing guide, which explains how to propose a new transformation model. +- This package was tested with BigQuery, Snowflake, and Postgres data warehouses. -* A complete source description -* ERD model for the source -* Diagram documentation for the source +--- + +## 🎯 Intructions how to use + +### Airbyte dbt Package + +For now Airbyte dbt packages aren't versioned. You must configure using git and subdirectory. For now there isn't any transformation model directly applied to this package. But you can generate docs and tests with dbt. + +Create the following files: + +**`dbt_project.yml`** + +```yaml +vars: + using_fivetran_model: False + airbyte_database: "airbyte_db_default" + airbyte_schema: "airbyte_dbt_hubspot_support" + + hubspot_database: "airbyte_db_default" + hubspot_schema: "airbyte_dbt_source_hubspot" +``` + +**`packages.yml`** + +```yaml +packages: + - git: "https://github.com/airbytehq/airbyte-dbt-models.git" + subdirectory: "connectors/source_hubspot" +``` + +After you can run `dbt tests` or `dbt docs generate` to have a preview of Airbyte output data. + +### Fivetran Hubspot Modeling dbt package + +This package transforms Airbyte connector output data, making it compatible with Fivetran's Hubspot dbt package. You can check the analytical models Fivetran creates [here](https://github.com/fivetran/dbt_hubspot/tree/main?tab=readme-ov-file#-what-does-this-dbt-package-do). The link also provides information about how the package works and what is configurable. + +Create the require files to use Airbyte and Fivetran dbt packages: + +**`packages.yml`** + +```yaml +packages: + - git: "https://github.com/airbytehq/airbyte-dbt-models.git" + subdirectory: "connectors/source_hubspot_support" + + - package: fivetran/hubspot + version: [">=0.16.0", "<0.17.0"] +``` + +This is a default variable definition you must configure to have the models created. +At the moment this package doesn't support (schedules, domains, user tags, ticket form history and organization tags) for that reason keep those variables set to `False`. +Variables starting with the prefix `hubspot_..._identifier` represent the names of tables generated by the Airbyte connector. If you configured your sync with this prefix, ensure you edit it accordingly. + +**`dbt_project.yml`** + +```yaml +vars: + # Required by Airbyte dbt model + using_fivetran_model: True + airbyte_database: "airbyte_db_default" + airbyte_schema: "airbyte_dbt_source_hubspot" + + # Required by Fivetran dbt model + hubspot_database: "airbyte_db_default" + hubspot_schema: "airbyte_dbt_source_hubspot" + + hubspot_marketing_enabled: false # Disables all marketing models + hubspot_contact_enabled: false # Disables the contact models + hubspot_contact_list_enabled: false # Disables contact list models + hubspot_contact_list_member_enabled: false # Disables contact list member models + hubspot_contact_property_enabled: false # Disables the contact property models + hubspot_contact_property_history_enabled: false # Disables the contact property history models + hubspot_email_event_enabled: false # Disables all email_event models and functionality + hubspot_email_event_bounce_enabled: false + hubspot_email_event_click_enabled: false + hubspot_email_event_deferred_enabled: false + hubspot_email_event_delivered_enabled: false + hubspot_email_event_dropped_enabled: false + hubspot_email_event_forward_enabled: false + hubspot_email_event_open_enabled: false + hubspot_email_event_print_enabled: false + hubspot_email_event_sent_enabled: false + hubspot_email_event_spam_report_enabled: false + hubspot_email_event_status_change_enabled: false + + hubspot_contact_merge_audit_enabled: false # Enables the use of the CONTACT_MERGE_AUDIT table (deprecated by Hubspot v3 API) for removing merged contacts in the final models. + # If false, ~~~contacts will still be merged~~~, but using the CONTACT.property_hs_calculated_merged_vids field (introduced in v3 of the Hubspot CRM API) + # Default = false + # Sales + + hubspot_sales_enabled: false # Disables all sales models + hubspot_company_enabled: false + hubspot_company_property_history_enabled: false # Disable the company property history models + hubspot_deal_enabled: false + hubspot_merged_deal_enabled: false # Enables the merged_deal table, which will be used to filter out merged deals from the final deal models. False by default. Note that `hubspot_sales_enabled` and `hubspot_deal_enabled` must not be set to False. + hubspot_deal_company_enabled: false + hubspot_deal_contact_enabled: false + hubspot_deal_property_history_enabled: false # Disables the deal property history tables + hubspot_engagement_enabled: false # Disables all engagement models and functionality + hubspot_engagement_contact_enabled: false + hubspot_engagement_company_enabled: false + hubspot_engagement_deal_enabled: false + hubspot_engagement_call_enabled: false + hubspot_engagement_email_enabled: false + hubspot_engagement_meeting_enabled: false + hubspot_engagement_note_enabled: false + hubspot_engagement_task_enabled: false + hubspot_owner_enabled: false + hubspot_property_enabled: false # Disables property and property_option tables + + # Service + hubspot_service_enabled: false # Enables all service models + hubspot_ticket_deal_enabled: false + + contact_identifier: "contacts" + contact_lists_identifier: "contact_lists" + deals_identifier: "deals" + companies_identifier: "companies" + company_property_history_identifier: "companies_property_history" + contact_list_membership_identifier: "contacts_list_memberships" + contact_property_history_identifier: "contacts_property_history" + deal_pipeline_identifier: "deal_pipelines" + deal_property_history_identifier: "deals_property_history" + email_event_identifier: "email_events" + engagements_calls_identifier: "engagements_calls" + engagement_email_identifier: "engagement_emails" + ticket_pipeline_identifier: "ticket_pipelines" + tickets_identifier: "tickets" + engagement_note_identifier: "engagement_notes" + +``` + +After run `dbt run`, you can see the models being created. + +--- + +## :package: Package Maintenance + +- This package is maintained by the Airbyte Community. +- You can contribute any time please read the Contributing Guidelines or enter the Airbyte Slack Channel `#airbyte-dbt-packages` + +## Supported models + +- `contact_list` +- `deal` +- \ No newline at end of file diff --git a/connectors/source_hubspot/integration_tests/dbt_project.yml b/connectors/source_hubspot/integration_tests/dbt_project.yml new file mode 100644 index 00000000..6dcd7acd --- /dev/null +++ b/connectors/source_hubspot/integration_tests/dbt_project.yml @@ -0,0 +1,107 @@ +name: integration_test_hubspot + +config-version: 2 + +version: 0.1.0 + +profile: integration_tests + +model-paths: + - models + +macro-paths: + - macros + +target-path: target + +clean-targets: + - target + - dbt_modules + - logs + +require-dbt-version: + - ">=1.0.0" + - <2.0.0 + +models: + airbyte_dbt_source_amplitude: + materialized: view + +schema: dbt_source_hubspot + staging: + materialized: view + tmp: + materialized: view + +vars: + # Required by Airbyte dbt model + using_fivetran_model: True + airbyte_database: "airbyte_db_default" + airbyte_schema: "airbyte_dbt_source_hubspot" + + # Required by Fivetran dbt model + hubspot_database: "airbyte_db_default" + hubspot_schema: "airbyte_dbt_source_hubspot" + + hubspot_marketing_enabled: false # Disables all marketing models + hubspot_contact_enabled: false # Disables the contact models + hubspot_contact_list_enabled: false # Disables contact list models + hubspot_contact_list_member_enabled: false # Disables contact list member models + hubspot_contact_property_enabled: false # Disables the contact property models + hubspot_contact_property_history_enabled: false # Disables the contact property history models + hubspot_email_event_enabled: false # Disables all email_event models and functionality + hubspot_email_event_bounce_enabled: false + hubspot_email_event_click_enabled: false + hubspot_email_event_deferred_enabled: false + hubspot_email_event_delivered_enabled: false + hubspot_email_event_dropped_enabled: false + hubspot_email_event_forward_enabled: false + hubspot_email_event_open_enabled: false + hubspot_email_event_print_enabled: false + hubspot_email_event_sent_enabled: false + hubspot_email_event_spam_report_enabled: false + hubspot_email_event_status_change_enabled: false + + hubspot_contact_merge_audit_enabled: false # Enables the use of the CONTACT_MERGE_AUDIT table (deprecated by Hubspot v3 API) for removing merged contacts in the final models. + # If false, ~~~contacts will still be merged~~~, but using the CONTACT.property_hs_calculated_merged_vids field (introduced in v3 of the Hubspot CRM API) + # Default = false + # Sales + + hubspot_sales_enabled: false # Disables all sales models + hubspot_company_enabled: false + hubspot_company_property_history_enabled: false # Disable the company property history models + hubspot_deal_enabled: false + hubspot_merged_deal_enabled: false # Enables the merged_deal table, which will be used to filter out merged deals from the final deal models. False by default. Note that `hubspot_sales_enabled` and `hubspot_deal_enabled` must not be set to False. + hubspot_deal_company_enabled: false + hubspot_deal_contact_enabled: false + hubspot_deal_property_history_enabled: false # Disables the deal property history tables + hubspot_engagement_enabled: false # Disables all engagement models and functionality + hubspot_engagement_contact_enabled: false + hubspot_engagement_company_enabled: false + hubspot_engagement_deal_enabled: false + hubspot_engagement_call_enabled: false + hubspot_engagement_email_enabled: false + hubspot_engagement_meeting_enabled: false + hubspot_engagement_note_enabled: false + hubspot_engagement_task_enabled: false + hubspot_owner_enabled: false + hubspot_property_enabled: false # Disables property and property_option tables + + # Service + hubspot_service_enabled: false # Enables all service models + hubspot_ticket_deal_enabled: false + + contact_identifier: "contacts" + contact_lists_identifier: "contact_lists" + deals_identifier: "deals" + companies_identifier: "companies" + company_property_history_identifier: "companies_property_history" + contact_list_membership_identifier: "contacts_list_memberships" + contact_property_history_identifier: "contacts_property_history" + deal_pipeline_identifier: "deal_pipelines" + deal_property_history_identifier: "deals_property_history" + email_event_identifier: "email_events" + engagements_calls_identifier: "engagements_calls" + engagement_email_identifier: "engagement_emails" + ticket_pipeline_identifier: "ticket_pipelines" + tickets_identifier: "tickets" + engagement_note_identifier: "engagement_notes" diff --git a/connectors/source_hubspot/integration_tests/package-lock.yml b/connectors/source_hubspot/integration_tests/package-lock.yml new file mode 100644 index 00000000..a11c465f --- /dev/null +++ b/connectors/source_hubspot/integration_tests/package-lock.yml @@ -0,0 +1,13 @@ +packages: + - local: ../ + - package: fivetran/hubspot + version: 0.18.0 + - package: fivetran/hubspot_source + version: 0.15.0 + - package: fivetran/fivetran_utils + version: 0.4.10 + - package: dbt-labs/spark_utils + version: 0.3.0 + - package: dbt-labs/dbt_utils + version: 1.2.0 +sha1_hash: 47c91937972f9cab62436ed1528654bc219bf3b8 diff --git a/connectors/source_hubspot/integration_tests/packages.yml b/connectors/source_hubspot/integration_tests/packages.yml new file mode 100644 index 00000000..c492aaac --- /dev/null +++ b/connectors/source_hubspot/integration_tests/packages.yml @@ -0,0 +1,11 @@ +packages: + - local: ../ + + - package: fivetran/hubspot + version: [">=0.18.0", "<0.19.0"] + + - package: fivetran/hubspot_source + version: [">=0.15.0", "<0.16.0"] + + - package: fivetran/fivetran_utils + version: [">=0.4.0", "<0.5.0"] diff --git a/connectors/source_hubspot/integration_tests/vars b/connectors/source_hubspot/integration_tests/vars new file mode 100644 index 00000000..f64b7801 --- /dev/null +++ b/connectors/source_hubspot/integration_tests/vars @@ -0,0 +1 @@ +{airbyte_database: $AB_DB, hubspot_database: $AB_DB} diff --git a/connectors/source_hubspot/models/fivetran_converter/company.sql b/connectors/source_hubspot/models/fivetran_converter/company.sql new file mode 100644 index 00000000..ac0768fa --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/company.sql @@ -0,0 +1,17 @@ +SELECT + id AS company_id, + archived AS is_company_deleted, + {{ dbt.current_timestamp() }} AS _fivetran_synced, + properties_name AS company_name, + properties_description AS description, + CAST(properties_createdate AS {{ dbt.type_timestamp() }}) AS created_date, + properties_industry AS industry, + properties_address AS street_address, + properties_address2 AS street_address_2, + properties_city AS city, + properties_state AS state, + properties_country AS country, + CAST(properties_annualrevenue AS {{ dbt.type_float() }}) AS company_annual_revenue +FROM + {{ source('source_hubspot', 'companies') }} + diff --git a/connectors/source_hubspot/models/fivetran_converter/company.yaml b/connectors/source_hubspot/models/fivetran_converter/company.yaml new file mode 100644 index 00000000..a41e67f1 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/company.yaml @@ -0,0 +1,43 @@ +version: 2 + +models: + - name: company + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('company_identifier', 'company') }}" + description: Transformed data from HubSpot companies table. + config: + +materialized: table + +schema: hubspot + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: company_id + description: Unique identifier for the company + tests: + - unique + - not_null + - name: is_company_deleted + description: Indicates whether the company is archived or active + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + - name: company_name + description: The name of the company + - name: description + description: Company description + - name: created_date + description: Date and time when the company was created + - name: industry + description: The industry of the company + - name: street_address + description: Primary street address of the company + - name: street_address_2 + description: Secondary street address of the company + - name: city + description: City where the company is located + - name: state + description: State where the company is located + - name: country + description: Country where the company is located + - name: company_annual_revenue + description: Annual revenue of the company + diff --git a/connectors/source_hubspot/models/fivetran_converter/company_property_history.sql b/connectors/source_hubspot/models/fivetran_converter/company_property_history.sql new file mode 100644 index 00000000..5ce9fee1 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/company_property_history.sql @@ -0,0 +1,12 @@ +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + companyId AS company_id, + property AS field_name, + sourceType AS change_source, + sourceId AS change_source_id, + CAST(timestamp AS {{ dbt.type_timestamp() }}) AS change_timestamp, + value AS new_value, + updatedByUserId AS updated_by_user_id, + archived AS is_archived +FROM + {{ source('source_hubspot', 'companies_property_history') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/company_property_history.yml b/connectors/source_hubspot/models/fivetran_converter/company_property_history.yml new file mode 100644 index 00000000..1cc12001 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/company_property_history.yml @@ -0,0 +1,29 @@ +version: 2 + +models: + - name: company_property_history + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('company_property_history_identifier', 'company_property_history') }}" + description: All fields and field values associated with company property history in HubSpot. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + - name: company_id + description: The unique identifier of the company to which the property history record belongs + - name: field_name + description: The specific property that was updated in the company record + - name: change_source + description: The type of the source that updated the property in the company record + - name: change_source_id + description: The identifier of the source that updated the property in the company record + - name: change_timestamp + description: The date and time when the property update occurred + - name: new_value + description: The new value of the property after the update + - name: updated_by_user_id + description: The user ID of the user who initiated the property update + - name: is_archived + description: Flag indicating if the company property history record is archived or not diff --git a/connectors/source_hubspot/models/fivetran_converter/contact.sql b/connectors/source_hubspot/models/fivetran_converter/contact.sql new file mode 100644 index 00000000..b2d001dc --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact.sql @@ -0,0 +1,14 @@ +SELECT + id AS contact_id, + properties_hs_is_contact AS is_contact_deleted, + properties_hs_calculated_merged_vids AS calculated_merged_vids, + properties_email AS email, + properties_company AS contact_company, + properties_firstname AS first_name, + properties_lastname AS last_name, + properties_createdate AS created_date, + properties_jobtitle AS job_title, + properties_annualrevenue AS company_annual_revenue, + {{ dbt.current_timestamp() }} AS _fivetran_synced +FROM + {{ source('source_hubspot', 'contacts') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/contact.yml b/connectors/source_hubspot/models/fivetran_converter/contact.yml new file mode 100644 index 00000000..6724ee3e --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact.yml @@ -0,0 +1,33 @@ +version: 2 + +models: + - name: contact + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('contact_identifier', 'contact') }}" + description: All fields and field values associated with HubSpot contacts. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: contact_id + description: Unique identifier for the contact + - name: is_contact_deleted + description: Indicates if the contact has been deleted + - name: calculated_merged_vids + description: Calculated merged visitor IDs + - name: email + description: Email address of the contact + - name: contact_company + description: Company associated with the contact + - name: first_name + description: First name of the contact + - name: last_name + description: Last name of the contact + - name: created_date + description: Timestamp when the contact was created + - name: job_title + description: Job title of the contact + - name: company_annual_revenue + description: Annual revenue of the company associated with the contact + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran diff --git a/connectors/source_hubspot/models/fivetran_converter/contact_list.sql b/connectors/source_hubspot/models/fivetran_converter/contact_list.sql new file mode 100644 index 00000000..aece563f --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact_list.sql @@ -0,0 +1,18 @@ +SELECT + _airbyte_extracted_at AS _fivetran_synced, + listId AS list_id, + name, + portalId AS portal_id, + createdAt AS created_at, + updatedAt AS updated_at, + dynamic, + internal, + deleteable, + archived, + metaData_processing AS metadata_processing, + metaData_size AS metadata_size, + metaData_error AS metadata_error, + metaData_lastProcessingStateChangeAt AS metadata_last_processing_state_change_at, + metaData_lastSizeChangeAt AS metadata_last_size_change_at +FROM + {{ source('source_hubspot', 'contact_lists') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/contact_list.yml b/connectors/source_hubspot/models/fivetran_converter/contact_list.yml new file mode 100644 index 00000000..c207549e --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact_list.yml @@ -0,0 +1,41 @@ +version: 2 + +models: + - name: contact_list + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('contact_list_identifier', 'contact_list') }}" + description: Fields and field values associated with HubSpot contact lists. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _airbyte_extracted_at + description: Timestamp of when this record was extracted by Airbyte + - name: list_id + description: The unique ID of the contact list + - name: name + description: The name or title of the contact list + - name: portal_id + description: The ID of the portal to which the contact list belongs + - name: created_at + description: The timestamp when the contact list was created + - name: updated_at + description: The timestamp of the last update to the contact list + - name: dynamic + description: Identifies if the contact list is dynamic in nature + - name: internal + description: Specifies if the contact list is internal (not accessible to customers) + - name: deleteable + description: Specifies if the contact list can be deleted + - name: archived + description: Indicates if the contact list is archived or not + - name: metadata_processing + description: Processing status related to list metadata + - name: metadata_size + description: Size of the list metadata + - name: metadata_error + description: Error information related to list metadata + - name: metadata_last_processing_state_change_at + description: Timestamp of the last processing state change for metadata + - name: metadata_last_size_change_at + description: Timestamp of the last size change for metadata \ No newline at end of file diff --git a/connectors/source_hubspot/models/fivetran_converter/contact_list_member.sql b/connectors/source_hubspot/models/fivetran_converter/contact_list_member.sql new file mode 100644 index 00000000..08fab306 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact_list_member.sql @@ -0,0 +1,7 @@ +SELECT + 'FALSE' AS is_contact_list_member_deleted, + {{ dbt.current_timestamp() }} AS _fivetran_synced, + NULL AS contact_id, + NULL AS contact_list_id +FROM + {{ source('source_hubspot', 'contacts_list_memberships') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/contact_list_member.yml b/connectors/source_hubspot/models/fivetran_converter/contact_list_member.yml new file mode 100644 index 00000000..79177bd3 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact_list_member.yml @@ -0,0 +1,21 @@ +version: 2 + +models: + - name: contact_list_member + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('contact_list_membership_identifier', 'contact_list_member') }}" + description: Represents the membership of contacts in various contact lists. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + - name: added_timestamp + description: The timestamp when the contact was added to the list + - name: contact_id + description: The unique ID associated with the contact in the HubSpot CRM (formerly 'canonical-vid' or 'vid') + - name: contact_list_id + description: The ID of the contact list (formerly 'static-list-id' or 'internal-list-id') + - name: is_contact_list_member_deleted + description: Indicates whether this membership record has been deleted diff --git a/connectors/source_hubspot/models/fivetran_converter/contact_property_history.sql b/connectors/source_hubspot/models/fivetran_converter/contact_property_history.sql new file mode 100644 index 00000000..e544b490 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact_property_history.sql @@ -0,0 +1,16 @@ +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + vid AS contact_id, + properties_firstname AS field_name, + {% if target.type == "postgres" %} + "source-id" AS change_source, + {% elif target.type == "snowflake" %} + value AS change_source, + {% elif target.type == "bigquery" %} + `source_id` AS change_source, + {% endif %} + properties_hs_created_by_user_id AS change_source_id, + timestamp AS change_timestamp, + value AS new_value +FROM + {{ source('source_hubspot', 'contacts_property_history') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/contact_property_history.yml b/connectors/source_hubspot/models/fivetran_converter/contact_property_history.yml new file mode 100644 index 00000000..7468d432 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/contact_property_history.yml @@ -0,0 +1,33 @@ +version: 2 + +models: + - name: contact_property_history + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('contact_property_history_identifier', 'contact_property_history') }}" + description: Historical data of property changes for HubSpot contacts. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + data_type: timestamp + - name: contact_id + description: The unique identifier of the contact + data_type: integer + - name: field_name + description: The name of the property that was changed + data_type: string + - name: change_source + description: The source of the property change + data_type: string + - name: change_source_id + description: The identifier of the change source + data_type: string + - name: change_timestamp + description: The timestamp when the property was changed + data_type: timestamp + - name: new_value + description: The new value of the property after the change + data_type: string + \ No newline at end of file diff --git a/connectors/source_hubspot/models/fivetran_converter/deal.sql b/connectors/source_hubspot/models/fivetran_converter/deal.sql new file mode 100644 index 00000000..5351d70e --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/deal.sql @@ -0,0 +1,20 @@ +SELECT + id AS deal_id, + properties_dealname AS deal_name, + CAST(properties_closedate AS {{ dbt.type_timestamp() }}) AS closed_date, + CAST(properties_createdate AS {{ dbt.type_timestamp() }}) AS created_date, + {{ dbt.current_timestamp() }} AS _fivetran_synced, + CAST(properties_pipeline AS {{ dbt.type_string() }}) AS deal_pipeline_id, + CAST(properties_dealstage AS {{ dbt.type_string() }}) AS deal_pipeline_stage_id, + properties_hubspot_owner_id AS owner_id, + CAST(properties_hs_object_id AS {{ dbt.type_string() }}) AS portal_id, + properties_description AS description, + CAST(properties_amount AS {{ dbt.type_float() }}) AS amount, + CAST(properties_hs_is_closed AS {{ dbt.type_boolean() }}) AS is_closed, + CAST(properties_hs_is_closed_won AS {{ dbt.type_boolean() }}) AS is_won, + CAST(archived AS {{ dbt.type_boolean() }}) AS is_deal_deleted, + CAST(properties_hs_created_by_user_id AS {{ dbt.type_string() }}) AS created_by_user_id, + CAST(properties_hs_lastmodifieddate AS {{ dbt.type_timestamp() }}) AS last_modified_date, + CAST(properties_hs_updated_by_user_id AS {{ dbt.type_string() }}) AS updated_by_user_id +FROM + {{ source('source_hubspot', 'deals') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/deal.yml b/connectors/source_hubspot/models/fivetran_converter/deal.yml new file mode 100644 index 00000000..7f63fcfd --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/deal.yml @@ -0,0 +1,48 @@ +version: 2 + +models: + - name: deal + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('deal_identifier', 'deal') }}" + description: Transformed data for HubSpot deals + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: deal_id + description: Unique identifier for the deal + tests: + - unique + - not_null + - name: deal_name + description: The name of the deal + - name: closed_date + description: The date when the deal was closed + - name: created_date + description: The date when the deal was created + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + - name: deal_pipeline_id + description: The ID of the pipeline the deal belongs to + - name: deal_pipeline_stage_id + description: The ID of the pipeline stage the deal is in + - name: owner_id + description: The ID of the deal owner + - name: portal_id + description: The ID of the HubSpot portal + - name: description + description: The description of the deal + - name: amount + description: The monetary amount of the deal + - name: is_closed + description: Indicates if the deal is closed + - name: is_won + description: Indicates if the deal is won + - name: is_deal_deleted + description: Indicates if the deal has been deleted + - name: created_by_user_id + description: The ID of the user who created the deal + - name: last_modified_date + description: The date when the deal was last modified + - name: updated_by_user_id + description: The ID of the user who last updated the deal diff --git a/connectors/source_hubspot/models/fivetran_converter/deal_pipeline.sql b/connectors/source_hubspot/models/fivetran_converter/deal_pipeline.sql new file mode 100644 index 00000000..11bc0e47 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/deal_pipeline.sql @@ -0,0 +1,11 @@ +SELECT + NOT active AS is_deal_pipeline_deleted, + {{ dbt.current_timestamp() }} AS _fivetran_synced, + active AS is_active, + displayOrder AS display_order, + label AS pipeline_label, + pipelineId AS deal_pipeline_id, + createdAt AS deal_pipeline_created_at, + updatedAt AS deal_pipeline_updated_at +FROM + {{ source('source_hubspot', 'deal_pipelines') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/deal_pipeline.yml b/connectors/source_hubspot/models/fivetran_converter/deal_pipeline.yml new file mode 100644 index 00000000..a8398cc8 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/deal_pipeline.yml @@ -0,0 +1,27 @@ +version: 2 + +models: + - name: deal_pipeline + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('deal_pipeline_identifier', 'deal_pipeline') }}" + description: All fields and field values associated with HubSpot deal pipelines. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + - name: is_deal_pipeline_deleted + description: Indicates whether the deal pipeline has been deleted + - name: is_active + description: Indicates if the deal pipeline is currently active or not + - name: display_order + description: The ordering of the deal pipeline for display + - name: pipeline_label + description: The label or name of the deal pipeline + - name: deal_pipeline_id + description: The unique identifier of the deal pipeline + - name: deal_pipeline_created_at + description: Timestamp for the creation date of the deal pipeline + - name: deal_pipeline_updated_at + description: Timestamp for the last update to the deal pipeline diff --git a/connectors/source_hubspot/models/fivetran_converter/deal_property_history.sql b/connectors/source_hubspot/models/fivetran_converter/deal_property_history.sql new file mode 100644 index 00000000..f289d92f --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/deal_property_history.sql @@ -0,0 +1,10 @@ +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + dealId AS deal_id, + property AS field_name, + sourceType AS change_source, + sourceId AS change_source_id, + {{ dbt.current_timestamp() }} AS change_timestamp, + value AS new_value +FROM + {{ source('source_hubspot', 'deals_property_history') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/deal_property_history.yml b/connectors/source_hubspot/models/fivetran_converter/deal_property_history.yml new file mode 100644 index 00000000..0d69fe0d --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/deal_property_history.yml @@ -0,0 +1,33 @@ +version: 2 + +models: + - name: deal_property_history + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('deal_property_history_identifier', 'deal_property_history') }}" + description: Historical changes to deal properties in HubSpot. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + tests: + - not_null + - name: deal_id + description: The unique identifier of the deal associated with this property history + tests: + - not_null + - name: field_name + description: The name of the property that was updated + tests: + - not_null + - name: change_source + description: The type of source that triggered this update + - name: change_source_id + description: The unique identifier of the source of this update + - name: change_timestamp + description: The date and time when the property was updated + tests: + - not_null + - name: new_value + description: The new value of the property after the change diff --git a/connectors/source_hubspot/models/fivetran_converter/email_event.sql b/connectors/source_hubspot/models/fivetran_converter/email_event.sql new file mode 100644 index 00000000..ae1dde33 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/email_event.sql @@ -0,0 +1,64 @@ +{% if target.type == "postgres" %} + +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + "appId" AS app_id, + causedBy->>'created' AS caused_timestamp, + causedBy->>'id' AS caused_by_event_id, + created AS created_timestamp, + "emailCampaignId" AS email_campaign_id, + "filteredEvent" AS is_filtered_event, + id AS event_id, + obsoleted_by->>'created' AS obsoleted_timestamp, + obsoleted_by->>'id' AS obsoleted_by_event_id, + "portalId" AS portal_id, + recipient AS recipient_email_address, + sent_by->>'created' AS sent_timestamp, + sent_by->>'id' AS sent_by_event_id, + type AS event_type +FROM + {{ source('source_hubspot', 'email_events') }} + +{% elif target.type == "snowflake" %} + +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + appId AS app_id, + causedBy:created AS caused_timestamp, + causedBy:id AS caused_by_event_id, + created AS created_timestamp, + emailCampaignId AS email_campaign_id, + filteredEvent AS is_filtered_event, + id AS event_id, + obsoletedBy:created AS obsoleted_timestamp, + obsoletedBy:id AS obsoleted_by_event_id, + portalId AS portal_id, + recipient AS recipient_email_address, + sentBy:created AS sent_timestamp, + sentBy:id AS sent_by_event_id, + type AS event_type +FROM + {{ source('source_hubspot', 'email_events') }} + +{% elif target.type == "bigquery" %} + +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + appId AS app_id, + JSON_EXTRACT_SCALAR(causedBy, '$.created') AS caused_timestamp, + JSON_EXTRACT_SCALAR(causedBy, '$.id') AS caused_by_event_id, + created AS created_timestamp, + emailCampaignId AS email_campaign_id, + filteredEvent AS is_filtered_event, + id AS event_id, + JSON_EXTRACT_SCALAR(obsoletedBy, '$.created') AS obsoleted_timestamp, + JSON_EXTRACT_SCALAR(obsoletedBy, '$.id') AS obsoleted_by_event_id, + portalId AS portal_id, + recipient AS recipient_email_address, + JSON_EXTRACT_SCALAR(sentBy, '$.created') AS sent_timestamp, + JSON_EXTRACT_SCALAR(sentBy, '$.id') AS sent_by_event_id, + type AS event_type +FROM + {{ source('source_hubspot', 'email_events') }} + +{% endif %} \ No newline at end of file diff --git a/connectors/source_hubspot/models/fivetran_converter/email_event.yml b/connectors/source_hubspot/models/fivetran_converter/email_event.yml new file mode 100644 index 00000000..9ae364e5 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/email_event.yml @@ -0,0 +1,41 @@ +version: 2 + +models: + - name: email_event + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('email_event_identifier', 'email_event') }}" + description: Fields and field values associated with HubSpot email events. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + - name: app_id + description: The unique identifier of the application associated with the email event + - name: caused_timestamp + description: The timestamp of when the causing event was created + - name: caused_by_event_id + description: The ID of the event that caused this email event + - name: created_timestamp + description: The timestamp of when the email event was created + - name: email_campaign_id + description: The ID of the email campaign associated with the event + - name: is_filtered_event + description: Indicates if the event is filtered + - name: event_id + description: The unique identifier of the email event + - name: obsoleted_timestamp + description: The timestamp of when this event was obsoleted + - name: obsoleted_by_event_id + description: The ID of the event that obsoleted this email event + - name: portal_id + description: The ID of the HubSpot portal + - name: recipient_email_address + description: The recipient of the email + - name: sent_timestamp + description: The timestamp of when the email was sent + - name: sent_by_event_id + description: The ID of the event that sent this email + - name: event_type + description: The type of email event diff --git a/connectors/source_hubspot/models/fivetran_converter/engagement_call.sql b/connectors/source_hubspot/models/fivetran_converter/engagement_call.sql new file mode 100644 index 00000000..fe4c34e8 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/engagement_call.sql @@ -0,0 +1,11 @@ +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + 'FALSE' AS _fivetran_deleted, + id AS engagement_id, + 'CALL' AS engagement_type, + createdAt AS created_timestamp, + properties_hs_timestamp AS occurred_timestamp, + properties_hubspot_owner_id AS owner_id, + properties_hubspot_team_id AS team_id +FROM + {{ source('source_hubspot', 'engagements_calls') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/engagement_call.yml b/connectors/source_hubspot/models/fivetran_converter/engagement_call.yml new file mode 100644 index 00000000..eb85c312 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/engagement_call.yml @@ -0,0 +1,27 @@ +version: 2 + +models: + - name: engagement_call + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('engagement_call_identifier', 'engagement_call') }}" + description: Fields and field values associated with HubSpot engagement calls. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when Fivetran synced a record + - name: _fivetran_deleted + description: Boolean to mark rows that were deleted in the source database + - name: engagement_id + description: The ID of the engagement + - name: engagement_type + description: The type of the engagement + - name: created_timestamp + description: Timestamp of call creation + - name: occurred_timestamp + description: Timestamp of call occurrence + - name: owner_id + description: The ID of the owner associated with the call + - name: team_id + description: The ID of the team associated with the call diff --git a/connectors/source_hubspot/models/fivetran_converter/engagement_email.sql b/connectors/source_hubspot/models/fivetran_converter/engagement_email.sql new file mode 100644 index 00000000..d9822fff --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/engagement_email.sql @@ -0,0 +1,17 @@ +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + 'FALSE' AS _fivetran_deleted, + 'EMAIL' AS engagement_type, + id AS engagement_id, + createdAt AS created_timestamp, + properties_hs_timestamp AS occurred_timestamp, + properties_hubspot_owner_id AS owner_id, + properties_hubspot_team_id AS team_id, + properties_hs_all_owner_ids AS all_owner_ids, + properties_hs_all_team_ids AS all_team_ids, + properties_hs_email_subject AS email_subject, + properties_hs_email_text AS email_text, + properties_hs_lastmodifieddate AS lastmodifieddate, + properties_hs_modified_by AS modified_by +FROM + {{ source('source_hubspot', 'engagements_emails') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/engagement_email.yml b/connectors/source_hubspot/models/fivetran_converter/engagement_email.yml new file mode 100644 index 00000000..209c516c --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/engagement_email.yml @@ -0,0 +1,39 @@ +version: 2 + +models: + - name: engagement_email + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('engagement_email_identifier', 'engagement_email') }}" + description: Fields and field values associated with HubSpot engagement emails. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when Fivetran synced a record + - name: _fivetran_deleted + description: Boolean to mark rows that were deleted in the source database + - name: engagement_type + description: The type of the engagement + - name: engagement_id + description: The ID of the engagement + - name: created_timestamp + description: Timestamp of email creation + - name: occurred_timestamp + description: Timestamp of email occurrence + - name: owner_id + description: The ID of the owner associated with the email + - name: team_id + description: The ID of the team associated with the email + - name: all_owner_ids + description: All owner IDs associated with the email + - name: all_team_ids + description: All team IDs associated with the email + - name: email_subject + description: Subject of the email + - name: email_text + description: Text content of the email + - name: lastmodifieddate + description: Date the email was last modified + - name: modified_by + description: ID of the user who last modified the email diff --git a/connectors/source_hubspot/models/fivetran_converter/engagement_note.sql b/connectors/source_hubspot/models/fivetran_converter/engagement_note.sql new file mode 100644 index 00000000..08cf1503 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/engagement_note.sql @@ -0,0 +1,15 @@ +SELECT + {{ dbt.current_timestamp() }} AS _fivetran_synced, + 'FALSE' AS _fivetran_deleted, + properties_hs_note_body AS note, + 'note' AS engagement_type, + id AS engagement_id, + properties_hs_createdate AS created_timestamp, + properties_hs_timestamp AS occurred_timestamp, + properties_hs_created_by AS owner_id, + properties_hubspot_team_id AS team_id, + properties_hs_body_preview AS body_preview, + properties_hs_lastmodifieddate AS lastmodifieddate, + properties_hs_note_body AS note_body +FROM + {{ source('source_hubspot', 'engagements_notes') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/engagement_note.yml b/connectors/source_hubspot/models/fivetran_converter/engagement_note.yml new file mode 100644 index 00000000..49057896 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/engagement_note.yml @@ -0,0 +1,35 @@ +version: 2 + +models: + - name: engagement_note + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('engagement_note_identifier', 'engagement_note') }}" + description: All fields and field values associated with HubSpot engagement notes. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when Fivetran synced a record + - name: _fivetran_deleted + description: Boolean to mark rows that were deleted in the source database + - name: note + description: The body of the note. The body has a limit of 65536 characters. + - name: engagement_type + description: The type of the engagement + - name: engagement_id + description: The ID of the engagement + - name: created_timestamp + description: This field marks the note's time of creation and determines where the note sits on the record timeline + - name: occurred_timestamp + description: This field marks the note's time of occurrence and determines where the note sits on the record timeline + - name: owner_id + description: The ID of the owner associated with the note + - name: team_id + description: The ID of the team associated with the note + - name: body_preview + description: A preview of the note body + - name: lastmodifieddate + description: The date the note was last modified + - name: note_body + description: The full body of the note diff --git a/connectors/source_hubspot/models/fivetran_converter/ticket.sql b/connectors/source_hubspot/models/fivetran_converter/ticket.sql new file mode 100644 index 00000000..5bba9aef --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/ticket.sql @@ -0,0 +1,16 @@ +SELECT + id AS ticket_id, + 'FALSE' AS is_ticket_deleted, + {{ dbt.current_timestamp() }} AS _fivetran_synced, + properties_closed_date AS closed_date, + properties_createdate AS created_date, + properties_first_agent_reply_date AS first_agent_reply_at, + properties_hs_pipeline AS ticket_pipeline_id, + properties_hs_pipeline_stage AS ticket_pipeline_stage_id, + properties_hs_ticket_category AS ticket_category, + properties_hs_ticket_priority AS ticket_priority, + properties_hubspot_owner_id AS owner_id, + properties_subject AS ticket_subject, + properties_content AS ticket_content +FROM + {{ source('source_hubspot', 'tickets') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/ticket.yml b/connectors/source_hubspot/models/fivetran_converter/ticket.yml new file mode 100644 index 00000000..d1605e65 --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/ticket.yml @@ -0,0 +1,37 @@ +version: 2 + +models: + - name: ticket + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('tickets_identifier', 'ticket') }}" + description: Transformed ticket data from Hubspot. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: ticket_id + description: ID of the ticket + - name: is_ticket_deleted + description: Boolean indicating whether a record has been deleted in Hubspot or inferred deleted by Fivetran + - name: _fivetran_synced + description: Timestamp of when Fivetran synced a record + - name: closed_date + description: The date the ticket was closed + - name: created_date + description: The date the ticket was created + - name: first_agent_reply_at + description: The date for the first agent reply on the ticket + - name: ticket_pipeline_id + description: The ID of the ticket's pipeline + - name: ticket_pipeline_stage_id + description: The ID of the ticket's pipeline stage + - name: ticket_category + description: The category of the ticket + - name: ticket_priority + description: The priority of the ticket + - name: owner_id + description: The ID of the ticket's owner + - name: ticket_subject + description: Short summary of ticket + - name: ticket_content + description: Text in body of the ticket \ No newline at end of file diff --git a/connectors/source_hubspot/models/fivetran_converter/ticket_pipeline.sql b/connectors/source_hubspot/models/fivetran_converter/ticket_pipeline.sql new file mode 100644 index 00000000..ad361e8f --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/ticket_pipeline.sql @@ -0,0 +1,12 @@ +SELECT + id AS ticket_pipeline_id, + 'FALSE' AS is_ticket_pipeline_deleted, + {{ dbt.current_timestamp() }} AS _fivetran_synced, + NOT archived AS is_active, + displayOrder AS display_order, + label AS pipeline_label, + id AS object_type_id, + createdAt AS ticket_pipeline_created_at, + updatedAt AS ticket_pipeline_updated_at +FROM + {{ source('source_hubspot', 'ticket_pipelines') }} diff --git a/connectors/source_hubspot/models/fivetran_converter/ticket_pipeline.yml b/connectors/source_hubspot/models/fivetran_converter/ticket_pipeline.yml new file mode 100644 index 00000000..e085c3cd --- /dev/null +++ b/connectors/source_hubspot/models/fivetran_converter/ticket_pipeline.yml @@ -0,0 +1,29 @@ +version: 2 + +models: + - name: ticket_pipeline + schema: "{{ var('airbyte_schema', target.schema) }}" + database: "{{ var('airbyte_database', target.database) }}" + identifier: "{{ var('ticket_pipeline_identifier', 'ticket_pipeline') }}" + description: All fields and field values associated with HubSpot ticket pipelines. + config: + +enabled: "{{ var('using_fivetran_model', False) }}" + columns: + - name: ticket_pipeline_id + description: The unique identifier of the ticket pipeline + - name: is_ticket_pipeline_deleted + description: Indicates whether the ticket pipeline has been deleted + - name: _fivetran_synced + description: Timestamp of when this record was last synced by Fivetran + - name: is_active + description: Indicates if the ticket pipeline is active + - name: display_order + description: The order in which the ticket pipeline is displayed + - name: pipeline_label + description: The label or name of the ticket pipeline + - name: object_type_id + description: The ID of the object type associated with this pipeline + - name: ticket_pipeline_created_at + description: The date and time when the ticket pipeline was created + - name: ticket_pipeline_updated_at + description: The date and time when the ticket pipeline was last updated