Feature/Contacts and StudentContactAssociations (#96)

* redraft

* add student contact associations

* switch from parent to contact

* add comments about name change

* can't use gsn_skey

* add note about new fields

* update changelog

* use dbt utils surrogate key

* use parents if contacts are not enabled

* add docs

* remove configurability

* extract extenstions configured with parent naming too

* fix bad syntax

---------

Co-authored-by: rlittle08 <rlittle@edanalytics.org>

CHANGELOG.md

@@ -1,9 +1,12 @@
 # Unreleased
 ## New features
+- Add base/stage models for `contacts` and `student_contact_associations`, added due to the rename from parent to contact in Ed-Fi data standard v5.0.
+- Rename `k_parent` to `k_contact` in `stg_ef3__survey_responses`.
 - Add `section_type` descriptor column to `sections` (Ed-Fi Data Standard v5.0 addition)
 - Add `responsible_teacher_staff_reference`, `v_programs`, `v_sections` columns to `course_transcripts` (Ed-Fi Data Standard v5.0 additions)
 - Add `v_programs` column to `course_transcripts` (Ed-Fi Data Standard v5.0 addition)
 ## Under the hood
+- Add columns to `base_ef3__parents` to allow data to be unioned into new `stg_ef3__contacts` model.
 ## Fixes
 - Fix typo in column name `courses.maxCompletionsForCredit`
 

macros/gen_skey.sql

@@ -190,6 +190,11 @@
             'col_list': ['learningStandardId'],
             'annualize': True
         },
+        'k_contact': {
+            'reference_name': 'contact_reference',
+            'col_list': ['contactUniqueId'],
+            'annualize': False
+        },
 
         'k_template': {
             'reference_name': '',

models/staging/edfi_3/base/_edfi_3__base.yml

@@ -21,6 +21,9 @@ models:
     config:
       tags: ['cohort']
       enabled: "{{ var('src:domain:cohort:enabled', True) }}"
+  - name: base_ef3__contacts
+    config:
+      tags: ['core']
   - name: base_ef3__course_offerings
     config:
       tags: ['core']
@@ -117,6 +120,9 @@ models:
     config:
       tags: ['cohort']
       enabled: "{{ var('src:domain:cohort:enabled', True) }}"
+  - name: base_ef3__student_contact_associations
+    config:
+      tags: ['core']
   - name: base_ef3__student_discipline_incident_associations
     config:
       tags: ['discipline']

models/staging/edfi_3/base/base_ef3__contacts.sql

@@ -0,0 +1,45 @@
+with contacts as (
+    {{ source_edfi3('contacts') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        file_row_number,
+        last_modified_timestamp,
+        filename,
+        is_deleted,
+
+        v:id::string                                                                     as record_guid,
+        v:contactUniqueId::string                                                        as contact_unique_id,
+        v:personReference:personId::string                                               as person_id,
+        v:firstName::string                                                              as first_name,
+        v:middleName::string                                                             as middle_name,
+        v:lastSurname::string                                                            as last_name,
+        v:maidenName::string                                                             as maiden_name,
+        v:generationCodeSuffix::string                                                   as generation_code_suffix,
+        v:personalTitlePrefix::string                                                    as personal_title_prefix,
+        v:genderIdentity::string                                                         as gender_identity,
+        v:preferredFirstName::string                                                     as preferred_first_name,
+        v:preferredLastSurname::string                                                   as preferred_last_name,
+        v:loginId::string                                                                as login_id,
+        {{ extract_descriptor('v:sexDescriptor::string') }}                              as sex,
+        {{ extract_descriptor('v:highestCompletedLevelOfEducationDescriptor::string') }} as highest_completed_level_of_education,
+        {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }}     as person_source_system,
+        -- references
+        v:personReference                 as person_reference,
+        -- unflattened lists
+        v:addresses                       as v_addresses,
+        v:internationalAddresses          as v_international_addresses,
+        v:electronicMails                 as v_electronic_mails,
+        v:telephones                      as v_telephones,
+        v:languages                       as v_languages,
+        v:otherNames                      as v_other_names,
+        v:personalIdentificationDocuments as v_personal_identification_documents,
+
+        -- edfi extensions
+        v:_ext as v_ext
+    from contacts
+)
+select * from renamed

models/staging/edfi_3/base/base_ef3__parents.sql

@@ -20,6 +20,11 @@ renamed as (
         v:maidenName::string                                                             as maiden_name,
         v:generationCodeSuffix::string                                                   as generation_code_suffix,
         v:personalTitlePrefix::string                                                    as personal_title_prefix,
+        -- the following three fields were introduced to the Contacts resource, which replaced Parents in v5.0
+        -- including them here (they will always be null) to allow the tables to be unioned in stage
+        v:genderIdentity::string                                                         as gender_identity, 
+        v:preferredFirstName::string                                                     as preferred_first_name,
+        v:preferredLastSurname::string                                                   as preferred_last_name,
         v:loginId::string                                                                as login_id,
         {{ extract_descriptor('v:sexDescriptor::string') }}                              as sex,
         {{ extract_descriptor('v:highestCompletedLevelOfEducationDescriptor::string') }} as highest_completed_level_of_education,

models/staging/edfi_3/base/base_ef3__student_contact_associations.sql

@@ -0,0 +1,30 @@
+with student_contact_associations as (
+    {{ source_edfi3('student_contact_associations') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                                             as record_guid,
+        v:contactPriority::int                                   as contact_priority,
+        v:contactRestrictions::string                            as contact_restrictions,
+        v:emergencyContactStatus::boolean                        as is_emergency_contact,
+        v:livesWith::boolean                                     as is_living_with,
+        v:primaryContactStatus::boolean                          as is_primary_contact,
+        v:legalGuardian::boolean                                 as is_legal_guardian,
+        {{ extract_descriptor('v:relationDescriptor::string') }} as relation_type,
+        -- references
+        v:contactReference                                       as contact_reference,
+        v:studentReference                                       as student_reference,
+
+        -- edfi extensions
+        v:_ext as v_ext
+    from student_contact_associations
+)
+select * from renamed

models/staging/edfi_3/base/base_ef3__survey_responses.sql

@@ -22,10 +22,10 @@ renamed as (
         v:responseDate::date                                          as response_date, 
         v:responseTime::int                                           as completion_time_seconds, 
         --references
-        v:surveyReference    as survey_reference,
-        v:studentReference   as student_reference,
-        v:staffReference     as staff_reference,
-        v:parentReference    as parent_reference,
+        v:surveyReference                               as survey_reference,
+        v:studentReference                              as student_reference,
+        v:staffReference                                as staff_reference,
+        coalesce(v:parentReference, v:contactReference) as contact_reference, -- parentReference renamed to contactReference in Data Standard v5.0
         -- lists
         v:surveyLevels  as v_survey_levels,    
         -- edfi extensions

models/staging/edfi_3/stage/_edfi_3__stage.yml

@@ -13,6 +13,12 @@ referential_integrity_tests:
         field: k_cohort
         tags: [ 'ref_integrity' ]
 
+  - k_contact: &ref_k_contact
+    - relationships:
+        to: ref('stg_ef3__contacts')
+        field: k_contact
+        tags: [ 'ref_integrity' ]
+
   - k_course: &ref_k_course
     - relationships:
         to: ref('stg_ef3__courses')
@@ -200,6 +206,22 @@ models:
       tags: ['cohort']
       enabled: "{{ var('src:domain:cohort:enabled', True) }}"
 
+  - name: stg_ef3__contacts
+    config:
+      tags: ['core']
+
+  - name: stg_ef3__contacts__addresses
+    config:
+      tags: ['core']
+
+  - name: stg_ef3__contacts__emails
+    config:
+      tags: ['core']
+
+  - name: stg_ef3__contacts__telephones
+    config:
+      tags: ['core']
+
   - name: stg_ef3__course_offerings
     config:
       tags: ['core']
@@ -619,6 +641,17 @@ models:
         tests:
           *ref_k_cohort
 
+  - name: stg_ef3__student_contact_associations
+    config:
+      tags: ['core']
+      columns:
+      - name: k_student
+        tests:
+          *ref_k_student
+      - name: k_contact
+        tests:
+          *ref_k_contact
+
   - name: stg_ef3__student_discipline_incident_behavior_associations
     config:
       tags: ['discipline']

models/staging/edfi_3/stage/stg_ef3__contacts.sql

@@ -0,0 +1,37 @@
+with base_contacts as (
+    select * from {{ ref('base_ef3__contacts') }}
+    where not is_deleted
+),
+base_parents as (
+    select * rename parent_unique_id as contact_unique_id
+    from {{ ref('base_ef3__parents') }}
+    where not is_deleted
+),
+-- parents were renamed to contacts in Data Standard v5.0
+unioned as (
+    select * from base_contacts
+    union 
+    select * from base_parents
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'lower(contact_unique_id)'
+            ]
+        ) }} as k_contact,
+        unioned.*
+        {{ extract_extension(model_name=[this.name, 'stg_ef3__parents'], flatten=True) }}
+    from unioned
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_contact', 
+            order_by='api_year desc, pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/edfi_3/stage/stg_ef3__contacts__addresses.sql

@@ -0,0 +1 @@
+{{ flatten_addresses('stg_ef3__contacts', ['k_contact']) }}

models/staging/edfi_3/stage/stg_ef3__contacts__emails.sql

@@ -0,0 +1 @@
+{{ flatten_emails('stg_ef3__contacts', ['k_contact']) }}

models/staging/edfi_3/stage/stg_ef3__contacts__telephones.sql

@@ -0,0 +1 @@
+{{ flatten_telephones('stg_ef3__contacts', ['k_contact']) }}

models/staging/edfi_3/stage/stg_ef3__student_contact_associations.sql

@@ -0,0 +1,41 @@
+with base_stu_contact as (
+    select 
+        *,
+        contact_reference:contactUniqueId as contact_unique_id
+    from {{ ref('base_ef3__student_contact_associations') }}
+    where not is_deleted
+),
+base_stu_parent as (
+    select 
+        *,
+        parent_reference:parentUniqueId as contact_unique_id --rename to support union and key generation
+    from {{ ref('base_ef3__student_parent_associations') }}
+    where not is_deleted
+),
+-- parents were renamed to contacts in Data Standard v5.0
+unioned as (
+    select * from base_stu_contact
+    union 
+    select * from base_stu_parent
+),
+keyed as (
+    select 
+        {{ gen_skey('k_student') }},
+        -- we can't use the gen_skey macro here because we're bringing in the deprecated parents endpoint data, which contains a parentReference that won't work
+        {{ dbt_utils.surrogate_key(['tenant_code', 'contact_unique_id']) }} as k_contact,
+        {{ gen_skey('k_student_xyear') }},
+        api_year as school_year,
+        unioned.*
+        {{ extract_extension(model_name=[this.name, 'stg_ef3__student_parent_associations'], flatten=True) }}
+    from unioned
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_student, k_contact',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/edfi_3/stage/stg_ef3__survey_responses.sql

@@ -15,7 +15,7 @@ keyed as (
         {{ gen_skey('k_survey') }},
         {{ gen_skey('k_staff') }},
         {{ gen_skey('k_student') }},
-        {{ gen_skey('k_parent') }},
+        {{ gen_skey('k_contact') }},
         base_survey_responses.*
         {{ extract_extension(model_name=this.name, flatten=True) }}
     from base_survey_responses