Feature/Contacts and StudentContactAssociations

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Unreleased
 ## New features
+- Add base/stage models for `contacts` and `student_contact_associations`, added due to the rename from parent to contact in Ed-Fi data standard v5.0.
+- Rename `k_parent` to `k_contact` in `stg_ef3__survey_responses`.
 ## Under the hood
+- Add columns to `base_ef3__parents` to allow data to be unioned into new `stg_ef3__contacts` model.
 ## Fixes
 
 # edu_edfi_source v0.3.6

macros/gen_skey.sql

@@ -190,6 +190,11 @@
             'col_list': ['learningStandardId'],
             'annualize': True
         },
+        'k_contact': {
+            'reference_name': 'contact_reference',
+            'col_list': ['contactUniqueId'],
+            'annualize': False
+        },
 
         'k_template': {
             'reference_name': '',

models/staging/edfi_3/base/base_ef3__contacts.sql

@@ -0,0 +1,45 @@
+with contacts as (
+    {{ source_edfi3('contacts') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        file_row_number,
+        last_modified_timestamp,
+        filename,
+        is_deleted,
+
+        v:id::string                                                                     as record_guid,
+        v:contactUniqueId::string                                                        as contact_unique_id,
+        v:personReference:personId::string                                               as person_id,
+        v:firstName::string                                                              as first_name,
+        v:middleName::string                                                             as middle_name,
+        v:lastSurname::string                                                            as last_name,
+        v:maidenName::string                                                             as maiden_name,
+        v:generationCodeSuffix::string                                                   as generation_code_suffix,
+        v:personalTitlePrefix::string                                                    as personal_title_prefix,
+        v:genderIdentity::string                                                         as gender_identity,
+        v:preferredFirstName::string                                                     as preferred_first_name,
+        v:preferredLastSurname::string                                                   as preferred_last_name,
+        v:loginId::string                                                                as login_id,
+        {{ extract_descriptor('v:sexDescriptor::string') }}                              as sex,
+        {{ extract_descriptor('v:highestCompletedLevelOfEducationDescriptor::string') }} as highest_completed_level_of_education,
+        {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }}     as person_source_system,
+        -- references
+        v:personReference                 as person_reference,
+        -- unflattened lists
+        v:addresses                       as v_addresses,
+        v:internationalAddresses          as v_international_addresses,
+        v:electronicMails                 as v_electronic_mails,
+        v:telephones                      as v_telephones,
+        v:languages                       as v_languages,
+        v:otherNames                      as v_other_names,
+        v:personalIdentificationDocuments as v_personal_identification_documents,
+
+        -- edfi extensions
+        v:_ext as v_ext
+    from contacts
+)
+select * from renamed

models/staging/edfi_3/base/base_ef3__parents.sql

@@ -20,6 +20,11 @@ renamed as (
         v:maidenName::string                                                             as maiden_name,
         v:generationCodeSuffix::string                                                   as generation_code_suffix,
         v:personalTitlePrefix::string                                                    as personal_title_prefix,
+        -- the following three fields were introduced to the Contacts resource, which replaced Parents in v5.0
+        -- including them here (they will always be null) to allow the tables to be unioned in stage
+        v:genderIdentity::string                                                         as gender_identity, 
+        v:preferredFirstName::string                                                     as preferred_first_name,
+        v:preferredLastSurname::string                                                   as preferred_last_name,
         v:loginId::string                                                                as login_id,
         {{ extract_descriptor('v:sexDescriptor::string') }}                              as sex,
         {{ extract_descriptor('v:highestCompletedLevelOfEducationDescriptor::string') }} as highest_completed_level_of_education,

models/staging/edfi_3/base/base_ef3__student_contact_associations.sql

@@ -0,0 +1,30 @@
+with student_contact_associations as (
+    {{ source_edfi3('student_contact_associations') }}
+),
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        last_modified_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+
+        v:id::string                                             as record_guid,
+        v:contactPriority::int                                   as contact_priority,
+        v:contactRestrictions::string                            as contact_restrictions,
+        v:emergencyContactStatus::boolean                        as is_emergency_contact,
+        v:livesWith::boolean                                     as is_living_with,
+        v:primaryContactStatus::boolean                          as is_primary_contact,
+        v:legalGuardian::boolean                                 as is_legal_guardian,
+        {{ extract_descriptor('v:relationDescriptor::string') }} as relation_type,
+        -- references
+        v:contactReference                                       as contact_reference,
+        v:studentReference                                       as student_reference,
+
+        -- edfi extensions
+        v:_ext as v_ext
+    from student_contact_associations
+)
+select * from renamed

models/staging/edfi_3/base/base_ef3__survey_responses.sql

@@ -22,10 +22,10 @@ renamed as (
         v:responseDate::date                                          as response_date, 
         v:responseTime::int                                           as completion_time_seconds, 
         --references
-        v:surveyReference    as survey_reference,
-        v:studentReference   as student_reference,
-        v:staffReference     as staff_reference,
-        v:parentReference    as parent_reference,
+        v:surveyReference                               as survey_reference,
+        v:studentReference                              as student_reference,
+        v:staffReference                                as staff_reference,
+        coalesce(v:parentReference, v:contactReference) as contact_reference, -- parentReference renamed to contactReference in Data Standard v5.0
         -- lists
         v:surveyLevels  as v_survey_levels,    
         -- edfi extensions

models/staging/edfi_3/stage/stg_ef3__contacts.sql

@@ -0,0 +1,37 @@
+with base_contacts as (
+    select * from {{ ref('base_ef3__contacts') }}
+    where not is_deleted
+),
+base_parents as (
+    select * rename parent_unique_id as contact_unique_id
+    from {{ ref('base_ef3__parents') }}
+    where not is_deleted
+),
+-- parents were renamed to contacts in Data Standard v5.0
+unioned as (
+    select * from base_contacts
+    union 
+    select * from base_parents
+),
+keyed as (
+    select 
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'lower(contact_unique_id)'
+            ]
+        ) }} as k_contact,
+        unioned.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from unioned
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_contact', 
+            order_by='api_year desc, pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/edfi_3/stage/stg_ef3__contacts__addresses.sql

@@ -0,0 +1 @@
+{{ flatten_addresses('stg_ef3__contacts', ['k_contact']) }}

models/staging/edfi_3/stage/stg_ef3__contacts__emails.sql

@@ -0,0 +1 @@
+{{ flatten_emails('stg_ef3__contacts', ['k_contact']) }}

models/staging/edfi_3/stage/stg_ef3__contacts__telephones.sql

@@ -0,0 +1 @@
+{{ flatten_telephones('stg_ef3__contacts', ['k_contact']) }}

models/staging/edfi_3/stage/stg_ef3__student_contact_associations.sql

@@ -0,0 +1,45 @@
+with base_stu_contact as (
+    select 
+        *,
+        contact_reference:contactUniqueId as contact_unique_id
+    from {{ ref('base_ef3__student_contact_associations') }}
+    where not is_deleted
+),
+base_stu_parent as (
+    select 
+        *,
+        parent_reference:parentUniqueId as contact_unique_id --rename to support union and key generation
+    from {{ ref('base_ef3__student_parent_associations') }}
+    where not is_deleted
+),
+-- parents were renamed to contacts in Data Standard v5.0
+unioned as (
+    select * from base_stu_contact
+    union 
+    select * from base_stu_parent
+),
+keyed as (
+    select 
+        {{ gen_skey('k_student') }},
+        -- we can't use the gen_skey macro here because we're bringing in the deprecated parents endpoint data, which contains a parentReference that won't work
+        iff(
+            contact_unique_id is not null, 
+            md5(cast(coalesce(cast(tenant_code as TEXT), '') || '-' || coalesce(cast(lower(contact_unique_id) as TEXT), '') as TEXT)), 
+            null
+        )::varchar(32) as k_contact,
+        {{ gen_skey('k_student_xyear') }},
+        api_year as school_year,
+        unioned.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from unioned
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_student, k_contact',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/edfi_3/stage/stg_ef3__survey_responses.sql

@@ -15,7 +15,7 @@ keyed as (
         {{ gen_skey('k_survey') }},
         {{ gen_skey('k_staff') }},
         {{ gen_skey('k_student') }},
-        {{ gen_skey('k_parent') }},
+        {{ gen_skey('k_contact') }},
         base_survey_responses.*
         {{ extract_extension(model_name=this.name, flatten=True) }}
     from base_survey_responses