Feature/staff stg updates (#62)

* fix credential state location * add staff employment base/stg * add staff races * remove outdated todo
edanalytics · Feb 6, 2024 · 6817dc9 · 6817dc9
1 parent d533454
commit 6817dc9
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 3 deletions.
diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml
@@ -94,6 +94,9 @@ models:
   - name: base_ef3__staff_education_organization_assignment_associations
     config:
       tags: ['core']
+  - name: base_ef3__staff_education_organization_employment_associations
+    config:
+      tags: ['core']
   - name: base_ef3__staff_school_associations
     config:
       tags: ['core']

diff --git a/...ls/staging/edfi_3/base/base_ef3__staff_education_organization_assignment_associations.sql b/...ls/staging/edfi_3/base/base_ef3__staff_education_organization_assignment_associations.sql
@@ -20,8 +20,8 @@ renamed as (
         v:orderOfAssignment::float                                    as order_of_assignment,
         v:credentialReference:credentialIdentifier::string            as credential_identifier,
         -- descriptors
-        {{ extract_descriptor('v:stateOfIssueStateAbbreviationDescriptor::string') }} as credential_state,
-        {{ extract_descriptor('v:staffClassificationDescriptor::string') }}           as staff_classification,
+        {{ extract_descriptor('v:credentialReference:stateOfIssueStateAbbreviationDescriptor::string') }} as credential_state,
+        {{ extract_descriptor('v:staffClassificationDescriptor::string') }}                               as staff_classification,
         -- references
         v:credentialReference            as credential_reference,
         v:educationOrganizationReference as education_organization_reference,

diff --git a/...ls/staging/edfi_3/base/base_ef3__staff_education_organization_employment_associations.sql b/...ls/staging/edfi_3/base/base_ef3__staff_education_organization_employment_associations.sql
@@ -0,0 +1,37 @@
+with staff_ed_org_employ as (
+    {{ source_edfi3('staff_education_organization_employment_associations') }}
+),
+renamed as (
+    select 
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+        v:id::string as record_guid,
+        v:educationOrganizationReference:educationOrganizationId::int as ed_org_id,
+        v:educationOrganizationReference:link:rel::string             as ed_org_type,
+        v:staffReference:staffUniqueId::string                        as staff_unique_id,
+        v:credentialReference:credentialIdentifier::string            as credential_identifier,
+        v:department::string                                          as department,
+        v:hireDate::date                                              as hire_date,
+        v:endDate::date                                               as end_date,
+        v:fullTimeEquivalency::float                                  as full_time_equivalency,
+        v:hourlyWage::float                                           as hourly_wage,
+        v:annualWage::float                                           as annual_wage,
+        v:offerDate::date                                             as offer_date,
+        -- descriptors
+        {{ extract_descriptor('v:employmentStatusDescriptor::string') }}                                  as employment_status,
+        {{ extract_descriptor('v:credentialReference:stateOfIssueStateAbbreviationDescriptor::string') }} as credential_state,
+        {{ extract_descriptor('v:separationDescriptor::string') }}                                        as separation,
+        {{ extract_descriptor('v:separationReasonDescriptor::string') }}                                  as separation_reason,
+        -- references
+        v:credentialReference            as credential_reference,
+        v:educationOrganizationReference as education_organization_reference,
+        v:staffReference                 as staff_reference,
+        -- edfi extensions
+        v:_ext as v_ext
+    from staff_ed_org_employ
+)
+select * from renamed
diff --git a/models/staging/edfi_3/stage/stg_ef3__staff__races.sql b/models/staging/edfi_3/stage/stg_ef3__staff__races.sql
@@ -0,0 +1,14 @@
+with stage_staffs as (
+    select * from {{ ref('stg_ef3__staffs') }}
+),
+flattened as (
+    select 
+        tenant_code,
+        api_year,
+        k_staff,
+        {{ extract_descriptor('value:raceDescriptor::string') }} as race
+    from stage_staffs
+        , lateral flatten(input=>v_races)
+)
+select * from flattened
+
diff --git a/...ls/staging/edfi_3/stage/stg_ef3__staff_education_organization_assignment_associations.sql b/...ls/staging/edfi_3/stage/stg_ef3__staff_education_organization_assignment_associations.sql
@@ -5,7 +5,6 @@ with base_staff_ed_org_assign as (
 keyed as (
     select 
         {{ gen_skey('k_staff') }},
-        --todo: edorg
         {{ edorg_ref() }},
         api_year as school_year,
         base_staff_ed_org_assign.*

diff --git a/...ls/staging/edfi_3/stage/stg_ef3__staff_education_organization_employment_associations.sql b/...ls/staging/edfi_3/stage/stg_ef3__staff_education_organization_employment_associations.sql
@@ -0,0 +1,23 @@
+with base_staff_ed_org_employ as (
+    select * from {{ ref('base_ef3__staff_education_organization_employment_associations') }}
+    where not is_deleted
+),
+keyed as (
+    select 
+        {{ gen_skey('k_staff') }},
+        {{ edorg_ref() }},
+        api_year as school_year,
+        base_staff_ed_org_employ.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from base_staff_ed_org_employ
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by= 'tenant_code, api_year, ed_org_id, employment_status, hire_date, staff_unique_id',
+            order_by='api_year desc, pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped