feature/: Exit Withdraw Date code to handle inclusivity/exclusivity

macros/end_date_utils.sql

@@ -0,0 +1,61 @@
+{% macro date_within_end_date(date_col, end_date, inclusive=true) %}
+{#- 
+    Produces sql to determine if a date is within an end date or not.
+
+    The inclusivity of the end_date is what matters here. If end_date
+    is inclusive and date_col is less than or equal to the end_date, 
+    then it is within the end_date. But if the end date is exclusive and
+    date_col equals end_date then it is not within the end date.
+
+    Inclusivity is assumed.
+
+    Edfi makes no mention of whether end_dates should be considered 
+    inclusive or exclusive. When asked they will say to do what makes
+    sense of your own implementation of Edfi.
+
+    If end_date is null, then date_col is within the end date.
+
+    Args:
+    date_col: the date to be compared against the end_date
+    end_date: the end date of some date range.
+    inclusive: should the end_date be considered inclusive or not
+ -#}
+    {% if inclusive %}
+        ({{ end_date }} is null or {{ date_col }} <= {{ end_date }})
+    {% else %}
+        ({{ end_date }} is null or {{ date_col }} < {{ end_date }})
+    {% endif %}
+{% endmacro %}
+
+{% macro day_count_in_range(begin_date, end_date, inclusive=true) %}
+{#- 
+    Inclusivity for an end date (or begin date, for that matter) matters
+    when trying to determine the dates within a date range. If the end date
+    is exclusive, then that day does not count towards the day count.
+
+    If the end date is null, then we will be calculating the day count as of 
+    today AND today COUNTS as a day in the range, regardless of inclusivity.
+
+    Args:
+    begin_date: the start of the date range
+    end_date: the end of the date range
+    inclusive: should the end_date be considered inclusive or not
+ -#}
+    {{ return(adapter.dispatch('day_count_in_range', 'edu_wh')(begin_date, end_date, inclusive=true)) }}
+{% endmacro %}
+
+{% macro snowflake__day_count_in_range(begin_date, end_date, inclusive) -%}
+    {% if inclusive %}
+        date(coalesce({{ end_date }}, getdate())) - {{ begin_date }} + 1
+    {% else %}
+        date(coalesce({{ end_date }}, getdate())) - {{ begin_date }}
+    {% endif %}
+{%- endmacro %}
+
+{% macro databricks__day_count_in_range(begin_date, end_date, inclusive) -%}
+    {% if inclusive %}
+        date_diff(dateadd(day, 1, coalesce({{ end_date }}, CURRENT_DATE())), {{ begin_date }})
+    {% else %}
+        date_diff(coalesce({{ end_date }}, dateadd(day, 1, CURRENT_DATE())), {{ begin_date }})
+    {% endif %}
+{%- endmacro %}

models/core_warehouse/fct_student_daily_attendance.sql

@@ -125,8 +125,7 @@ fill_positive_attendance as (
         fct_student_school_att.attendance_event_reason,
         -- set enrollment flag: 1 during enrollment, 0 after, no row prior
         case 
-            when stu_enr_att_cal.calendar_date <= stu_enr_att_cal.exit_withdraw_date
-                or stu_enr_att_cal.exit_withdraw_date is null
+            when {{ date_within_end_date('stu_enr_att_cal.calendar_date', 'stu_enr_att_cal.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
             then 1.0
             else 0.0
         end is_enrolled,

models/core_warehouse/fct_student_school_association.sql

@@ -66,8 +66,7 @@ formatted as (
             stg_stu_school.school_year = max(stg_stu_school.school_year) 
                 over(partition by stg_stu_school.tenant_code)
             -- not yet exited
-            and (exit_withdraw_date is null
-                or exit_withdraw_date >= current_date())
+            and {{ date_within_end_date('current_date()', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
             -- enrollment has begun
             and entry_date <= current_date(),
             true, false
@@ -113,14 +112,12 @@ formatted as (
     where true
    {% if var('edu:enroll:exclude_exit_before_first_day', True) -%}
       -- exclude students who exited before the first school day
-      and (exit_withdraw_date >= bld_school_calendar_windows.first_school_day
-          or exit_withdraw_date is null
+      and ({{ date_within_end_date('bld_school_calendar_windows.first_school_day', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
           or bld_school_calendar_windows.first_school_day is null
           )
     {% endif %}
     -- exclude students whose exit day is before their entry day
-    and (exit_withdraw_date >= entry_date
-        or exit_withdraw_date is null)
+    and ( {{ date_within_end_date('entry_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} )
     -- exclude students who never actually enrolled
     {% set excl_withdraw_codes =  var('edu:enroll:exclude_withdraw_codes')  %}
     {% if excl_withdraw_codes | length -%}

models/core_warehouse/fct_student_school_attendance_event.sql

@@ -41,7 +41,7 @@ fct_student_school_assoc as (
     */
     select 
         *,
-        date(coalesce(exit_withdraw_date,getdate())) - entry_date as enrollment_length
+        {{ day_count_in_range('entry_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} as enrollment_length
     from {{ ref('fct_student_school_association') }}
 ),
 xwalk_att_events as (
@@ -79,7 +79,8 @@ joined as (
     join dim_calendar_date
          on fct_student_school_assoc.k_school_calendar = dim_calendar_date.k_school_calendar
          and stg_stu_sch_attend.attendance_event_date = dim_calendar_date.calendar_date
-         and dim_calendar_date.calendar_date between fct_student_school_assoc.entry_date and coalesce(fct_student_school_assoc.exit_withdraw_date,current_date())
+         and dim_calendar_date.calendar_date >= fct_student_school_assoc.entry_date 
+         and {{ date_within_end_date('dim_calendar_date.calendar_date', 'fct_student_school_assoc.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
     join xwalk_att_events
         on stg_stu_sch_attend.attendance_event_category = xwalk_att_events.attendance_event_descriptor
 ),

models/qc/sections_per_enrollment.sql

@@ -19,16 +19,12 @@ left join fct_student_section_association
     and fct_student_school_association.k_school = fct_student_section_association.k_school
     and (
         (
-            fct_student_section_association.begin_date
-            between fct_student_school_association.entry_date and coalesce(
-                fct_student_school_association.exit_withdraw_date, current_date()
-            )
+            fct_student_section_association.begin_date >= fct_student_school_association.entry_date 
+            and {{ date_within_end_date('fct_student_section_association.begin_date', 'fct_student_school_association.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
         )
         or (
-            fct_student_section_association.end_date
-            between fct_student_school_association.entry_date and coalesce(
-                fct_student_school_association.exit_withdraw_date, current_date()
-            )
+            fct_student_section_association.end_date >= fct_student_school_association.entry_date
+            and {{ date_within_end_date('fct_student_section_association.end_date', 'fct_student_school_association.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
         )
     )
 group by 1, 2, 3, 4, 5

tests/integrity_tests/all_student_school_associations_have_calendars.sql

@@ -55,8 +55,6 @@ join first_school_day
   and first_school_day.k_school_calendar is null
 where true 
 -- exclude students who exited before first day of school
-and (fct_stu_school_assoc.exit_withdraw_date > first_school_day.first_school_day
-or fct_stu_school_assoc.exit_withdraw_date is null)
-
+and {{ date_within_end_date('first_school_day.first_school_day', 'fct_stu_school_assoc.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
 group by 1,2,3
 having n_students_missing_calendar != 0

tests/value_tests/invalid_enrollments.sql

@@ -31,8 +31,8 @@ first_school_day as (
     group by 1
 )
 select 
-    exit_withdraw_date < first_school_day.calendar_date as exit_before_first_day,
-    exit_withdraw_date < entry_date as exit_before_entry,
+    not ( {{ date_within_end_date('first_school_day.calendar_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} ) as exit_before_first_day,
+    not ( {{ date_within_end_date('entry_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} ) as exit_before_entry,
     {% set excl_withdraw_codes =  var('edu:enroll:exclude_withdraw_codes')  %}
     {% if excl_withdraw_codes | length -%}
       {% if excl_withdraw_codes is string -%}

tests/value_tests/overlapping_enrollments.sql

@@ -38,9 +38,8 @@ counted as (
     from dim_calendar_date
     join fct_student_school_assoc
         on dim_calendar_date.k_school_calendar = fct_student_school_assoc.k_school_calendar
-        and dim_calendar_date.calendar_date between 
-            fct_student_school_assoc.entry_date and
-            coalesce(fct_student_school_assoc.exit_withdraw_date, current_date())
+        and dim_calendar_date.calendar_date >= fct_student_school_assoc.entry_date
+        and {{ date_within_end_date('dim_calendar_date.calendar_date', 'fct_student_school_assoc.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }}
     group by 1,2,3
     having duplicate_days > 1
 ),