Feature/mvpa update

assessments/MVPA/.gitignore

@@ -0,0 +1,2 @@
+packages/*
+packages

assessments/MVPA/README.md

@@ -0,0 +1,45 @@
+* **Title**: MVPA
+* **Description**: This is a template bundle that will map MVPA assessments. 
+* **Submitter name**: Theo Kaufman
+* **Submitter organization**: Education Analytics
+
+To run this bundle, please add your own source file(s):
+<code>data/MVPA.csv</code>
+
+Or use the sample file (`data/sample_anonymized_file.csv`).
+
+## CLI Parameters
+
+### Required
+- OUTPUT_DIR: Where the output files will be written
+- STATE_FILE: Where to store the earthmover runs.csv file
+- INPUT_FILE: The MVPA assessment results file to be mapped
+- STUDENT_ID_NAME: Which column to use as the Ed-Fi `studentUniqueId`. Default column is `StudentID`.
+
+### Examples
+
+Using the `StudentID` column from the assessment file:
+```bash
+earthmover run -c ./earthmover.yaml -p '{
+"STATE_FILE": "./runs.csv",
+"INPUT_FILE": "data/sample_anonymized_file.csv",
+"OUTPUT_DIR": "output/" ,
+"STUDENT_ID_NAME": "StudentID",
+"EDFI_DS_VERSION": "4"
+}'
+```
+
+Once you have inspected the output JSONL for issues, check the settings in lightbeam.yaml and transmit them to your Ed-Fi API with:
+```bash
+lightbeam validate+send -c ./lightbeam.yaml -p '{
+"DATA_DIR": "./output/",
+"EDFI_API_BASE_URL": "yourURL",
+"EDFI_API_CLIENT_ID": "yourID",
+"EDFI_API_CLIENT_SECRET": "yourSecret",
+"API_YEAR": "yourAPIYear" }'
+```
+
+## DAG Graph
+![DAG view of transformations](graph.png)
+
+(**Above**: a graphical depiction of the dataflow.)

assessments/MVPA/data/sample_anonymized_file.csv

@@ -0,0 +1,6 @@
+TestID,LeaID,TestDate,SchoolYear,TestName,TestSchedule,TestAdmin,SchID,TchName,Period,StudentID,Fname,Lname,PercCorr,AchLev,AchLevExt,SuggNMark,SuggLMark
+XYZ,District A,1900-01-01,2024-25,8th Grade Math,Year Round/Traditional,1st,School 1,Teacher 1,1,abc_1,First Name 1,Last Name 1,34.2857142857143,2,2-,60,D
+XYZ,District A,1900-01-01,2024-25,8th Grade Math,Year Round/Traditional,1st,School 1,Teacher 2,1,abc_2,First Name 2,Last Name 2,37.1428571428571,2,2-,62,D
+XYZ,District A,1900-01-01,2024-25,8th Grade Math,Year Round/Traditional,1st,School 1,Teacher 3,1,abc_3,First Name 3,Last Name 3,34.2857142857143,2,2-,60,D
+XYZ,District A,1900-01-01,2024-25,8th Grade Math,Year Round/Traditional,1st,School 1,Teacher 4,1,abc_4,First Name 4,Last Name 4,22.8571428571429,1,1,53,F
+XYZ,District A,1900-01-01,2024-25,8th Grade Math,Year Round/Traditional,1st,School 1,Teacher 5,2,abc_5,First Name 5,Last Name 5,17.1428571428571,1,1-,52,F

assessments/MVPA/earthmover.yaml

@@ -0,0 +1,132 @@
+version: 2
+
+
+config:
+  log_level: DEBUG
+  output_dir: ${OUTPUT_DIR}
+  memory_limit: 1GB
+  state_file: ${STATE_FILE}
+  show_graph: False
+  show_stacktrace: true
+  parameter_defaults:
+    STUDENT_ID_NAME: 'edFi_studentUniqueID' # default to the column added by the apply_xwalk package of student ID xwalking feature
+    POSSIBLE_STUDENT_ID_COLUMNS: StudentID
+    DESCRIPTOR_NAMESPACE: uri://ed-fi.org
+
+sources:
+  assessments:
+    file: ./seeds/assessments.csv
+    header_rows: 1
+  assessmentReportingMethodDescriptors:
+    file: ./seeds/assessmentReportingMethodDescriptors.csv
+    header_rows: 1
+  performanceLevelDescriptors:
+    file: ./seeds/performanceLevelDescriptors.csv
+    header_rows: 1
+  gradeLevelDescriptors:
+    file: ./seeds/gradeLevelDescriptors.csv
+    header_rows: 1
+
+  input:
+    file: ${INPUT_FILE}
+    header_rows: 1
+
+
+transformations:
+  input:
+    source: $sources.input
+    operations: []
+
+  clean_results:
+    source: $transformations.input
+    operations:
+      - operation: rename_columns
+        columns:
+          SchoolYear: schoolYear
+          TestDate: adminDate
+          TestID: testId
+      - operation: duplicate_columns
+        columns: 
+          ${STUDENT_ID_NAME}: studentUniqueId
+      - operation: combine_columns
+        columns:
+          - StudentID
+          - adminDate
+          - testId
+        new_column: studentAssessmentIdentifier
+        separator: '-'
+      - operation: modify_columns
+        columns:
+          studentAssessmentIdentifier: "{%raw%}{{ md5(studentAssessmentIdentifier) }}{%endraw%}"
+          schoolYear: "{%raw%}{{ schoolYear[:4] | int +1 }}{%endraw%}"
+      - operation: join
+        sources:
+          - $sources.assessments
+        join_type: left
+        left_keys:
+          - TestName
+        right_keys:
+          - assessmentTitle
+
+  grade_level_descriptors:
+    source: $sources.gradeLevelDescriptors
+    operations:
+      - operation: add_columns
+        columns:
+          grade_json: >
+            {%raw-%}  
+            {
+              "gradeLevelDescriptor": "{{gradeLevelDescriptor}}"
+            },
+            {%-endraw%}
+      - operation: modify_columns
+        columns:
+          grade_json: "{%raw%}{{grade_json | replace('\n', '')-}}{%endraw%}"
+      # group and aggregate
+      - operation: group_by
+        group_by_columns:
+          - assessmentIdentifier
+        create_columns: 
+          grade_json: agg(grade_json,)
+      # remove trailing comma
+      - operation: modify_columns
+        columns:
+          grade_json: "{%raw%}{{value.rstrip().rstrip(',')}}{%endraw%}"
+
+  assessments:
+    source: $transformations.clean_results
+    operations:
+      - operation: distinct_rows
+        column: assessmentTitle
+      - operation: keep_columns
+        columns:
+          - assessmentIdentifier
+          - assessmentTitle
+          - assessmentFamily
+          - namespace
+          - assessmentCategoryDescriptor
+          - academicSubjectDescriptor
+      - operation: join
+        sources:
+          - $transformations.grade_level_descriptors
+        join_type: inner
+        left_key: assessmentIdentifier
+        right_key: assessmentIdentifier
+
+destinations:
+  assessments:
+    source: $transformations.assessments
+    template: ./templates/assessments.jsont
+    extension: jsonl
+  assessmentReportingMethodDescriptors:
+    source: $sources.assessmentReportingMethodDescriptors
+    template: ./templates/descriptors.jsont
+    extension: jsonl
+  performanceLevelDescriptors:
+    source: $sources.performanceLevelDescriptors
+    template: ./templates/descriptors.jsont
+    extension: jsonl
+  studentAssessments:
+    source: $transformations.clean_results
+    template: ./templates/studentAssessments.jsont
+    extension: jsonl