[DMS-396] Coerce date middleware (#339)

* Coerce date middleware

* E2E tests

src/Directory.Packages.props

@@ -10,7 +10,7 @@
     <PackageVersion Include="dbup-core" Version="5.0.87" />
     <PackageVersion Include="dbup-postgresql" Version="5.0.40" />
     <PackageVersion Include="dbup-sqlserver" Version="5.0.41" />
-    <PackageVersion Include="EdFi.DataStandard51.ApiSchema" Version="1.0.33" />
+    <PackageVersion Include="EdFi.DataStandard51.ApiSchema" Version="1.0.35" />
     <PackageVersion Include="Keycloak.Net.Core" Version="1.0.29" />
     <PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.8" />
     <PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.2.2" />

src/dms/core/EdFi.DataManagementService.Core/ApiSchema/ApiSchema_Schema.json

@@ -52,6 +52,7 @@
                     },
                     "additionalProperties": false
                 },
+                "compatibleDsRange": true,
                 "description": {
                     "type": "string"
                 },

src/dms/core/EdFi.DataManagementService.Core/ApiSchema/Extensions/JsonHelperExtensions.cs

@@ -3,6 +3,7 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
+using System.Globalization;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Nodes;
@@ -297,6 +298,30 @@ public static void TryCoerceStringToNumber(this JsonNode jsonNode)
         }
     }
 
+    public static void TryCoerceDateToDateTime(this JsonNode jsonNode)
+    {
+        var jsonValue = jsonNode.AsValue();
+        if (jsonValue.GetValueKind() == JsonValueKind.String)
+        {
+            string stringValue = jsonValue.GetValue<string>();
+            if (
+                DateOnly.TryParse(
+                    stringValue,
+                    CultureInfo.InvariantCulture,
+                    DateTimeStyles.None,
+                    out DateOnly dateValue
+                )
+            )
+            {
+                jsonNode.ReplaceWith(
+                    dateValue
+                        .ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc)
+                        .ToString("yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture)
+                );
+            }
+        }
+    }
+
     /// <summary>
     /// Helper to extract a list of JsonNodes as the values of all the properties of a JsonNode
     /// </summary>

src/dms/core/EdFi.DataManagementService.Core/ApiSchema/ResourceSchema.cs

@@ -189,6 +189,23 @@ public JsonNode JsonSchemaForRequestMethod(RequestMethod requestMethod)
                 );
         });
 
+    /// <summary>
+    /// An ordered list of the JsonPaths that are of type dateTime, for use in type coercion
+    /// </summary>
+    public IEnumerable<JsonPath> DateTimeJsonPaths => _dateTimeJsonPaths.Value;
+
+    private readonly Lazy<IEnumerable<JsonPath>> _dateTimeJsonPaths =
+        new(() =>
+        {
+            return _resourceSchemaNode["dateTimeJsonPaths"]
+                       ?.AsArray()
+                       .GetValues<string>()
+                       .Select(x => new JsonPath(x))
+                   ?? throw new InvalidOperationException(
+                       "Expected dateTimeJsonPaths to be on ResourceSchema, invalid ApiSchema"
+                   );
+        });
+
     /// <summary>
     /// An ordered list of the JsonPaths that are of type boolean, for use in type coercion
     /// </summary>

src/dms/core/EdFi.DataManagementService.Core/ApiService.cs

@@ -56,6 +56,7 @@ internal class ApiService(
                     new DuplicatePropertiesMiddleware(_logger),
                     new ValidateEndpointMiddleware(_logger),
                     new RejectResourceIdentifierMiddleware(_logger),
+                    new CoerceDateTimesMiddleware(_logger),
                 ]
             );
 
@@ -154,6 +155,7 @@ internal class ApiService(
                     new RequestDataBodyLoggingMiddleware(_logger, _appSettings.Value.MaskRequestBodyInLogs),
                     new DuplicatePropertiesMiddleware(_logger),
                     new ValidateEndpointMiddleware(_logger),
+                    new CoerceDateTimesMiddleware(_logger),
                 ]
             );
 

src/dms/core/EdFi.DataManagementService.Core/Middleware/CoerceDateTimesMiddleware.cs

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+using EdFi.DataManagementService.Core.ApiSchema.Extensions;
+using System.Text.Json.Nodes;
+using EdFi.DataManagementService.Core.Pipeline;
+using Microsoft.Extensions.Logging;
+
+namespace EdFi.DataManagementService.Core.Middleware;
+
+internal class CoerceDateTimesMiddleware(ILogger logger) : IPipelineStep
+{
+    public async Task Execute(PipelineContext context, Func<Task> next)
+    {
+        logger.LogDebug("Entering CoerceDateTimesMiddleware - {TraceId}", context.FrontendRequest.TraceId);
+
+        foreach (string path in context.ResourceSchema.DateTimeJsonPaths.Select(path => path.Value))
+        {
+            IEnumerable<JsonNode?> jsonNodes = context.ParsedBody.SelectNodesFromArrayPath(path, logger);
+            foreach (JsonNode? jsonNode in jsonNodes)
+            {
+                jsonNode?.TryCoerceDateToDateTime();
+            }
+        }
+
+        await next();
+    }
+}

src/dms/core/EdFi.DataManagementService.Core/Middleware/CoerceFromStringsMiddleware.cs

@@ -8,37 +8,36 @@
 using EdFi.DataManagementService.Core.Pipeline;
 using Microsoft.Extensions.Logging;
 
-namespace EdFi.DataManagementService.Core.Middleware
+namespace EdFi.DataManagementService.Core.Middleware;
+
+/// <summary>
+/// For boolean and numeric properties that were submitted as strings, i.e. surrounded with double quotes,
+/// this middleware tries to coerce those back to their proper type.
+/// </summary>
+internal class CoerceFromStringsMiddleware(ILogger logger) : IPipelineStep
 {
-    /// <summary>
-    /// For boolean and numeric properties that were submitted as strings, i.e. surrounded with double quotes,
-    /// this middleware tries to coerce those back to their proper type.
-    /// </summary>
-    internal class CoerceFromStringsMiddleware(ILogger logger) : IPipelineStep
+    public async Task Execute(PipelineContext context, Func<Task> next)
     {
-        public async Task Execute(PipelineContext context, Func<Task> next)
-        {
-            logger.LogDebug("Entering CoerceFromStringsMiddleware - {TraceId}", context.FrontendRequest.TraceId);
+        logger.LogDebug("Entering CoerceFromStringsMiddleware - {TraceId}", context.FrontendRequest.TraceId);
 
-            foreach (string path in context.ResourceSchema.BooleanJsonPaths.Select(path => path.Value))
+        foreach (string path in context.ResourceSchema.BooleanJsonPaths.Select(path => path.Value))
+        {
+            IEnumerable<JsonNode?> jsonNodes = context.ParsedBody.SelectNodesFromArrayPath(path, logger);
+            foreach (JsonNode? jsonNode in jsonNodes)
             {
-                IEnumerable<JsonNode?> jsonNodes = context.ParsedBody.SelectNodesFromArrayPath(path, logger);
-                foreach (JsonNode? jsonNode in jsonNodes)
-                {
-                    jsonNode?.TryCoerceStringToBoolean();
-                }
+                jsonNode?.TryCoerceStringToBoolean();
             }
+        }
 
-            foreach (string path in context.ResourceSchema.NumericJsonPaths.Select(path => path.Value))
+        foreach (string path in context.ResourceSchema.NumericJsonPaths.Select(path => path.Value))
+        {
+            IEnumerable<JsonNode?> jsonNodes = context.ParsedBody.SelectNodesFromArrayPath(path, logger);
+            foreach (JsonNode? jsonNode in jsonNodes)
             {
-                IEnumerable<JsonNode?> jsonNodes = context.ParsedBody.SelectNodesFromArrayPath(path, logger);
-                foreach (JsonNode? jsonNode in jsonNodes)
-                {
-                    jsonNode?.TryCoerceStringToNumber();
-                }
+                jsonNode?.TryCoerceStringToNumber();
             }
-
-            await next();
         }
+
+        await next();
     }
 }

src/dms/tests/EdFi.DataManagementService.Tests.E2E/Features/General/DataStrictnessValidation.feature

@@ -299,12 +299,8 @@ Feature: Data strictness
                   """
              Then it should respond with 400
 
-        @API-255 @ignore
-        # Currently the DMS is doing what an API _should_ do (require a time), but the ODS/API does NOT
-        # require a time. Changing to require a time would be a breaking change, so we can't do that
-        # in the DMS, at least not with Data Standard < 6.
-        # DMS-396
-        Scenario: 18 Enforce presence of time in a POST request with a datetime property
+        @API-255
+        Scenario: 18 Accept missing time in a POST request with a datetime property
             Given a POST request is made to "/ed-fi/assessments" with
                   """
                   {
@@ -353,3 +349,21 @@ Feature: Data strictness
                   }
                   """
              Then it should respond with 201
+              And the record can be retrieved with a GET request
+                  """
+                  {
+                      "id": "{id}",
+                      "assessmentReference": {
+                          "assessmentIdentifier": "01774fa3-06f1-47fe-8801-c8b1e65057f2",
+                          "namespace": "uri://ed-fi.org/Assessment/Assessment.xml"
+                      },
+                      "schoolYearTypeReference": {
+                          "schoolYear": 2022
+                      },
+                      "studentReference": {
+                          "studentUniqueId": "604906"
+                      },
+                      "studentAssessmentIdentifier": "/Qhqqe/gI4p3RguP68ZEDArGHM64FKnCg/RLHG8c",
+                      "administrationDate": "2021-09-28T00:00:00+00:00"
+                  }
+                  """