[DMS-92] Basic eTag creation (#381)

* Generate Etag with LastModifiedDate and return in header after POST * ignore etag in e2e expected body comparisons * E2E coverage of basic etag * etag update on cascading updates * always check etag * mimic ods api etag format * rename middleware * unit test * Remove unnecessary backend code
Ed-Fi-Alliance-OSS · Jan 9, 2025 · 7759e8f · 7759e8f
1 parent 18c1ef3
commit 7759e8f
Show file tree

Hide file tree

Showing 10 changed files with 125 additions and 50 deletions.
diff --git a/build-dms.ps1 b/build-dms.ps1
@@ -264,7 +264,7 @@ function E2ETests {
         Invoke-Execute {
             try {
                 Push-Location eng/docker-compose/
-                ./start-local-dms.ps1 -EnvironmentFile "./.env.e2e"
+                ./start-local-dms.ps1 -EnvironmentFile "./.env.e2e" -r
             }
             finally {
                 Pop-Location
@@ -275,7 +275,7 @@ function E2ETests {
         Invoke-Execute {
             try {
                 Push-Location eng/docker-compose/
-                ./start-local-dms.ps1 -EnvironmentFile "./.env.e2e" -SearchEngine "ElasticSearch"
+                ./start-local-dms.ps1 -EnvironmentFile "./.env.e2e" -SearchEngine "ElasticSearch" -r
             }
             finally {
                 Pop-Location

diff --git a/src/dms/core/EdFi.DataManagementService.Core.Tests.Unit/Handler/UpsertHandlerTests.cs b/src/dms/core/EdFi.DataManagementService.Core.Tests.Unit/Handler/UpsertHandlerTests.cs
@@ -4,7 +4,6 @@
 // See the LICENSE and NOTICES files in the project root for more information.
 
 using System.Text.Json.Nodes;
-using EdFi.DataManagementService.Core.ApiSchema;
 using EdFi.DataManagementService.Core.Backend;
 using EdFi.DataManagementService.Core.External.Backend;
 using EdFi.DataManagementService.Core.External.Interface;
@@ -58,7 +57,7 @@ public void It_has_the_correct_response()
         {
             context.FrontendResponse.StatusCode.Should().Be(200);
             context.FrontendResponse.Body.Should().BeNull();
-            context.FrontendResponse.Headers.Count.Should().Be(0);
+            context.FrontendResponse.Headers.Count.Should().Be(1);
             context.FrontendResponse.LocationHeaderPath.Should().NotBeNullOrEmpty();
         }
     }

diff --git a/...ifiedDateToEdFiDocumentMiddlewareTests.cs → ...nMetadataToEdFiDocumentMiddlewareTests.cs b/...ifiedDateToEdFiDocumentMiddlewareTests.cs → ...nMetadataToEdFiDocumentMiddlewareTests.cs
@@ -3,6 +3,8 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
+using System.Diagnostics;
+using System.Globalization;
 using System.Text.Json.Nodes;
 using System.Text.RegularExpressions;
 using EdFi.DataManagementService.Core.Middleware;
@@ -16,20 +18,20 @@
 namespace EdFi.DataManagementService.Core.Tests.Unit.Middleware;
 
 [TestFixture]
-public class InjectLastModifiedDateToEdFiDocumentMiddlewareTests
+public class InjectVersionMetadataToEdFiDocumentMiddlewareTests
 {
     // Middleware to test
     internal static IPipelineStep Middleware()
     {
-        return new InjectLastModifiedDateToEdFiDocumentMiddleware(NullLogger.Instance);
+        return new InjectVersionMetadataToEdFiDocumentMiddleware(NullLogger.Instance);
     }
 
     [TestFixture]
-    public class Given_Valid_Request_Body : InjectLastModifiedDateToEdFiDocumentMiddlewareTests
+    public class Given_Valid_Request_Body : InjectVersionMetadataToEdFiDocumentMiddlewareTests
     {
         private readonly PipelineContext _context = No.PipelineContext();
         private readonly string _pattern = @"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$";
-        private readonly string _propertyName = "_lastModifiedDate";
+        private readonly string _lastModifiedDatePropertyName = "_lastModifiedDate";
 
         [SetUp]
         public async Task Setup()
@@ -70,10 +72,32 @@ public void It_should_not_have_response()
         [Test]
         public void It_should_have_parsed_body_with_formatted_lastmodifieddate()
         {
-            var lastModifiedDate = _context?.ParsedBody[_propertyName]?.AsValue();
+            var lastModifiedDate = _context?.ParsedBody[_lastModifiedDatePropertyName]?.AsValue();
             lastModifiedDate.Should().NotBeNull();
             var IsValid = Regex.IsMatch(lastModifiedDate!.ToString(), _pattern);
             IsValid.Should().BeTrue();
         }
+
+        [Test]
+        public void It_should_have_parsed_body_with_etag()
+        {
+            var lastModifiedDate = _context?.ParsedBody[_lastModifiedDatePropertyName]?.AsValue();
+            lastModifiedDate.Should().NotBeNull();
+
+            var eTag = _context?.ParsedBody["_etag"]?.AsValue();
+            eTag.Should().NotBeNull();
+
+            Trace.Assert(lastModifiedDate != null);
+            Trace.Assert(eTag != null);
+
+            var datetime = DateTime.ParseExact(
+                lastModifiedDate.GetValue<string>(),
+                "yyyy-MM-ddTHH:mm:ssZ",
+                DateTimeFormatInfo.InvariantInfo
+            );
+            var reverseEtag = datetime.ToBinary().ToString(CultureInfo.InvariantCulture);
+
+            reverseEtag.Should().BeEquivalentTo(eTag.GetValue<string>());
+        }
     }
 }
diff --git a/src/dms/core/EdFi.DataManagementService.Core/ApiService.cs b/src/dms/core/EdFi.DataManagementService.Core/ApiService.cs
@@ -79,7 +79,7 @@ internal class ApiService(
                 ),
                 new ExtractDocumentInfoMiddleware(_logger),
                 new DisallowDuplicateReferencesMiddleware(_logger),
-                new InjectLastModifiedDateToEdFiDocumentMiddleware(_logger),
+                new InjectVersionMetadataToEdFiDocumentMiddleware(_logger),
                 new UpsertHandler(_documentStoreRepository, _logger, _resiliencePipeline, _apiSchemaProvider),
             ]
         );
@@ -171,7 +171,7 @@ internal class ApiService(
                 ),
                 new ExtractDocumentInfoMiddleware(_logger),
                 new DisallowDuplicateReferencesMiddleware(_logger),
-                new InjectLastModifiedDateToEdFiDocumentMiddleware(_logger),
+                new InjectVersionMetadataToEdFiDocumentMiddleware(_logger),
                 new UpdateByIdHandler(
                     _documentStoreRepository,
                     _logger,

diff --git a/src/dms/core/EdFi.DataManagementService.Core/Backend/SuccessDocumentStoreRepository.cs b/src/dms/core/EdFi.DataManagementService.Core/Backend/SuccessDocumentStoreRepository.cs
@@ -5,6 +5,7 @@
 using System.Text.Json.Nodes;
 using EdFi.DataManagementService.Core.External.Backend;
 using EdFi.DataManagementService.Core.External.Interface;
+using EdFi.DataManagementService.Core.External.Model;
 using EdFi.DataManagementService.Core.Model;
 using Microsoft.Extensions.Logging;
 

diff --git a/src/dms/core/EdFi.DataManagementService.Core/Backend/UpdateCascadeHandler.cs b/src/dms/core/EdFi.DataManagementService.Core/Backend/UpdateCascadeHandler.cs
@@ -3,6 +3,7 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
+using System.Globalization;
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using EdFi.DataManagementService.Core.ApiSchema;
@@ -200,10 +201,14 @@ var originalIdentityJsonPath in originalIdentityJsonPaths
             }
         }
 
-        // finally update _lastModifiedDate
+        // finally update _lastModifiedDate and _etag
         DateTimeOffset utcNow = DateTimeOffset.UtcNow;
         string formattedUtcDateTime = utcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
         returnEdFiDoc["_lastModifiedDate"] = formattedUtcDateTime;
+        returnEdFiDoc["_etag"] = DateTime
+            .ParseExact(formattedUtcDateTime, "yyyy-MM-ddTHH:mm:ssZ", DateTimeFormatInfo.InvariantInfo)
+            .ToBinary()
+            .ToString(CultureInfo.InvariantCulture);
 
         return new UpdateCascadeResult(
             referencingEdFiDoc,

diff --git a/src/dms/core/EdFi.DataManagementService.Core/Handler/UpsertHandler.cs b/src/dms/core/EdFi.DataManagementService.Core/Handler/UpsertHandler.cs
@@ -5,7 +5,6 @@
 
 using EdFi.DataManagementService.Core.ApiSchema;
 using EdFi.DataManagementService.Core.Backend;
-using EdFi.DataManagementService.Core.External.Backend;
 using EdFi.DataManagementService.Core.External.Interface;
 using EdFi.DataManagementService.Core.External.Model;
 using EdFi.DataManagementService.Core.Model;
@@ -60,9 +59,9 @@ public async Task Execute(PipelineContext context, Func<Task> next)
         context.FrontendResponse = upsertResult switch
         {
             InsertSuccess insertSuccess => new FrontendResponse(
-                StatusCode: 201,
-                Body: null,
-                Headers: [],
+            StatusCode: 201,
+            Body: null,
+                Headers: new Dictionary<string, string>() { { "etag", context.ParsedBody["_etag"]?.ToString() ?? "" } },
                 LocationHeaderPath: PathComponents.ToResourcePath(
                     context.PathComponents,
                     insertSuccess.NewDocumentUuid
@@ -71,7 +70,7 @@ public async Task Execute(PipelineContext context, Func<Task> next)
             UpdateSuccess updateSuccess => new(
                 StatusCode: 200,
                 Body: null,
-                Headers: [],
+                Headers: new Dictionary<string, string>() { { "etag", context.ParsedBody["_etag"]?.ToString() ?? "" } },
                 LocationHeaderPath: PathComponents.ToResourcePath(
                     context.PathComponents,
                     updateSuccess.ExistingDocumentUuid

diff --git a/...stModifiedDateToEdFiDocumentMiddleware.cs → ...ersionMetadataToEdFiDocumentMiddleware.cs b/...stModifiedDateToEdFiDocumentMiddleware.cs → ...ersionMetadataToEdFiDocumentMiddleware.cs
@@ -3,12 +3,13 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
+using System.Globalization;
 using EdFi.DataManagementService.Core.Pipeline;
 using Microsoft.Extensions.Logging;
 
 namespace EdFi.DataManagementService.Core.Middleware
 {
-    internal class InjectLastModifiedDateToEdFiDocumentMiddleware(ILogger _logger) : IPipelineStep
+    internal class InjectVersionMetadataToEdFiDocumentMiddleware(ILogger _logger) : IPipelineStep
     {
         public async Task Execute(PipelineContext context, Func<Task> next)
         {
@@ -20,6 +21,10 @@ public async Task Execute(PipelineContext context, Func<Task> next)
             DateTimeOffset utcNow = DateTimeOffset.UtcNow;
             string formattedUtcDateTime = utcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
             context.ParsedBody["_lastModifiedDate"] = formattedUtcDateTime;
+            context.ParsedBody["_etag"] = DateTime
+                .ParseExact(formattedUtcDateTime, "yyyy-MM-ddTHH:mm:ssZ", DateTimeFormatInfo.InvariantInfo)
+                .ToBinary()
+                .ToString(CultureInfo.InvariantCulture);
             await next();
         }
     }

diff --git a/.../Features/Resources/etag.feature.disabled → ...Tests.E2E/Features/Resources/etag.feature b/.../Features/Resources/etag.feature.disabled → ...Tests.E2E/Features/Resources/etag.feature
@@ -1,28 +1,31 @@
-Feature: Validates the functionality of the ETag
-
-        Background:
-
-            Given the system has these "students"
-                  | studentUniqueId | birthDate  | firstName | lastSurname |
-                  | 111111          | 2014-08-14 | Russella  | Mayers      |
+Feature: ETag validations
 
         @API-260
         Scenario: 01 Ensure that clients can retrieve an ETag in the response header
-             When a GET request is made to "/ed-fi/students/{id}"
-             Then it should respond with 200
-              And the response body is
+             When a POST request is made to "/ed-fi/students" with
+                  """
+                  {
+                      "studentUniqueId": "111111",
+                      "birthDate": "2014-08-14",
+                      "firstName": "Russella",
+                      "lastSurname": "Mayers"
+                  }
+                  """
+             Then it should respond with 201
+              And the ETag is in the response header
+              And the record can be retrieved with a GET request
                   """
                   {
+                    "id": "{id}",
                     "studentUniqueId": "111111",
                     "birthDate": "2014-08-14",
                     "firstName": "Russella",
                     "lastSurname": "Mayers",
                     "_etag": "{etag}"
                   }
                   """
-              And the ETag is in the response header
 
-        @API-261
+        @ignore @API-261
         Scenario: 02 Ensure that clients can pass an ETag in the request header
              When a PUT if-match "{etag}" request is made to "/ed-fi/students/{id}" with
                   """
@@ -35,7 +38,7 @@ Feature: Validates the functionality of the ETag
                   """
              Then it should respond with 204
 
-        @API-262
+        @ignore @API-262
         Scenario: 03 Ensure that clients cannot pass a different ETag in the If-Match header
              When a PUT if-match "0000000000" request is made to "/ed-fi/students/{id}" with
                   """
@@ -61,7 +64,7 @@ Feature: Validates the functionality of the ETag
                   }
                   """
 
-        @API-263
+        @ignore @API-263
         Scenario: 04 Ensure that clients cannot pass a different ETag in the If-Match header to delete a resource
              When a DELETE if-match "0000000000" request is made to "/ed-fi/students/{id}"
              Then it should respond with 412
@@ -79,7 +82,7 @@ Feature: Validates the functionality of the ETag
                   }
                   """
 
-        @API-264
+        @ignore @API-264
         Scenario: 05 Ensure that clients can pass an ETag to delete a resource
              When a DELETE if-match "{etag}" request is made to "/ed-fi/students/{id}"
              Then it should respond with 204