Add maximum page size (#377)

Ed-Fi-Alliance-OSS · Jan 7, 2025 · 18c1ef3 · 18c1ef3
1 parent e7207cd
commit 18c1ef3
Show file tree

Hide file tree

Showing 10 changed files with 303 additions and 236 deletions.
diff --git a/eng/docker-compose/.env.e2e b/eng/docker-compose/.env.e2e
@@ -28,6 +28,7 @@ NEED_DATABASE_SETUP=true
 BYPASS_STRING_COERCION=false
 LOG_LEVEL=DEBUG
 MASK_REQUEST_BODY_IN_LOGS=true
+MAXIMUM_PAGE_SIZE=500
 CORRELATION_ID_HEADER=correlationid
 DMS_DATASTORE=postgresql
 DMS_QUERYHANDLER=opensearch

diff --git a/eng/docker-compose/.env.example b/eng/docker-compose/.env.example
@@ -51,6 +51,7 @@ ASPNETCORE_HTTP_PORTS=8080
 OAUTH_TOKEN_ENDPOINT=http://localhost:8045/realms/edfi/protocol/openid-connect/token
 NEED_DATABASE_SETUP=true
 BYPASS_STRING_COERCION=false
+MAXIMUM_PAGE_SIZE=500
 
 # INFORMATION, WARNING
 LOG_LEVEL=DEBUG

diff --git a/eng/docker-compose/local-dms.yml b/eng/docker-compose/local-dms.yml
@@ -26,6 +26,7 @@ services:
       LOG_LEVEL: ${LOG_LEVEL:-DEBUG}
       # Mask incoming HTTP POST and PUT request body structures in DEBUG logging
       MASK_REQUEST_BODY_IN_LOGS: ${MASK_REQUEST_BODY_IN_LOGS:-true}
+      MAXIMUM_PAGE_SIZE: ${MAXIMUM_PAGE_SIZE:-500}
       # The default correlation id header is deliberately blank on the following line
       CORRELATION_ID_HEADER: ${CORRELATION_ID_HEADER:-}
       DMS_DATASTORE: ${DMS_DATASTORE:-postgresql}

diff --git a/src/dms/appsettings.template.json b/src/dms/appsettings.template.json
@@ -7,7 +7,8 @@
         "BypassStringTypeCoercion": "${BYPASS_STRING_COERCION}",
         "CorrelationIdHeader": "${CORRELATION_ID_HEADER}",
         "AllowIdentityUpdateOverrides": "${ALLOW_IDENTITY_UPDATE_OVERRIDES}",
-        "MaskRequestBodyInLogs": "${MASK_REQUEST_BODY_IN_LOGS}"
+        "MaskRequestBodyInLogs": "${MASK_REQUEST_BODY_IN_LOGS}",
+        "MaximumPageSize": "${MAXIMUM_PAGE_SIZE}"
     },
     "IdentitySettings": {
         "EnforceAuthorization": "${IDENTITY_ENFORCE_AUTHORIZATION}",

diff --git a/...ore/EdFi.DataManagementService.Core.Tests.Unit/Middleware/ValidateQueryMiddlewareTests.cs b/...ore/EdFi.DataManagementService.Core.Tests.Unit/Middleware/ValidateQueryMiddlewareTests.cs
@@ -20,9 +20,11 @@ namespace EdFi.DataManagementService.Core.Tests.Unit.Middleware;
 [TestFixture]
 public class ValidateQueryMiddlewareTests
 {
+    private static readonly int _maxPageSize = 500;
+
     internal static IPipelineStep Middleware()
     {
-        return new ValidateQueryMiddleware(NullLogger.Instance);
+        return new ValidateQueryMiddleware(NullLogger.Instance, _maxPageSize);
     }
 
     [TestFixture]
@@ -40,13 +42,12 @@ public async Task Setup()
                 { "totalCount", "100" },
             };
 
-            FrontendRequest frontendRequest =
-                new(
-                    Path: "/ed-fi/schools",
-                    Body: null,
-                    QueryParameters: queryParameters,
-                    TraceId: new TraceId("")
-                );
+            FrontendRequest frontendRequest = new(
+                Path: "/ed-fi/schools",
+                Body: null,
+                QueryParameters: queryParameters,
+                TraceId: new TraceId("")
+            );
             _context = new(frontendRequest, RequestMethod.GET);
             await Middleware().Execute(_context, NullNext);
         }
@@ -81,7 +82,7 @@ public void It_should_be_limit_errors()
             _context
                 .FrontendResponse.Body?.ToJsonString()
                 .Should()
-                .Contain("Limit must be a numeric value greater than or equal to 0.");
+                .Contain($"Limit must be omitted or set to a numeric value between 0 and {_maxPageSize}.");
         }
 
         [Test]
@@ -94,6 +95,56 @@ public void It_should_be_total_count_errors()
         }
     }
 
+    [TestFixture]
+    public class Given_Pipeline_Context_With_Greater_Limit_Value : ValidateQueryMiddlewareTests
+    {
+        private PipelineContext _context = No.PipelineContext();
+
+        [SetUp]
+        public async Task Setup()
+        {
+            var queryParameters = new Dictionary<string, string>
+            {
+                { "offset", "0" },
+                { "limit", "800" },
+                { "totalCount", "100" },
+            };
+
+            FrontendRequest frontendRequest = new(
+                Path: "/ed-fi/schools",
+                Body: null,
+                QueryParameters: queryParameters,
+                TraceId: new TraceId("")
+            );
+            _context = new(frontendRequest, RequestMethod.GET);
+            await Middleware().Execute(_context, NullNext);
+        }
+
+        [Test]
+        public void It_should_send_bad_request()
+        {
+            _context?.FrontendResponse.StatusCode.Should().Be(400);
+        }
+
+        [Test]
+        public void It_should_be_errors()
+        {
+            _context
+                .FrontendResponse.Body?.ToJsonString()
+                .Should()
+                .Contain("The request could not be processed.");
+        }
+
+        [Test]
+        public void It_should_be_limit_errors()
+        {
+            _context
+                .FrontendResponse.Body?.ToJsonString()
+                .Should()
+                .Contain($"Limit must be omitted or set to a numeric value between 0 and {_maxPageSize}.");
+        }
+    }
+
     [TestFixture]
     public class Given_Pipeline_Context_With_Invalid_Type_Query_Parameters : ValidateQueryMiddlewareTests
     {
@@ -124,16 +175,15 @@ private static PipelineContext NewPipelineContext(
             RequestMethod method
         )
         {
-            PipelineContext docRefContext =
-                new(frontendRequest, method)
-                {
-                    ApiSchemaDocument = NewApiSchemaDocument(),
-                    PathComponents = new(
-                        ProjectNamespace: new("ed-fi"),
-                        EndpointName: new("academicWeeks"),
-                        DocumentUuid: No.DocumentUuid
-                    ),
-                };
+            PipelineContext docRefContext = new(frontendRequest, method)
+            {
+                ApiSchemaDocument = NewApiSchemaDocument(),
+                PathComponents = new(
+                    ProjectNamespace: new("ed-fi"),
+                    EndpointName: new("academicWeeks"),
+                    DocumentUuid: No.DocumentUuid
+                ),
+            };
             docRefContext.ProjectSchema = new ProjectSchema(
                 docRefContext.ApiSchemaDocument.FindProjectSchemaNode(new("ed-fi")) ?? new JsonObject(),
                 NullLogger.Instance
@@ -167,13 +217,12 @@ public async Task Setup()
                 { "classStartTime", "44:80:99.123" },
             };
 
-            FrontendRequest frontendRequest =
-                new(
-                    Path: "/ed-fi/academicWeeks",
-                    Body: null,
-                    QueryParameters: queryParameters,
-                    TraceId: new TraceId("")
-                );
+            FrontendRequest frontendRequest = new(
+                Path: "/ed-fi/academicWeeks",
+                Body: null,
+                QueryParameters: queryParameters,
+                TraceId: new TraceId("")
+            );
 
             _context = NewPipelineContext(frontendRequest, RequestMethod.GET);
 
@@ -271,16 +320,15 @@ private static PipelineContext NewPipelineContext(
             RequestMethod method
         )
         {
-            PipelineContext docRefContext =
-                new(frontendRequest, method)
-                {
-                    ApiSchemaDocument = NewApiSchemaDocument(),
-                    PathComponents = new(
-                        ProjectNamespace: new("ed-fi"),
-                        EndpointName: new("academicWeeks"),
-                        DocumentUuid: No.DocumentUuid
-                    ),
-                };
+            PipelineContext docRefContext = new(frontendRequest, method)
+            {
+                ApiSchemaDocument = NewApiSchemaDocument(),
+                PathComponents = new(
+                    ProjectNamespace: new("ed-fi"),
+                    EndpointName: new("academicWeeks"),
+                    DocumentUuid: No.DocumentUuid
+                ),
+            };
             docRefContext.ProjectSchema = new ProjectSchema(
                 docRefContext.ApiSchemaDocument.FindProjectSchemaNode(new("ed-fi")) ?? new JsonObject(),
                 NullLogger.Instance
@@ -314,13 +362,12 @@ public async Task Setup()
                 { "classStartTime", "10:30:00" },
             };
 
-            FrontendRequest frontendRequest =
-                new(
-                    Path: "/ed-fi/academicWeeks",
-                    Body: null,
-                    QueryParameters: queryParameters,
-                    TraceId: new TraceId("")
-                );
+            FrontendRequest frontendRequest = new(
+                Path: "/ed-fi/academicWeeks",
+                Body: null,
+                QueryParameters: queryParameters,
+                TraceId: new TraceId("")
+            );
 
             _context = NewPipelineContext(frontendRequest, RequestMethod.GET);
 
@@ -359,16 +406,15 @@ private static PipelineContext NewPipelineContext(
             RequestMethod method
         )
         {
-            PipelineContext docRefContext =
-                new(frontendRequest, method)
-                {
-                    ApiSchemaDocument = NewApiSchemaDocument(),
-                    PathComponents = new(
-                        ProjectNamespace: new("ed-fi"),
-                        EndpointName: new("academicWeeks"),
-                        DocumentUuid: No.DocumentUuid
-                    ),
-                };
+            PipelineContext docRefContext = new(frontendRequest, method)
+            {
+                ApiSchemaDocument = NewApiSchemaDocument(),
+                PathComponents = new(
+                    ProjectNamespace: new("ed-fi"),
+                    EndpointName: new("academicWeeks"),
+                    DocumentUuid: No.DocumentUuid
+                ),
+            };
             docRefContext.ProjectSchema = new ProjectSchema(
                 docRefContext.ApiSchemaDocument.FindProjectSchemaNode(new("ed-fi")) ?? new JsonObject(),
                 NullLogger.Instance
@@ -392,18 +438,14 @@ RequestMethod method
         [SetUp]
         public async Task Setup()
         {
-            var queryParameters = new Dictionary<string, string>
-            {
-                { "isRequired", "false" }
-            };
+            var queryParameters = new Dictionary<string, string> { { "isRequired", "false" } };
 
-            FrontendRequest frontendRequest =
-                new(
-                    Path: "/ed-fi/academicWeeks",
-                    Body: null,
-                    QueryParameters: queryParameters,
-                    TraceId: new TraceId("")
-                );
+            FrontendRequest frontendRequest = new(
+                Path: "/ed-fi/academicWeeks",
+                Body: null,
+                QueryParameters: queryParameters,
+                TraceId: new TraceId("")
+            );
 
             _context = NewPipelineContext(frontendRequest, RequestMethod.GET);
 
@@ -418,7 +460,8 @@ public void It_provides_no_response()
     }
 
     [TestFixture]
-    public class Given_Pipeline_Context_With_Valid_Type_Query_DateTime_Parameter : ValidateQueryMiddlewareTests
+    public class Given_Pipeline_Context_With_Valid_Type_Query_DateTime_Parameter
+        : ValidateQueryMiddlewareTests
     {
         private PipelineContext _context = No.PipelineContext();
 
@@ -442,16 +485,15 @@ private static PipelineContext NewPipelineContext(
             RequestMethod method
         )
         {
-            PipelineContext docRefContext =
-                new(frontendRequest, method)
-                {
-                    ApiSchemaDocument = NewApiSchemaDocument(),
-                    PathComponents = new(
-                        ProjectNamespace: new("ed-fi"),
-                        EndpointName: new("academicWeeks"),
-                        DocumentUuid: No.DocumentUuid
-                    ),
-                };
+            PipelineContext docRefContext = new(frontendRequest, method)
+            {
+                ApiSchemaDocument = NewApiSchemaDocument(),
+                PathComponents = new(
+                    ProjectNamespace: new("ed-fi"),
+                    EndpointName: new("academicWeeks"),
+                    DocumentUuid: No.DocumentUuid
+                ),
+            };
             docRefContext.ProjectSchema = new ProjectSchema(
                 docRefContext.ApiSchemaDocument.FindProjectSchemaNode(new("ed-fi")) ?? new JsonObject(),
                 NullLogger.Instance
@@ -477,16 +519,15 @@ public async Task Setup()
         {
             var queryParameters = new Dictionary<string, string>
             {
-                { "beginDate", "2025-12-30 22:33:55.000" }
+                { "beginDate", "2025-12-30 22:33:55.000" },
             };
 
-            FrontendRequest frontendRequest =
-                new(
-                    Path: "/ed-fi/academicWeeks",
-                    Body: null,
-                    QueryParameters: queryParameters,
-                    TraceId: new TraceId("")
-                );
+            FrontendRequest frontendRequest = new(
+                Path: "/ed-fi/academicWeeks",
+                Body: null,
+                QueryParameters: queryParameters,
+                TraceId: new TraceId("")
+            );
 
             _context = NewPipelineContext(frontendRequest, RequestMethod.GET);