[DMS-803] Add filtered read test suite

src/edfi-paging-test/README.md

@@ -26,19 +26,21 @@ poetry run python edfi_paging_test -b "https://localhost:54746" -k "testkey" -s
 
 ### Supported arguments
 
-| Command Line Argument                | Required                             | Description                                                                                   |
-| ------------------------------------ | ------------------------------------ | --------------------------------------------------------------------------------------------- |
-| `-b` or `--baseUrl`                  | yes (no default)                     | ​The base url used to derived api, metadata, oauth, and dependency urls (e.g., http://server)  |
-| `-k` or `--key`                      | yes (no default)                     | The web API OAuth key                                                                         |
-| `-s` or `--secret`                   | yes (no default)                     | The web API OAuth secret                                                                      |
-| `-i` or `--ignoreCertificateErrors`  | no (default: false)                  | Ignore certificate errors                                                                     |
-| `-c` or `--connectionLimit`          | no (default: 5)                      | Maximum concurrent connections to api                                                         |
-| `-o` or `--output`                   | no (default: out)                    | Directory for writing results                                                                 |
-| `-t` or `--contentType`              | no (default: csv)                    | Output file content type: CSV, JSON                                                           |
-| `-r` or `--resourceList`             | no (no default)                      | (Optional) List of resources to test  - if not provided, all resources will be retrieved      |
-| `-p` or `--pageSize`                 | no (default: 100)                    | The page size to request. Max: 500.                                                           |
-| `-l` or `--logLevel`                 | no (default: INFO)                   | Override the console output log level: VERBOSE, DEBUG, INFO, WARN, ERROR                      |
-| `-d` or `--description`              | no (default: Paging Volume Test Run) | Description for the test run                                                                  |
+| Command Line Argument                | Required                             | Description                                                                                                        |
+| ------------------------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------|
+| `-b` or `--baseUrl`                  | yes (no default)                     | ​The base url used to derived api, metadata, oauth, and dependency urls (e.g., http://server)                       |
+| `-k` or `--key`                      | yes (no default)                     | The web API OAuth key                                                                                              |
+| `-s` or `--secret`                   | yes (no default)                     | The web API OAuth secret                                                                                           |
+| `-i` or `--ignoreCertificateErrors`  | no (default: false)                  | Ignore certificate errors                                                                                          |
+| `-c` or `--connectionLimit`          | no (default: 5)                      | Maximum concurrent connections to api                                                                              |
+| `-o` or `--output`                   | no (default: out)                    | Directory for writing results                                                                                      |
+| `-t` or `--contentType`              | no (default: csv)                    | Output file content type: CSV, JSON                                                                                |
+| `-r` or `--resourceList`             | no (no default)                      | (Optional) List of resources to test  - if not provided, all resources will be retrieved                           |
+| `-p` or `--pageSize`                 | no (default: 100)                    | The page size to request. Max: 500.                                                                                |
+| `-l` or `--logLevel`                 | no (default: INFO)                   | Override the console output log level: VERBOSE, DEBUG, INFO, WARN, ERROR                                           |
+| `-d` or `--description`              | no (default: Paging Volume Test Run) | Description for the test run                                                                                       |
+| `-e` or `--testType`                 | no (default: DEEP_PAGING)            | Type of test to run: DEEP_PAGING, FILTERED_READ                                                                    |
+| `-f` or `--combinationSizeLimit`     | no (default: 6)                      | The maximumum size of the filter combinations to test per resource. For use with the 'FILTERED_READ' testType only |
 
 Each argument can also be set by environment variable, or by using as `.env`
 file. See [.env.example](edfi_paging_test/.env.example). Arguments provided at

src/edfi-paging-test/edfi_paging_test/.env.example

@@ -14,3 +14,6 @@ PERF_RESOURCE_LIST="["students", "studentEducationOrganizationAssociations"]"
 
 # Set to true when using HTTP instead of HTTPS or when using self-signed certificates
 IGNORE_TLS_CERTIFICATE=false
+
+PERF_PAGING_TEST_TYPE=DEEP_PAGING
+PERF_FILTER_COMBINATION_SIZE_LIMIT=6

src/edfi-paging-test/edfi_paging_test/api/request_client.py

@@ -19,8 +19,10 @@
 from edfi_paging_test.api.paginated_result import PaginatedResult
 from edfi_paging_test.api.api_info import APIInfo
 from edfi_paging_test.helpers.argparser import MainArguments
-from edfi_paging_test.reporter.request_logger import log_request
+from edfi_paging_test.reporter.paging_request_logger import PaggingRequestLogger
+from edfi_paging_test.reporter.filtered_read_request_logger import FilteredReadRequestLogger
 from edfi_paging_test.helpers.api_metadata import get_base_api_response
+from urllib.parse import quote
 
 EDFI_DATA_MODEL_NAME = "ed-fi"
 
@@ -110,7 +112,7 @@ def _authorize(self) -> None:
         self.oauth.fetch_token(
             self._get_api_info().oauth_url, auth=self.auth,
             verify=self.verify_cert
-            )
+        )
 
     def _urljoin(self, base_url: str, relative_url: str) -> str:
         """
@@ -214,7 +216,7 @@ def get_total(self, resource: str) -> int:
         )
         return 0
 
-    def get_page(self, resource: str, page: int = 1) -> PaginatedResult:
+    def get_page(self, resource: str, pagingRequestLogger: PaggingRequestLogger, page: int = 1) -> PaginatedResult:
         """Send an HTTP GET request for the next page.
 
         Returns
@@ -230,9 +232,10 @@ def get_page(self, resource: str, page: int = 1) -> PaginatedResult:
         logger.debug(f"GET {next_url}")
         elapsed, response = timeit(lambda: self._get(next_url))
 
-        items = response.json() if len(response.text) > 0 else []
+        items = response.json() if len(
+            response.text) > 0 and response.status_code == HTTPStatus.OK else []
 
-        log_request(
+        pagingRequestLogger.log_request(
             resource,
             next_url,
             page,
@@ -250,7 +253,7 @@ def get_page(self, resource: str, page: int = 1) -> PaginatedResult:
             status_code=response.status_code,
         )
 
-    def get_all(self, resource: str) -> List[Dict[str, Any]]:
+    def get_all(self, resource: str, pagingRequestLogger: PaggingRequestLogger) -> List[Dict[str, Any]]:
         """
         Send an HTTP GET request for all pages of a resource.
 
@@ -267,18 +270,48 @@ def get_all(self, resource: str) -> List[Dict[str, Any]]:
 
         logger.info(f"Retrieving all {resource} records...")
 
-        pagination_result = self.get_page(resource, 1)
+        pagination_result = self.get_page(resource, pagingRequestLogger, 1)
         page_items = pagination_result.current_page_items
         # Assign to empty list if result is not a list, e.g. an error response from the API
         items: List[Any] = page_items if (isinstance(page_items, list)) else []
 
         while True:
             pagination_result = self.get_page(
-                resource, pagination_result.current_page + 1
+                resource, pagingRequestLogger, pagination_result.current_page + 1
             )
             items.extend(pagination_result.current_page_items)
 
             if pagination_result.size < self.page_size:
                 break
 
         return items
+
+    def filtered_get(self, resource_name: str, filters: Dict[str, str], limit: int, filteredReadRequestLogger: FilteredReadRequestLogger) -> List[Dict[str, Any]]:
+        """
+        Sends an HTTP GET request to the given resource applying the provider filters.
+
+        Args:
+            resource_name: Name of the resource to query
+            filters: Dictionary of filter name and value
+            limit: Number of entries to retrieve
+
+        Returns:
+            List containing the matching resource entries
+        """
+
+        query_string = '&'.join([f"{key}={quote(str(value))}" for key, value in filters.items()])
+        url = f"{self._build_url_for_resource(resource_name)}?limit={limit}&{query_string}"
+
+        elapsed, response = timeit(lambda: self._get(url))
+
+        items = response.json() if len(response.text) > 0 and response.status_code == HTTPStatus.OK else []
+
+        filteredReadRequestLogger.log_request(
+            resource_name,
+            url,
+            len(filters),
+            elapsed,
+            response.status_code,
+        )
+
+        return items

src/edfi-paging-test/edfi_paging_test/helpers/api_metadata.py

@@ -35,7 +35,7 @@ def get_base_api_response(api_base_url: str, verify_cert: bool = True) -> Dict[s
         return requests.get(
             api_base_url,
             verify=verify_cert
-            ).json()
+        ).json()
     except requests.exceptions.RequestException as e:
         raise RuntimeError(f"Error: {e}.") from e
 
@@ -61,7 +61,7 @@ def get_openapi_metadata_response(api_base_url: str, verify_cert: bool = True) -
     return requests.get(
         base_api_response["urls"]["openApiMetadata"],
         verify=verify_cert
-        ).json()
+    ).json()
 
 
 @cache
@@ -88,44 +88,48 @@ def get_resource_metadata_response(api_base_url: str, verify_cert: bool = True)
     return requests.get(
         resource_metadata["endpointUri"],
         verify=verify_cert
-        ).json()
+    ).json()
 
 
-def get_resource_paths(api_base_url: str, verify_cert: bool = True) -> List[str]:
+def get_filters_by_resource_name(api_base_url: str, verify_cert: bool = True) -> Dict[str, List[str]]:
     """
-    Gets the resources for the API as relative paths, including the
-    project/extension prefix.
+    Analyzes the OpenAPI metadata to determine which filters are available
+    for each resource endpoint.
 
-    Parameters
-    ----------
-    api_base_url : str
-        The base URL of the API.
+    Args:
+        api_base_url (str): Base URL of the Ed-Fi API
+        verify_cert (bool): Whether to verify SSL certificates
 
-    Returns
-    -------
-    List[str]
-        A list of resource relative paths, including the extension prefix if
-        relevant. For example: ["schools", "tpdm/candidates"]
+    Returns:
+        Dict[str, List[str]]: Dictionary mapping resource names to their
+                                    available filters
     """
-    resource_metadata_response: Dict[
-        str, Dict[str, str]
-    ] = get_resource_metadata_response(api_base_url, verify_cert)
-    all_paths: List[str] = list(resource_metadata_response["paths"].keys())
+
+    resource_metadata_response: Dict[str, Dict[str, Any]
+                                     ] = get_resource_metadata_response(api_base_url, verify_cert)
+
     # filter out paths that are for get by id, deletes, keyChanges or partitions
-    return list(
-        filter(
-            lambda p: (
-                "{id}" not in p
-                and "/deletes" not in p
-                and "/keyChanges" not in p
-                and "/partitions" not in p
-            )
-            , all_paths
-        )
-    )
+    operations_by_resource = [(normalize_resource_path(path), operations) for path, operations in resource_metadata_response["paths"].items()
+                              if "{id}" not in path
+                              and "/deletes" not in path
+                              and "/keyChanges" not in path
+                              and "/partitions" not in path]
+
+    filters_by_resource = {resource: get_filters(
+        operations) for resource, operations in operations_by_resource}
 
+    return filters_by_resource
 
-def normalize_resource_paths(resource_paths: List[str]) -> List[str]:
+
+def get_filters(operations) -> List[str]:
+    pagination_params = {'limit', 'offset', 'totalCount'}
+    return [param['name'] for param in operations['get']['parameters']
+            if 'in' in param and param['in'] == 'query'
+            and param['name'] != 'id'
+            and param['name'] not in pagination_params]
+
+
+def normalize_resource_path(resource_path: str) -> str:
     """
     Takes a list of resource relative paths and normalizes to lowercase
     and with the "ed-fi" namespace prefix removed.
@@ -137,13 +141,8 @@ def normalize_resource_paths(resource_paths: List[str]) -> List[str]:
 
     Returns
     -------
-    List[str]
-        A list of normalized resource relative paths.
-        For example: ["studentschoolassociations", "tpdm/candidates"]
+    str
+        A normalized resource relative path.
+        For example: "studentschoolassociations", "tpdm/candidates"
     """
-    return list(
-        map(
-            lambda r: r.removeprefix("/").removeprefix("ed-fi/"),
-            resource_paths,
-        )
-    )
+    return resource_path.removeprefix("/").removeprefix("ed-fi/")

src/edfi-paging-test/edfi_paging_test/helpers/argparser.py

@@ -8,6 +8,7 @@
 from edfi_paging_test.helpers.output_format import OutputFormat
 from edfi_paging_test.helpers.log_level import LogLevel
 from edfi_paging_test.helpers.main_arguments import MainArguments
+from edfi_paging_test.helpers.test_type import TestType
 
 
 def parse_main_arguments() -> MainArguments:
@@ -113,6 +114,23 @@ def parse_main_arguments() -> MainArguments:
         default="Paging Volume Test Run",
         env_var="PERF_DESCRIPTION",
     )
+    parser.add(  # type: ignore
+        "-e",
+        "--testType",
+        help="Type of test to run: DEEP_PAGING, FILTERED_READ",
+        type=TestType,
+        choices=list(TestType),
+        default=TestType.DEEP_PAGING,
+        env_var="PERF_PAGING_TEST_TYPE",
+    )
+    parser.add(  # type: ignore
+        "-f",
+        "--combinationSizeLimit",
+        help="The maximumum size of the filter combinations to test per resource. For use with the 'FILTERED_READ' testType only",
+        type=int,
+        default=6,
+        env_var="PERF_FILTER_COMBINATION_SIZE_LIMIT",
+    )
 
     args_parsed = parser.parse_args()
 
@@ -128,6 +146,8 @@ def parse_main_arguments() -> MainArguments:
         args_parsed.resourceList or [],
         args_parsed.pageSize,
         args_parsed.logLevel,
+        args_parsed.testType,
+        args_parsed.combinationSizeLimit,
     )
 
     return arguments

src/edfi-paging-test/edfi_paging_test/helpers/main_arguments.py

@@ -8,6 +8,7 @@
 
 from edfi_paging_test.helpers.output_format import OutputFormat
 from edfi_paging_test.helpers.log_level import LogLevel
+from edfi_paging_test.helpers.test_type import TestType
 
 
 @dataclass
@@ -27,3 +28,5 @@ class MainArguments:
     resourceList: List[str]
     pageSize: int = 100
     log_level: LogLevel = LogLevel.INFO
+    test_type: TestType = TestType.DEEP_PAGING
+    combination_size_limit: int = 6

src/edfi-paging-test/edfi_paging_test/helpers/resource_entries_cache.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: Apache-2.0
+# Licensed to the Ed-Fi Alliance under one or more agreements.
+# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+# See the LICENSE and NOTICES files in the project root for more information.
+
+import random
+from typing import Any, Dict, List, Tuple
+
+
+class ResourceEntriesCache:
+
+    def __init__(self, entries: List[Dict[str, Any]], filters: List[str]):
+        """
+        Stores the given resource entries in a columnar fashion to speed up local searches.
+
+        Args:
+            entries: Resource entries
+            filters: List of filters that a resource supports
+        """
+
+        # Create a dictionary where the key is the filter name and the value is a list of entries
+        # with a non-null value for the given filter
+        self.resources_by_filter_name: Dict[str, list[Dict[str, Any]]] = {filter: [] for filter in filters}
+
+        for entry in entries:
+            for filter in filters:
+                if filter in entry and entry[filter] is not None:
+                    self.resources_by_filter_name[filter].append(entry)
+
+    def get_entries_with_non_null_filters(self, filters: Tuple[str, ...], count: int) -> List[Dict[str, Any]]:
+        """
+        Searches through the resource entries to find the ones that have non-null values
+        for all the specified filters, up to the requested count.
+
+        Args:
+            filters: The required filters
+            count: Maximum number of matching entries to return
+        """
+
+        # Get the filter with the highest cardinality
+        entries = list(self.resources_by_filter_name.items())[0][1]
+        for filter in filters:
+            if len(self.resources_by_filter_name[filter]) < len(entries):
+                entries = self.resources_by_filter_name[filter]
+
+        random.shuffle(entries)
+
+        result: List[Dict[str, Any]] = []
+        for entry in entries:
+            has_all_non_null_filters = True
+            for filter in filters:
+                if filter not in entry or entry[filter] is None:
+                    has_all_non_null_filters = False
+                    break
+
+            if has_all_non_null_filters:
+                result.append(entry)
+
+            if len(result) >= count:
+                break
+
+        return result

src/edfi-paging-test/edfi_paging_test/helpers/test_type.py

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# Licensed to the Ed-Fi Alliance under one or more agreements.
+# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+# See the LICENSE and NOTICES files in the project root for more information.
+
+from edfi_paging_test.helpers.case_insensitive_enum import CaseInsensitiveEnum
+
+
+class TestType(CaseInsensitiveEnum):
+    DEEP_PAGING = "DEEP_PAGING"
+    FILTERED_READ = "FILTERED_READ"
+
+    def __eq__(self, other: object) -> bool:
+        try:
+            return self.value == TestType(other).value
+        except:  # noqa: E722
+            return False
+
+    # Prevent pytest from trying to discover tests in this class
+    __test__ = False