diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f43aed..b74c0ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,18 @@ +# edfi_api_client v0.2.0 +## New Features +- `EdFiClient.get_swagger()` now returns an EdFiSwagger class that parses OpenAPI Swagger specification. +- `EdFiClient.resources` and `EdFiClient.descriptors` lazily retrieves lists of respective endpoints from Swagger. +- `EdFiEndpoint` child class attributes `description` and `has_deletes` lazily retrieves this metadata from Swagger. + +## Under the hood +- Requests re-authenticate automatically, based on the expiration-time retrieved from the API. + + # edfi_api_client v0.1.4 ## Fixes - Compatibility fix for Ed-Fi 6.0: casing changed for change version API responses + # edfi_api_client v0.1.2 ## New features - New "reverse_paging" pagination method for `EdFiResource.get_pages()` @@ -12,9 +23,11 @@ ## Fixes - Fix bug in `EdFiResource.get_pages()` where default `change_version_step_size` was used instead of argument + # edfi_api_client v0.1.1 ## Fixes -- retry on 500 errors +- Retry on 500 errors + # edfi_api_client v0.1.0 Initial release diff --git a/README.md b/README.md index 795860a..8245750 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,50 @@ Client key and secret not provided. Connection with ODS will not be attempted. Connection to ODS successful! ``` +### Attributes + +Authentication with the ODS is not required: + +
+resources + +----- + +### resources +This method is unavailable in Ed-Fi2. + +Retrieve a list of namespaced-resources from the `resources` Swagger payload. + +```python +>>> api.resources +[('ed-fi', 'academicWeeks'), ('ed-fi', 'accounts'), ('ed-fi', 'accountCodes'), ...] +``` + +----- + +
+ + +
+descriptors + +----- + +### descriptors +This method is unavailable in Ed-Fi2. + +Retrieve a list of namespaced-descriptors from the `descriptors` Swagger payload. + +```python +>>> api.descriptors +[('ed-fi', 'absenceEventCategoryDescriptors'), ('ed-fi', 'academicHonorCategoryDescriptors'), ...] +``` +----- + +
+ + + ### Methods Authentication with the ODS is not required: @@ -187,8 +231,7 @@ If `component` is unspecified, `resources` will be collected. ...} ``` -Note: the returned dictionary is large and unwieldy*. -A future update will add an `EdFiSwagger` class to assist in navigation. +Returns an `EdFiSwagger` class containing the complete JSON payload, as well as extracted metadata from the Swagger. ----- @@ -348,6 +391,46 @@ All methods that return `EdFiEndpoint` and child classes require a session with ``` +### Attributes + +
+description + +----- + +### description +This attribute retrieves the Ed-Fi endpoint's description if present in its respective Swagger payload. + +```python +>>> api.resource('bellSchedules').description +'This entity represents the schedule of class period meeting times.' +``` + + +----- + +
+ + +
+has_deletes + +----- + +### has_deletes +This attribute returns whether a deletes path is present the Ed-Fi endpoint's respective Swagger payload. + +```python +>>> api.resource('bellSchedules').has_deletes +True +``` + +----- + +
+ + + ### Methods
diff --git a/edfi_api_client/edfi_client.py b/edfi_api_client/edfi_client.py index 1fb36cc..8a2e1ec 100644 --- a/edfi_api_client/edfi_client.py +++ b/edfi_api_client/edfi_client.py @@ -7,7 +7,8 @@ from typing import Callable, Optional from edfi_api_client import util -from edfi_api_client.edfi_endpoint import EdFiResource, EdFiComposite +from edfi_api_client.edfi_endpoint import EdFiResource, EdFiDescriptor, EdFiComposite +from edfi_api_client.edfi_swagger import EdFiSwagger class EdFiClient: @@ -69,6 +70,15 @@ def __init__(self, self.version_url_string = self._get_version_url_string() self.instance_locator = self.get_instance_locator() + # Swagger variables for populating resource metadata (retrieved lazily) + self.swaggers = { + 'resources' : None, + 'descriptors': None, + 'composites' : None, + } + self._resources = None + self._descriptors = None + # If ID and secret are passed, build a session. self.session = None @@ -151,7 +161,8 @@ def get_data_model_version(self) -> Optional[str]: return None - def get_swagger(self, component: str = 'resources') -> dict: + ### Methods related to retrieving the Swagger or attributes retrieved therein + def get_swagger(self, component: str = 'resources') -> EdFiSwagger: """ OpenAPI Specification describes the entire Ed-Fi API surface in a JSON payload. @@ -163,7 +174,49 @@ def get_swagger(self, component: str = 'resources') -> dict: swagger_url = util.url_join( self.base_url, 'metadata', self.version_url_string, component, 'swagger.json' ) - return requests.get(swagger_url, verify=self.verify_ssl).json() + + payload = requests.get(swagger_url, verify=self.verify_ssl).json() + swagger = EdFiSwagger(component, payload) + + # Save the swagger in memory to save time on subsequent calls. + self.swaggers[component] = swagger + return swagger + + def _set_swagger(self, component: str): + """ + Populate the respective swagger object in `self.swaggers` if not already populated. + + :param component: + :return: + """ + if self.swaggers.get(component) is None: + self.verbose_log( + f"[Get {component.title()} Swagger] Retrieving Swagger into memory..." + ) + self.get_swagger(component) + + + @property + def resources(self): + """ + + :return: + """ + if self._resources is None: + self._set_swagger('resources') + self._resources = self.swaggers['resources'].endpoints + return self._resources + + @property + def descriptors(self): + """ + + :return: + """ + if self._descriptors is None: + self._set_swagger('descriptors') + self._descriptors = self.swaggers['descriptors'].endpoints + return self._descriptors ### Helper methods for building elements of endpoint URLs for GETs and POSTs @@ -246,8 +299,9 @@ def connect(self) -> requests.Session: self.session = requests.Session() self.session.headers.update(req_header) - # Add new attribute to track when connection was established. + # Add new attributes to track when connection was established and when to refresh the access token. self.session.timestamp_unix = int(time.time()) + self.session.refresh_time = int(self.session.timestamp_unix + access_response.json().get('expires_in') - 120) self.session.verify = self.verify_ssl self.verbose_log("Connection to ODS successful!") @@ -321,12 +375,13 @@ def descriptor(self, params: Optional[dict] = None, **kwargs - ) -> EdFiResource: + ) -> EdFiDescriptor: """ Even though descriptors and resources are accessed via the same endpoint, this may not be known to users, so a separate method is defined. """ - return self.resource( + return EdFiDescriptor( + client=self, name=name, namespace=namespace, get_deletes=False, params=params, **kwargs ) @@ -384,6 +439,18 @@ def get_swagger(self, component: str = 'resources') -> dict: "Swagger specification not implemented in Ed-Fi 2." ) + @property + def resources(self): + raise NotImplementedError( + "Resources collected from Swagger specification that is not implemented in Ed-Fi 2." + ) + + @property + def descriptors(self): + raise NotImplementedError( + "Descriptors collected from Swagger specification that is not implemented in Ed-Fi 2." + ) + ### Helper methods for building elements of endpoint URLs for GETs and POSTs def _get_version_url_string(self) -> str: @@ -442,8 +509,9 @@ def connect(self) -> requests.Session: self.session.headers.update(req_header) self.session.headers.update(json_header) - # Add new attribute to track when connection was established. + # Add new attributes to track when connection was established and when to refresh the access token. self.session.timestamp_unix = int(time.time()) + self.session.refresh_time = int(self.session.timestamp_unix + access_response.json().get('expires_in') - 120) self.session.verify = self.verify_ssl self.verbose_log("Connection to ODS successful!") diff --git a/edfi_api_client/edfi_endpoint.py b/edfi_api_client/edfi_endpoint.py index 07a7ee6..d2a9fdc 100644 --- a/edfi_api_client/edfi_endpoint.py +++ b/edfi_api_client/edfi_endpoint.py @@ -3,8 +3,9 @@ import requests import time +from functools import wraps from requests.exceptions import HTTPError, RequestsWarning -from typing import Iterator, List, Optional +from typing import Callable, Iterator, List, Optional, Tuple, Union from edfi_api_client.edfi_params import EdFiParams from edfi_api_client import util @@ -20,26 +21,47 @@ class EdFiEndpoint: """ client: 'EdFiClient' name: str - namespace: str + namespace: Optional[str] url: str params: EdFiParams + # Swagger name and attributes loaded lazily from Swagger + swagger_type: str + _description: Optional[str] = None + _has_deletes: Optional[bool] = None - @abc.abstractmethod - def build_url(self, - name: str, - *, - namespace: str = 'ed-fi', - **kwargs + def __init__(self, + client: 'EdFiClient', + name: Union[str, Tuple[str, str]], + namespace: str = 'ed-fi' ): + self.client: 'EdFiClient' = client + + # Name and namespace can be passed manually + if isinstance(name, str): + self.name: str = util.snake_to_camel(name) + self.namespace: str = namespace + + # Or as a `(namespace, name)` tuple as output from Swagger + else: + try: + self.namespace, self.name = name + except ValueError: + logging.error( + "Arguments `name` and `namespace` must be passed explicitly, or as a `(namespace, name)` tuple." + ) + + # Namespaces are not implemented in EdFi 2.x. + if self.client.is_edfi2(): + self.namespace = None + + + @abc.abstractmethod + def build_url(self): """ This method builds the endpoint URL with namespacing and optional pathing. - - :param name: - :param namespace: - :param kwargs: :return: """ raise NotImplementedError @@ -146,7 +168,58 @@ def total_count(self) -> int: raise NotImplementedError + @property + def description(self): + if self._description is None: + self._description = self._get_attributes_from_swagger()['description'] + return self._description + + @property + def has_deletes(self): + if self._has_deletes is None: + self._has_deletes = self._get_attributes_from_swagger()['has_deletes'] + return self._has_deletes + + + def _get_attributes_from_swagger(self): + """ + Retrieve endpoint-metadata from the Swagger document. + + Populate the respective swagger object in `self.client` if not already populated. + + :return: + """ + # Only GET the Swagger if not already populated in the client. + self.client._set_swagger(self.swagger_type) + swagger = self.client.swaggers[self.swagger_type] + + # Populate the attributes found in the swagger. + return { + 'description': swagger.descriptions.get(self.name), + 'has_deletes': (self.namespace, self.name) in swagger.deletes, + } + + ### Internal GET response methods and error-handling + def reconnect_if_expired(func: Callable) -> Callable: + """ + This decorator resets the connection with the API if expired. + + :param func: + :return: + """ + @wraps(func) + def wrapped(self, *args, **kwargs): + # Refresh token if refresh_time has passed + if self.client.session.refresh_time < int(time.time()): + self.client.verbose_log( + "Session authentication is expired. Attempting reconnection..." + ) + self.client.connect() + return func(self, *args, **kwargs) + return wrapped + + @reconnect_if_expired def _get_response(self, url: str, params: Optional[EdFiParams] = None @@ -163,6 +236,7 @@ def _get_response(self, return response + @reconnect_if_expired def _get_response_with_exponential_backoff(self, url: str, params: Optional[EdFiParams] = None, @@ -182,14 +256,10 @@ def _get_response_with_exponential_backoff(self, :return: """ # Attempt the GET until success or `max_retries` reached. - response = None - for n_tries in range(max_retries): try: - response = self.client.session.get(url, params=params) - self.custom_raise_for_status(response) - return response + return self._get_response(url, params=params) except RequestsWarning: # If an API call fails, it may be due to rate-limiting. @@ -197,28 +267,18 @@ def _get_response_with_exponential_backoff(self, time.sleep( min((2 ** n_tries) * 2, max_wait) ) - - # Tokens have expiry times; refresh the token if it expires mid-run. - authentication_delta = int(time.time()) - self.client.session.timestamp_unix - - self.client.verbose_log( - f"Maybe the session needs re-authentication? ({util.seconds_to_text(authentication_delta)} since last authentication)\n" - "Attempting reconnection..." - ) - self.client.connect() - logging.warning(f"Retry number: {n_tries}") # This block is reached only if max_retries has been reached. else: - logging.error("API GET failed: max retries exceeded for URL.") - self.client.verbose_log(message=( f"[Get with Retry Failed] Endpoint : {url}\n" f"[Get with Retry Failed] Parameters: {params}" ), verbose=True) - self.custom_raise_for_status(response) + raise RuntimeError( + "API GET failed: max retries exceeded for URL." + ) @staticmethod @@ -235,12 +295,12 @@ def custom_raise_for_status(response): f"API Error: {response.status_code} {response.reason}" ) if response.status_code == 400: - raise RequestsWarning( - "400: Bad request. Check your params. Is 'limit' set too high? Does the connection need to be reset?" + raise HTTPError( + "400: Bad request. Check your params. Is 'limit' set too high?" ) elif response.status_code == 401: raise RequestsWarning( - "401: Unauthorized for URL. The connection may need to be reset." + "401: Unauthenticated for URL. The connection may need to be reset." ) elif response.status_code == 403: # Only raise an HTTPError where the resource is impossible to access. @@ -284,19 +344,18 @@ def __init__(self, params: Optional[dict] = None, **kwargs ): - self.client: 'EdFiClient' = client - self.name: str = util.snake_to_camel(name) - self.namespace: str = namespace + super().__init__(client, name, namespace) self.get_deletes: bool = get_deletes - self.url = self.build_url(self.name, namespace=self.namespace, get_deletes=self.get_deletes) + self.url = self.build_url() self.params = EdFiParams(params, **kwargs) + self.swagger_type = 'resources' + def __repr__(self): """ - Resource (Deletes) [{namespace}/{name}] - with {N} parameters + Resource (Deletes) (with {N} parameters) [{namespace}/{name}] """ _deletes_string = " Deletes" if self.get_deletes else "" _params_string = f" with {len(self.params.keys())} parameters" if self.params else "" @@ -305,13 +364,7 @@ def __repr__(self): return f"" - def build_url(self, - name: str, - - *, - namespace: str = 'ed-fi', - get_deletes: bool = False - ) -> str: + def build_url(self) -> str: """ Build the name/descriptor URL to GET from the API. @@ -320,20 +373,14 @@ def build_url(self, :param get_deletes: :return: """ - # Namespaces are not implemented in EdFi 2.x. - if self.client.is_edfi2(): - namespace = None - # Deletes are an optional path addition. - deletes = None - if get_deletes: - deletes = 'deletes' + deletes = 'deletes' if self.get_deletes else None return util.url_join( self.client.base_url, self.client.version_url_string, self.client.instance_locator, - namespace, name, deletes + self.namespace, self.name, deletes ) @@ -488,6 +535,15 @@ def _get_total_count(self, params: EdFiParams): return int(res.headers.get('Total-Count')) +class EdFiDescriptor(EdFiResource): + """ + Ed-Fi Descriptors are used identically to Resources, but they are listed in a separate Swagger. + + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.swagger_type = 'descriptors' + class EdFiComposite(EdFiEndpoint): """ @@ -506,19 +562,16 @@ def __init__(self, params: Optional[dict] = None, **kwargs ): - self.client: 'EdFiClient' = client - self.name: str = util.snake_to_camel(name) - self.namespace: str = namespace + super().__init__(client, name, namespace) self.composite: str = composite self.filter_type: Optional[str] = filter_type self.filter_id: Optional[str] = filter_id - self.url = self.build_url( - self.name, namespace=self.namespace, composite=self.composite, - filter_type=self.filter_type, filter_id=self.filter_id - ) + self.url = self.build_url() self.params = EdFiParams(params, **kwargs) + self.swagger_type = 'composites' + def __repr__(self): """ @@ -533,40 +586,26 @@ def __repr__(self): return f"<{_composite} Composite{_params_string} [{_full_name}]{_filter_string}>" - def build_url(self, - name: str, - - *, - namespace: str = 'ed-fi', - composite: str = 'enrollment', - - filter_type: Optional[str] = None, - filter_id: Optional[str] = None, - ) -> str: + def build_url(self) -> str: """ Build the composite URL to GET from the API. - :param name: - :param namespace: - :param composite: - :param filter_type: - :param filter_id: :return: """ - # Namespaces are not implemented in EdFi 2.x. - if self.client.is_edfi2(): - namespace = None - # If a filter is applied, the URL changes to match the filter type. - if filter_type is None and filter_id is None: + if self.filter_type is None and self.filter_id is None: return util.url_join( - self.client.base_url, 'composites/v1', self.client.instance_locator, namespace, composite, name.title() + self.client.base_url, 'composites/v1', + self.client.instance_locator, + self.namespace, self.composite, self.name.title() ) - elif filter_type is not None and filter_id is not None: + elif self.filter_type is not None and self.filter_id is not None: return util.url_join( - self.client.base_url, 'composites/v1', self.client.instance_locator, namespace, composite, - filter_type, filter_id, name + self.client.base_url, 'composites/v1', + self.client.instance_locator, + self.namespace, self.composite, + self.filter_type, self.filter_id, self.name ) else: diff --git a/edfi_api_client/edfi_swagger.py b/edfi_api_client/edfi_swagger.py new file mode 100644 index 0000000..81a95a1 --- /dev/null +++ b/edfi_api_client/edfi_swagger.py @@ -0,0 +1,112 @@ +from collections import defaultdict +from typing import List + +from edfi_api_client.util import camel_to_snake + + +class EdFiSwagger: + """ + """ + def __init__(self, component: str, swagger_payload: dict): + """ + TODO: Can `component` be extracted from the swagger? + + :param component: Type of swagger payload passed (i.e., 'resources' or 'descriptors') + :param swagger_payload: + :return: + """ + self.type: str = component + self.json: dict = swagger_payload + + self.version: str = self.json.get('swagger') + self.version_url_string: str = self.json.get('basePath') + + self.token_url: str = ( + self.json + .get('securityDefinitions', {}) + .get('oauth2_client_credentials', {}) + .get('tokenUrl') + ) + + # Extract namespaces and endpoints, and whether there is a deletes endpoint from `paths` + _endpoint_deletes = self._get_namespaced_endpoints_and_deletes() + self.endpoints: list = list(_endpoint_deletes.keys()) + self.deletes : list = list(filter(_endpoint_deletes.get, _endpoint_deletes)) # Filter where values are True + + # Extract resource descriptions from `tags` + self.descriptions: dict = self.get_descriptions() + + # Extract surrogate keys from `definitions` + self.reference_skeys: dict = self.get_reference_skeys(exclude=['link',]) + + + def __repr__(self): + """ + Ed-Fi {self.type} OpenAPI Swagger Specification + """ + return f"" + + + def _get_namespaced_endpoints_and_deletes(self): + """ + Internal function to parse values in `paths`. + + Extract each Ed-Fi namespace and resource, and whether it has an optional deletes tag. + (namespace: str, resource: str) -> has_deletes: bool + + Swagger's `paths` is a dictionary of Ed-Fi pathing keys (up-to-three keys per resource/descriptor). + For example: + '/ed-fi/studentSchoolAssociations' + '/ed-fi/studentSchoolAssociations/{id}' + '/ed-fi/studentSchoolAssociations/deletes' + + :return: + """ + resource_deletes = defaultdict(bool) + + for path in self.json.get('paths', {}).keys(): + namespace = path.split('/')[1] + resource = path.split('/')[2] + has_deletes = ('/deletes' in path) + + resource_deletes[ (namespace, resource) ] |= has_deletes + + return resource_deletes + + + def get_descriptions(self): + """ + Descriptions for all EdFi endpoints are found under `tags` as [name, description] JSON objects. + Their extraction is optional for YAML templates, but they look nice. + + :param swagger: Swagger JSON object + :return: + """ + return { + tag['name']: tag['description'] + for tag in self.json['tags'] + } + + + def get_reference_skeys(self, exclude: List[str]): + """ + Build surrogate key definition column mappings for each Ed-Fi reference. + + :return: + """ + skey_mapping = {} + + for key, definition in self.json.get('definitions', {}).items(): + + # Only reference surrogate keys are used + if not key.endswith('Reference'): + continue + + reference = key.split('_')[1] # e.g.`edFi_staffReference` + + columns = definition.get('properties', {}).keys() + columns = list(filter(lambda x: x not in exclude, columns)) # Remove columns to be excluded. + + skey_mapping[reference] = columns + + return skey_mapping diff --git a/edfi_api_client/util.py b/edfi_api_client/util.py index 27c275f..f938b3f 100644 --- a/edfi_api_client/util.py +++ b/edfi_api_client/util.py @@ -30,27 +30,3 @@ def url_join(*args) -> str: return '/'.join( map(lambda x: str(x).rstrip('/'), filter(lambda x: x is not None, args)) ) - - -def seconds_to_text(seconds: int) -> str: - """ - - :param seconds: - :return: - """ - delta_str = str(datetime.timedelta(seconds=seconds)) - hours, minutes, seconds = list(map(int, delta_str.split(':'))) - - time_strings = [] - - if hours: - desc = "hours" if hours > 1 else "hour" - time_strings.append( f"{hours} {desc}") - if minutes: - desc = "minutes" if minutes > 1 else "minute" - time_strings.append( f"{minutes} {desc}" ) - if seconds: - desc = "seconds" if seconds > 1 else "second" - time_strings.append( f"{seconds} {desc}" ) - - return ", ".join(time_strings) diff --git a/setup.py b/setup.py index d7fef7a..c218fb8 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name='edfi_api_client', - version='0.1.4', + version='0.2.0', description='Ed-Fi API client and tools', license_files=['LICENSE'], url='https://github.com/edanalytics/edfi_api_client',