From c82e19ef4ce5b42352ee650ff20a43af0cbaece0 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Mon, 7 Oct 2024 17:04:13 -0500 Subject: [PATCH 1/4] partial implementation of new lightbeam create functionality --- lightbeam/__main__.py | 2 + lightbeam/create.py | 111 +++++++++++++++++++++++++++++++++++++++++ lightbeam/lightbeam.py | 2 + 3 files changed, 115 insertions(+) create mode 100644 lightbeam/create.py diff --git a/lightbeam/__main__.py b/lightbeam/__main__.py index f614605..2187e85 100644 --- a/lightbeam/__main__.py +++ b/lightbeam/__main__.py @@ -23,6 +23,7 @@ def emit(self, record): "truncate": "truncate", "count": "count", "fetch": "fetch", + "create": "create" } command_list = ', '.join(f"'{c}'" for c in ALLOWED_COMMANDS.values()) @@ -163,6 +164,7 @@ def main(argv=None): try: logger.info("starting...") if args.command==ALLOWED_COMMANDS['count']: lb.counter.count() + if args.command==ALLOWED_COMMANDS['create']: lb.creator.create() elif args.command==ALLOWED_COMMANDS['fetch']: lb.fetcher.fetch() elif args.command==ALLOWED_COMMANDS['validate']: lb.validator.validate() elif args.command==ALLOWED_COMMANDS['send']: lb.sender.send() diff --git a/lightbeam/create.py b/lightbeam/create.py new file mode 100644 index 0000000..93e4f8a --- /dev/null +++ b/lightbeam/create.py @@ -0,0 +1,111 @@ +import os +import json +import yaml +from lightbeam import util + +class Creator: + + def __init__(self, lightbeam=None): + self.lightbeam = lightbeam + self.logger = self.lightbeam.logger + self.template_folder = "templates/" + self.earthmover_file = "earthmover.yml" + + def create(self): + os.makedirs(self.template_folder, exist_ok=True) + earthmover_yaml = {} + # check if file exists! + if os.path.isfile(self.earthmover_file): + with open(self.earthmover_file) as file: + earthmover_yaml = yaml.safe_load(file) + for endpoint in self.lightbeam.endpoints: + if endpoint in (earthmover_yaml.get("destinations", {}) or {}).keys(): + self.logger.critical(f"The file `{self.earthmover_file}` already exists in the current directory and contains `$destinations.{endpoint}`; to re-create it, please first manually remove it (and `{self.template_folder}{endpoint}.jsont`).") + self.create_jsont(endpoint) + # write out earthmover_yaml + if not os.path.isfile(self.earthmover_file): + self.logger.info(f"creating file `{self.earthmover_file}`...") + with open(self.earthmover_file, 'w+') as file: + file.write("""version: 2.0 + +# This is an earthmover.yml file, generated with `lightbeam create`, for creating Ed-Fi JSON payloads +# using earthmover. See https://github.com/edanalytics/earthmover for documentation. + +# Define your source data here: +sources: + # Example: + # mysource: + # file: path/to/myfile.csv + # ... + +# (If needed, define your data transformations here:) +# transformations: + +destinations:""") + for endpoint in self.lightbeam.endpoints: + file.write(self.create_em_destination_node(endpoint)) + else: + self.logger.info(f"appending to file `{self.earthmover_file}`...") + with open(self.earthmover_file, 'a+') as file: + for endpoint in self.lightbeam.endpoints: + file.write(self.create_em_destination_node(endpoint)) + + def create_em_destination_node(self, endpoint): + return f""" + {endpoint}: + source: $transformations.{endpoint} + template: {self.template_folder}{endpoint}.jsont + extension: jsonl + linearize: True""" + + def create_jsont(self, endpoint): + template_file = f"{self.template_folder}{endpoint}.jsont" + # check if file exists! + if os.path.isfile(template_file): + self.logger.critical(f"The file `{template_file}` already exists in the current directory; to re-create it, please first manually delete it.") + # write out json template + self.logger.info(f"creating file `{template_file}`...") + with open(template_file, 'w+') as file: + # TODO: implement a function in `lightbeam/api.py` that constructs a "sample" payload for the endpoint + # Example: + # { + # "property_bool": true, + # "property_int": 1, + # "property_float": 1.0, + # "property_string": "string", + # "property_date": "date", + # "property_descriptor": "uri://ed-fi.org/SomeDescriptor#SomeValue", + # "property_object": { + # "property_object_1": "string", + # "property_object_2": "string" + # }, + # "property_array": [ + # { + # "property_array_1": "string", + # "property_array_2": "string" + # } + # ] + # } + # TODO: turn the "sample" payload into a Jinja template + # Example: + # { + # "property_bool": {{property_bool}}, + # "property_int": {{property_int}}, + # "property_float": {{property_float}}, + # "property_string": "{{property_string}}", + # "property_date": "{{property_date}}", + # "property_descriptor": "uri://ed-fi.org/SomeDescriptor#{{property_descriptor}}", + # "property_object": { + # "property_object_1": "{{property_object_1}}", + # "property_object_2": "{{property_object_2}}" + # }, + # "property_array": [ + # {% for item in property_array %} + # { + # "property_array_1": "{{item.property_array_1}}", + # "property_array_2": "{{item.property_array_2}}" + # } {% if not loop.last %},{% endif %} + # {% endfor %} + # ] + # } + file.write("coming soon...") # (for now) diff --git a/lightbeam/lightbeam.py b/lightbeam/lightbeam.py index 9355e54..116f552 100644 --- a/lightbeam/lightbeam.py +++ b/lightbeam/lightbeam.py @@ -11,6 +11,7 @@ from lightbeam import util from lightbeam.api import EdFiAPI from lightbeam.count import Counter +from lightbeam.create import Creator from lightbeam.fetch import Fetcher from lightbeam.validate import Validator from lightbeam.send import Sender @@ -73,6 +74,7 @@ def __init__(self, config_file, logger=None, selector="*", exclude="", keep_keys self.endpoints = [] self.results = [] self.counter = Counter(self) + self.creator = Creator(self) self.fetcher = Fetcher(self) self.validator = Validator(self) self.sender = Sender(self) From 02d17a2c4a79c24402f3f21407459e0249661829 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Mon, 7 Oct 2024 17:23:56 -0500 Subject: [PATCH 2/4] update comments/examples --- lightbeam/create.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lightbeam/create.py b/lightbeam/create.py index 93e4f8a..715af66 100644 --- a/lightbeam/create.py +++ b/lightbeam/create.py @@ -74,6 +74,7 @@ def create_jsont(self, endpoint): # "property_float": 1.0, # "property_string": "string", # "property_date": "date", + # "property_string_optional": "string", # "property_descriptor": "uri://ed-fi.org/SomeDescriptor#SomeValue", # "property_object": { # "property_object_1": "string", @@ -94,6 +95,9 @@ def create_jsont(self, endpoint): # "property_float": {{property_float}}, # "property_string": "{{property_string}}", # "property_date": "{{property_date}}", + # {% if property_string_optional %} + # "property_string_optional": "{{property_string_optional}}", + # {% endif %} # "property_descriptor": "uri://ed-fi.org/SomeDescriptor#{{property_descriptor}}", # "property_object": { # "property_object_1": "{{property_object_1}}", From 269bc958d51bd43b67e462acbcac8b835d19b89f Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Fri, 22 Nov 2024 16:02:07 -0600 Subject: [PATCH 3/4] updates to finish out functionality --- lightbeam/api.py | 24 +++++++++++++ lightbeam/create.py | 84 ++++++++++++++++++++------------------------- 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/lightbeam/api.py b/lightbeam/api.py index f3ba6f2..f6e2197 100644 --- a/lightbeam/api.py +++ b/lightbeam/api.py @@ -337,6 +337,8 @@ def get_params_for_endpoint(self, endpoint, type='required'): definition = util.get_swagger_ref_for_endpoint(self.lightbeam.config["namespace"], swagger, endpoint) if type=='required': return self.get_required_params_from_swagger(swagger, definition) + elif type=='all': + return self.get_all_params_from_swagger(swagger, definition) else: # descriptor endpoints all have the same structure and identity fields: if "Descriptor" in endpoint: @@ -360,6 +362,28 @@ def get_required_params_from_swagger(self, swagger, definition, prefix=""): params[prop] = prefix + prop return params + def get_all_params_from_swagger(self, swagger, definition, prefix=""): + params = {} + schema = util.resolve_swagger_ref(swagger, definition) + if not schema: + self.logger.critical(f"Swagger contains neither `definitions` nor `components.schemas` - check that the Swagger is valid.") + + for prop in schema["properties"].keys(): + if prop in ["_etag", "id", "link"]: continue + if "required" in schema.keys() and prop in schema["required"]: prop_name = "[required]"+prop + else: prop_name = "[optional]"+prop + if "$ref" in schema["properties"][prop].keys(): + params[prop_name] = {} + sub_definition = schema["properties"][prop]["$ref"] + sub_params = self.get_all_params_from_swagger(swagger, sub_definition, prefix=prop+"_") + for k,v in sub_params.items(): + params[prop_name][k] = v + elif schema["properties"][prop]["type"]!="array": + params[prop_name] = f"[{schema['properties'][prop]['type']}]" + prefix + prop + else: + params[prop_name] = [self.get_all_params_from_swagger(swagger, schema["properties"][prop]["items"]["$ref"], prefix=prop+"_")] + return params + def get_identity_params_from_swagger(self, swagger, definition, prefix=""): params = {} schema = util.resolve_swagger_ref(swagger, definition) diff --git a/lightbeam/create.py b/lightbeam/create.py index 715af66..6ca388e 100644 --- a/lightbeam/create.py +++ b/lightbeam/create.py @@ -1,4 +1,5 @@ import os +import re import json import yaml from lightbeam import util @@ -12,6 +13,7 @@ def __init__(self, lightbeam=None): self.earthmover_file = "earthmover.yml" def create(self): + self.lightbeam.api.load_swagger_docs() os.makedirs(self.template_folder, exist_ok=True) earthmover_yaml = {} # check if file exists! @@ -31,6 +33,12 @@ def create(self): # This is an earthmover.yml file, generated with `lightbeam create`, for creating Ed-Fi JSON payloads # using earthmover. See https://github.com/edanalytics/earthmover for documentation. +config: + macros: > + {% macro descriptor_namespace() -%} + uri://ed-fi.org + {%- endmacro %} + # Define your source data here: sources: # Example: @@ -57,59 +65,41 @@ def create_em_destination_node(self, endpoint): template: {self.template_folder}{endpoint}.jsont extension: jsonl linearize: True""" + + def upper_repl(self, match): + value = match.group(1) + return "/" + value[0].upper() + value[1:] + "Descriptor#" + def create_jsont(self, endpoint): template_file = f"{self.template_folder}{endpoint}.jsont" # check if file exists! if os.path.isfile(template_file): self.logger.critical(f"The file `{template_file}` already exists in the current directory; to re-create it, please first manually delete it.") + # generate base JSON structure: + content = self.lightbeam.api.get_params_for_endpoint(endpoint, type='all') + # pretty-print it: + content = json.dumps(content, indent=2) + # annotate required/optional properties: + content = content.replace('"[required]', '{# (required) #} "') + content = content.replace('"[optional]', '{# (optional) #} "') + # appropriate quoting based on property data type: + content = re.sub('"\[string\](.*)Descriptor"', r'"{{descriptor_namespace()}}/\1Descriptor#{{\1Descriptor}}"', content) + content = re.sub('/(.*)_(.*)Descriptor#', r'/\2Descriptor#', content) + content = re.sub(r'/(.*)Descriptor#', self.upper_repl, content) + content = re.sub('"\[string\](.*)"', r'"{{\1}}"', content) + content = re.sub('"\[(integer|boolean)\](.*)"', r'{{\2}}', content) + # for loops over arrays: + content = re.sub('"(.*)": \[', r'"\1": [ {% for item in \1 %}', content) + content = re.sub('\]', r'{% endfor %} ]', content) + content = re.sub('{{(.*)_(.*)}}', r'{{item.\2}}', content) + # add info header message: + content = """{# + This is an earthmover JSON template file, generated with `lightbeam create`, for creating Ed-Fi JSON `"""+endpoint+"""` + payloads using earthmover. See https://github.com/edanalytics/earthmover for documentation. +#} +""" + content # write out json template self.logger.info(f"creating file `{template_file}`...") with open(template_file, 'w+') as file: - # TODO: implement a function in `lightbeam/api.py` that constructs a "sample" payload for the endpoint - # Example: - # { - # "property_bool": true, - # "property_int": 1, - # "property_float": 1.0, - # "property_string": "string", - # "property_date": "date", - # "property_string_optional": "string", - # "property_descriptor": "uri://ed-fi.org/SomeDescriptor#SomeValue", - # "property_object": { - # "property_object_1": "string", - # "property_object_2": "string" - # }, - # "property_array": [ - # { - # "property_array_1": "string", - # "property_array_2": "string" - # } - # ] - # } - # TODO: turn the "sample" payload into a Jinja template - # Example: - # { - # "property_bool": {{property_bool}}, - # "property_int": {{property_int}}, - # "property_float": {{property_float}}, - # "property_string": "{{property_string}}", - # "property_date": "{{property_date}}", - # {% if property_string_optional %} - # "property_string_optional": "{{property_string_optional}}", - # {% endif %} - # "property_descriptor": "uri://ed-fi.org/SomeDescriptor#{{property_descriptor}}", - # "property_object": { - # "property_object_1": "{{property_object_1}}", - # "property_object_2": "{{property_object_2}}" - # }, - # "property_array": [ - # {% for item in property_array %} - # { - # "property_array_1": "{{item.property_array_1}}", - # "property_array_2": "{{item.property_array_2}}" - # } {% if not loop.last %},{% endif %} - # {% endfor %} - # ] - # } - file.write("coming soon...") # (for now) + file.write(content) From 0379182cf9c2aaf5e59a6edf082a4c9eee9f6a72 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Fri, 22 Nov 2024 16:15:43 -0600 Subject: [PATCH 4/4] bugfixes --- lightbeam/api.py | 4 ++-- lightbeam/create.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lightbeam/api.py b/lightbeam/api.py index f6e2197..e1304da 100644 --- a/lightbeam/api.py +++ b/lightbeam/api.py @@ -375,13 +375,13 @@ def get_all_params_from_swagger(self, swagger, definition, prefix=""): if "$ref" in schema["properties"][prop].keys(): params[prop_name] = {} sub_definition = schema["properties"][prop]["$ref"] - sub_params = self.get_all_params_from_swagger(swagger, sub_definition, prefix=prop+"_") + sub_params = self.get_all_params_from_swagger(swagger, sub_definition, prefix=prefix+prop+"_") for k,v in sub_params.items(): params[prop_name][k] = v elif schema["properties"][prop]["type"]!="array": params[prop_name] = f"[{schema['properties'][prop]['type']}]" + prefix + prop else: - params[prop_name] = [self.get_all_params_from_swagger(swagger, schema["properties"][prop]["items"]["$ref"], prefix=prop+"_")] + params[prop_name] = [self.get_all_params_from_swagger(swagger, schema["properties"][prop]["items"]["$ref"], prefix=prefix+prop+"-")] return params def get_identity_params_from_swagger(self, swagger, definition, prefix=""): diff --git a/lightbeam/create.py b/lightbeam/create.py index 6ca388e..cc5bdba 100644 --- a/lightbeam/create.py +++ b/lightbeam/create.py @@ -92,7 +92,7 @@ def create_jsont(self, endpoint): # for loops over arrays: content = re.sub('"(.*)": \[', r'"\1": [ {% for item in \1 %}', content) content = re.sub('\]', r'{% endfor %} ]', content) - content = re.sub('{{(.*)_(.*)}}', r'{{item.\2}}', content) + content = re.sub('{{(.*)-(.*)}}', r'{{item.\2}}', content) # add info header message: content = """{# This is an earthmover JSON template file, generated with `lightbeam create`, for creating Ed-Fi JSON `"""+endpoint+"""`