From b85e3e0c28504a0e92b87c861d8b434ca92cf5c9 Mon Sep 17 00:00:00 2001 From: Daniel Bauer Date: Tue, 5 Nov 2024 15:24:43 +0100 Subject: [PATCH 1/8] script to convert linkml to shacl during build --- build.py | 45 +++++++++++++++++++ pyproject.toml | 4 ++ .../workflow-ro-crate.ttl | 40 ----------------- 3 files changed, 49 insertions(+), 40 deletions(-) create mode 100755 build.py delete mode 100644 rocrate_validator/profiles/workflow-ro-crate-linkml/workflow-ro-crate.ttl diff --git a/build.py b/build.py new file mode 100755 index 00000000..3a960b15 --- /dev/null +++ b/build.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys + +def convert_to_shacl(input_file, output_file): + # Construct the command + command = [ + "linkml", "generate", "shacl", + "--include-annotations", "--non-closed", + input_file + ] + + # Run the command and redirect output to the output file + with open(output_file, "w") as outfile: + comment = ( + f"# This SHACL file was generated using the LinkML\n" + f"# Source file: {input_file}\n" + f"# Command: {' '.join(command)}\n\n" + ) + outfile.write(comment) + + with open(output_file, "a") as outfile: + # TODO error handling! + subprocess.run(command, stdout=outfile) + + +def convert_folder_recursively(path): + # recursively iterate through subfolders searching for yaml files + for root, _, files, in os.walk(path): + for file in files: + + # skip non-yaml files + if not file.endswith(".yaml"): + continue + + input_file = os.path.join(root, file) + output_file = input_file.replace(".yaml", ".ttl") + print("Converting", input_file) + convert_to_shacl(input_file, output_file) + + +if __name__ == "__main__": + convert_folder_recursively("rocrate_validator/profiles") diff --git a/pyproject.toml b/pyproject.toml index a385af64..3d379ce6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,10 @@ max-line-length = 120 requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" +[tool.poetry.build] +generate-setup-file = false +script = "build.py" + [tool.poetry.scripts] rocrate-validator = "rocrate_validator.cli:cli" diff --git a/rocrate_validator/profiles/workflow-ro-crate-linkml/workflow-ro-crate.ttl b/rocrate_validator/profiles/workflow-ro-crate-linkml/workflow-ro-crate.ttl deleted file mode 100644 index e686637d..00000000 --- a/rocrate_validator/profiles/workflow-ro-crate-linkml/workflow-ro-crate.ttl +++ /dev/null @@ -1,40 +0,0 @@ -@prefix rdf: . -@prefix ro-crate: . -@prefix schema1: . -@prefix sh: . -@prefix xsd: . - -ro-crate:RootDataEntity a sh:NodeShape ; - sh:closed false ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class schema1:CreativeWork ; - sh:description "Check if the Main Workflow is specified through a `mainEntity` property in the root data entity" ; - sh:maxCount 1 ; - sh:message "The Main Workflow must be specified through a `mainEntity` property in the root data entity"^^xsd:string ; - sh:minCount 1 ; - sh:nodeKind sh:BlankNodeOrIRI ; - sh:order 0 ; - sh:path schema1:mainEntity ] ; - sh:targetClass ro-crate:RootDataEntity . - - a sh:NodeShape ; - sh:closed false ; - sh:description "The Main Workflow must be specified through a `mainEntity` property in the root data entity" ; - sh:ignoredProperties ( rdf:type ) ; - sh:name "Main Workflow entity existence" ; - sh:property [ sh:class schema1:CreativeWork ; - sh:description "Check if the Main Workflow is specified through a `mainEntity` property in the root data entity" ; - sh:maxCount 1 ; - sh:message "The Main Workflow must be specified through a `mainEntity` property in the root data entity"^^xsd:string ; - sh:minCount 1 ; - sh:nodeKind sh:BlankNodeOrIRI ; - sh:order 0 ; - sh:path schema1:mainEntity ] ; - sh:targetClass . - -schema1:CreativeWork a sh:NodeShape ; - sh:closed false ; - sh:ignoredProperties ( rdf:type ) ; - sh:targetClass schema1:CreativeWork . - - From 208791955ed9d9d5413e8d57e06a7e1c1f51d570 Mon Sep 17 00:00:00 2001 From: Daniel Bauer Date: Wed, 6 Nov 2024 10:46:10 +0100 Subject: [PATCH 2/8] convert linkml files before running unit tests --- tests/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 47a867cf..e4b96b5a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,8 +15,9 @@ # calculate the absolute path of the rocrate-validator package # and add it to the system path import os +import subprocess -from pytest import fixture +from pytest import fixture, hookimpl import rocrate_validator.log as logging @@ -36,6 +37,10 @@ # profiles paths PROFILES_PATH = os.path.abspath(f"{CURRENT_PATH}/../rocrate_validator/profiles") +@hookimpl(tryfirst=True) +def pytest_configure(): + # make sure all linkml files are converted so shacl + subprocess.run(["python", "build.py"], check=True) @fixture def random_path(): From 84447811ad57bc62c4da482e7a18c66894df766d Mon Sep 17 00:00:00 2001 From: Daniel Bauer Date: Wed, 6 Nov 2024 10:55:34 +0100 Subject: [PATCH 3/8] only convert files if yaml changed --- build.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build.py b/build.py index 3a960b15..26bc7c0f 100755 --- a/build.py +++ b/build.py @@ -26,7 +26,7 @@ def convert_to_shacl(input_file, output_file): subprocess.run(command, stdout=outfile) -def convert_folder_recursively(path): +def convert_folder_recursively(path, force=False): # recursively iterate through subfolders searching for yaml files for root, _, files, in os.walk(path): for file in files: @@ -37,9 +37,10 @@ def convert_folder_recursively(path): input_file = os.path.join(root, file) output_file = input_file.replace(".yaml", ".ttl") - print("Converting", input_file) - convert_to_shacl(input_file, output_file) + if force or not os.path.exists(output_file) or os.path.getmtime(input_file) > os.path.getmtime(output_file): + print("Converting", input_file) + convert_to_shacl(input_file, output_file) if __name__ == "__main__": - convert_folder_recursively("rocrate_validator/profiles") + convert_folder_recursively("rocrate_validator/profiles", force="--force" in sys.argv) From 2442aba9403faf87ed6035b85bfc1d57d9a1b090 Mon Sep 17 00:00:00 2001 From: Daniel Bauer Date: Wed, 6 Nov 2024 11:02:57 +0100 Subject: [PATCH 4/8] add type hints --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 26bc7c0f..94db49df 100755 --- a/build.py +++ b/build.py @@ -4,7 +4,7 @@ import subprocess import sys -def convert_to_shacl(input_file, output_file): +def convert_to_shacl(input_file: str, output_file: str): # Construct the command command = [ "linkml", "generate", "shacl", @@ -26,7 +26,7 @@ def convert_to_shacl(input_file, output_file): subprocess.run(command, stdout=outfile) -def convert_folder_recursively(path, force=False): +def convert_folder_recursively(path:str, force:bool=False): # recursively iterate through subfolders searching for yaml files for root, _, files, in os.walk(path): for file in files: From 402b2ef9fb2335070d6e660d695bfa53dd3317a6 Mon Sep 17 00:00:00 2001 From: Daniel Bauer Date: Wed, 6 Nov 2024 11:11:05 +0100 Subject: [PATCH 5/8] fix build script writing shacl files if conversion fails --- build.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/build.py b/build.py index 94db49df..1693017a 100755 --- a/build.py +++ b/build.py @@ -11,19 +11,20 @@ def convert_to_shacl(input_file: str, output_file: str): "--include-annotations", "--non-closed", input_file ] - + + result = subprocess.run(command, capture_output=True, check=True) + + comment = ( + f"# This SHACL file was generated using the LinkML\n" + f"# Source file: {input_file}\n" + f"# Command: {' '.join(command)}\n\n" + ) + # Run the command and redirect output to the output file with open(output_file, "w") as outfile: - comment = ( - f"# This SHACL file was generated using the LinkML\n" - f"# Source file: {input_file}\n" - f"# Command: {' '.join(command)}\n\n" - ) - outfile.write(comment) + outfile.write(comment) + outfile.write(str(result.stdout.decode("utf-8"))) - with open(output_file, "a") as outfile: - # TODO error handling! - subprocess.run(command, stdout=outfile) def convert_folder_recursively(path:str, force:bool=False): From 0d49f4a85f1613953be8fe4a2f24d5a47bf25caf Mon Sep 17 00:00:00 2001 From: Daniel Bauer Date: Wed, 6 Nov 2024 13:54:04 +0100 Subject: [PATCH 6/8] make linter happy --- build.py | 10 +++++----- tests/conftest.py | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/build.py b/build.py index 1693017a..4e9f35a4 100755 --- a/build.py +++ b/build.py @@ -4,6 +4,7 @@ import subprocess import sys + def convert_to_shacl(input_file: str, output_file: str): # Construct the command command = [ @@ -11,7 +12,7 @@ def convert_to_shacl(input_file: str, output_file: str): "--include-annotations", "--non-closed", input_file ] - + result = subprocess.run(command, capture_output=True, check=True) comment = ( @@ -22,12 +23,11 @@ def convert_to_shacl(input_file: str, output_file: str): # Run the command and redirect output to the output file with open(output_file, "w") as outfile: - outfile.write(comment) - outfile.write(str(result.stdout.decode("utf-8"))) - + outfile.write(comment) + outfile.write(str(result.stdout.decode("utf-8"))) -def convert_folder_recursively(path:str, force:bool=False): +def convert_folder_recursively(path: str, force: bool = False): # recursively iterate through subfolders searching for yaml files for root, _, files, in os.walk(path): for file in files: diff --git a/tests/conftest.py b/tests/conftest.py index e4b96b5a..cf20ce0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,11 +37,13 @@ # profiles paths PROFILES_PATH = os.path.abspath(f"{CURRENT_PATH}/../rocrate_validator/profiles") + @hookimpl(tryfirst=True) def pytest_configure(): # make sure all linkml files are converted so shacl subprocess.run(["python", "build.py"], check=True) + @fixture def random_path(): return "/tmp/random_path" From 73e6418cff8887e1d85e5e9b55d0c4ad4fd49b21 Mon Sep 17 00:00:00 2001 From: Daniel Bauer Date: Wed, 6 Nov 2024 17:27:52 +0100 Subject: [PATCH 7/8] fix linkml not available during build --- build.py | 19 ++++++++----------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/build.py b/build.py index 4e9f35a4..3124f4fc 100755 --- a/build.py +++ b/build.py @@ -3,28 +3,25 @@ import os import subprocess import sys +from linkml.generators.shaclgen import ShaclGenerator +import linkml._version as linkml_version def convert_to_shacl(input_file: str, output_file: str): - # Construct the command - command = [ - "linkml", "generate", "shacl", - "--include-annotations", "--non-closed", - input_file - ] - - result = subprocess.run(command, capture_output=True, check=True) + # Generate shacl + shacl_shapes = ShaclGenerator(input_file).serialize() comment = ( - f"# This SHACL file was generated using the LinkML\n" + f"# This SHACL file was auto-generated with LinkML {linkml_version.__version__}.\n" + f"#Changes will be overwritten on install.\n" f"# Source file: {input_file}\n" - f"# Command: {' '.join(command)}\n\n" + "\n" ) # Run the command and redirect output to the output file with open(output_file, "w") as outfile: outfile.write(comment) - outfile.write(str(result.stdout.decode("utf-8"))) + outfile.write(shacl_shapes) def convert_folder_recursively(path: str, force: bool = False): diff --git a/pyproject.toml b/pyproject.toml index 3d379ce6..ccaa42df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ pytest-xdist = "^3.6.1" max-line-length = 120 [build-system] -requires = ["poetry-core"] +requires = ["poetry-core", "linkml"] build-backend = "poetry.core.masonry.api" [tool.poetry.build] From bbe98f08e741004f575dc36bb289e8b1e848cfbd Mon Sep 17 00:00:00 2001 From: Daniel Bauer <4690162+dnlbauer@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:28:51 +0100 Subject: [PATCH 8/8] Update tests/conftest.py Co-authored-by: Eli Chadwick --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index cf20ce0a..88cef2d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,7 +40,7 @@ @hookimpl(tryfirst=True) def pytest_configure(): - # make sure all linkml files are converted so shacl + # make sure all linkml files are converted to shacl subprocess.run(["python", "build.py"], check=True)