From 6edb27c7801c608246448adaa7d005fe95edeba7 Mon Sep 17 00:00:00 2001 From: jhnnsrs Date: Mon, 22 Jan 2024 20:52:17 +0100 Subject: [PATCH] support definitions --- bridge/backends/base.py | 44 +++- bridge/backends/docker/backend.py | 157 +++++++------ bridge/backends/messages.py | 2 +- bridge/inputs.py | 14 +- ...col_remove_flavour_definitions_and_more.py | 209 +++++++++++++++++ bridge/migrations/0009_definition_flavous.py | 21 ++ ...0010_rename_flavous_definition_flavours.py | 17 ++ ...0011_remove_setup_release_setup_flavour.py | 28 +++ bridge/models.py | 154 +++++++++++-- bridge/mutations/__init__.py | 5 +- bridge/mutations/flavour.py | 1 + bridge/mutations/repo.py | 103 +++++---- bridge/mutations/setup.py | 18 +- bridge/repo/db.py | 85 +++++++ bridge/repo/errors.py | 7 + bridge/repo/models.py | 14 ++ bridge/subscriptions/__init__.py | 3 +- bridge/subscriptions/flavour.py | 14 +- bridge/types.py | 81 ++++++- mydatabase | Bin 290816 -> 385024 bytes port_server/schema.py | 14 +- pyproject.toml | 1 + rekuest_core/__init__.py | 0 rekuest_core/constants.py | 15 ++ rekuest_core/enums.py | 63 +++++ rekuest_core/hash.py | 10 + rekuest_core/inputs.py | 0 rekuest_core/inputs/__init__.py | 0 rekuest_core/inputs/models.py | 104 +++++++++ rekuest_core/inputs/types.py | 218 ++++++++++++++++++ rekuest_core/objects/__init__.py | 0 rekuest_core/objects/models.py | 167 ++++++++++++++ rekuest_core/objects/types.py | 180 +++++++++++++++ rekuest_core/scalars.py | 49 ++++ tests/conftest.py | 18 ++ tests/deployments/deployments.yaml | 88 +++++++ tests/test_init.py | 2 + tests/test_retrieve_repo.py | 75 ++++++ tests/utils.py | 16 ++ 39 files changed, 1850 insertions(+), 147 deletions(-) create mode 100644 bridge/migrations/0008_collection_protocol_remove_flavour_definitions_and_more.py create mode 100644 bridge/migrations/0009_definition_flavous.py create mode 100644 bridge/migrations/0010_rename_flavous_definition_flavours.py create mode 100644 bridge/migrations/0011_remove_setup_release_setup_flavour.py create mode 100644 bridge/repo/db.py create mode 100644 bridge/repo/errors.py create mode 100644 rekuest_core/__init__.py create mode 100644 rekuest_core/constants.py create mode 100644 rekuest_core/enums.py create mode 100644 rekuest_core/hash.py create mode 100644 rekuest_core/inputs.py create mode 100644 rekuest_core/inputs/__init__.py create mode 100644 rekuest_core/inputs/models.py create mode 100644 rekuest_core/inputs/types.py create mode 100644 rekuest_core/objects/__init__.py create mode 100644 rekuest_core/objects/models.py create mode 100644 rekuest_core/objects/types.py create mode 100644 rekuest_core/scalars.py create mode 100644 tests/conftest.py create mode 100644 tests/deployments/deployments.yaml create mode 100644 tests/test_retrieve_repo.py create mode 100644 tests/utils.py diff --git a/bridge/backends/base.py b/bridge/backends/base.py index 4a7e370..5073cb4 100644 --- a/bridge/backends/base.py +++ b/bridge/backends/base.py @@ -1,4 +1,4 @@ -from bridge.models import Flavour, Setup, Pod +from bridge.models import Flavour, Setup, Pod, Release from bridge.backends.errors import NotImplementedByBackendError from typing import AsyncGenerator from channels.layers import get_channel_layer, InMemoryChannelLayer @@ -46,6 +46,27 @@ async def apull_flavour(self, flavour: Flavour) -> None: raise NotImplementedByBackendError("Subclasses should implement this!") + + async def aget_fitting_flavour(self, release: Release) -> Flavour: + """ A function to get the flavour that best fits a release + + This function is called by the API when a user requests a release to be run. This should + cause the flavour that best fits the release to be returned. This function + should return immediately + + Args: + release (Release): The release to get the flavour for + + Raises: + NotImplementedError: This function should be implemented by subclasses + + """ + raise NotImplementedByBackendError("Subclasses should implement this!") + + + async def ais_image_pulled(self,image: str) -> bool: + """ A function to check if a flavour is pulled""" + async def aup_setup(self, setup: Setup) -> Pod: @@ -123,6 +144,27 @@ def awatch_flavour(self, info: Info, flavour: Flavour) -> AsyncGenerator[message raise NotImplementedByBackendError("Subclasses should implement this!") pass + def awatch_flavours(self, info: Info) -> AsyncGenerator[messages.FlavourUpdate, None]: + """ A async generator that yields updates to all flavours + + This function is called by the API when a user requests to watch all flavours within a graphql + subscription. This function should then yield updates to all flavours, which will be sent to + the user. + + Most like this should delegate to the alisten function, which is implemented by the backend + base class and allows for easy listening to channels. + + Args: + info (Info): The info object from the graphql query + + Raises: + NotImplementedError: This function should be implemented by subclasses + + + + """ + raise NotImplementedByBackendError("Subclasses should implement this!") + pass async def asend_background(self, function: str, *args: typing.Any, **kwargs: typing.Any) -> None: diff --git a/bridge/backends/docker/backend.py b/bridge/backends/docker/backend.py index b074d65..5b0da9f 100644 --- a/bridge/backends/docker/backend.py +++ b/bridge/backends/docker/backend.py @@ -2,8 +2,9 @@ from bridge.backends.base import ContainerBackend from channels.layers import get_channel_layer from bridge.backends.messages import PullUpdate, UpUpdate, FlavourUpdate -from bridge.models import Flavour, Setup, Pod +from bridge.models import Flavour, Release, Setup, Pod from bridge.repo import selectors +from bridge.enums import PodStatus import docker import os from kante.types import Info @@ -84,6 +85,45 @@ async def awatch_flavour(self, info: Info, flavour: Flavour) -> AsyncGenerator[F """ Watches a flavour for updates and sends them to the client """ async for i in self.alisten(info, "whale_pull", FlavourUpdate, groups=[flavour.id]): yield i + + async def awatch_flavours(self, info: Info): + """ Watches a flavour for updates and sends them to the client """ + async for i in self.alisten(info, "whale_pull", FlavourUpdate, groups=["all"]): + yield i + + + async def aget_fitting_flavour(self, release: Release) -> Flavour: + + flavour_dict = {} + + error_dict = {} + + async for flavour in Flavour.objects.filter(release=release).all(): + try: + rating = await self.arate_flavour(flavour) + flavour_dict[flavour] = rating + except RateError as e: + logger.warning(f"Flavour {flavour} cannot be deployed") + error_dict[flavour] = e + + + + # Sort flavours by rating + sorted_flavours = sorted(flavour_dict.items(), key=lambda x: x[1]) + + # Pull the best flavour + + try: + best_flavour = sorted_flavours[0][0] + return best_flavour + except IndexError: + logger.error("No flavours available for setup") + + error_string = "\n".join([f"{k.name}: {v}" for k, v in error_dict.items()]) + + + raise Exception("No flavours available for setup: " +error_string) + @@ -111,7 +151,7 @@ async def abackground_pull_flavour(self, flavour_id: str) -> None: finished = [] - await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=0.5, flavour=flavour.id), groups=[flavour.id]) + await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=0.1, id=flavour.id), groups=[flavour.id, "all"]) for f in s: @@ -132,16 +172,26 @@ async def abackground_pull_flavour(self, flavour_id: str) -> None: "Progress: " + flavour.image + " " + str(progress) ) - await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=progress, flavour=flavour.id), groups=[flavour.id]) + await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=progress, id=flavour.id), groups=[flavour.id, "all"]) - await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=1, flavour=flavour.id), groups=[flavour.id]) + await self.abroadcast("whale_pull", FlavourUpdate(status="Pulled", progress=1, id=flavour.id), groups=[flavour.id, "all"]) async def apull_flavour(self, flavour: Flavour) -> None: """ A function to pull a flavour""" await self.asend_background("abackground_pull_flavour", flavour.id) return None + + async def ais_image_pulled(self, image: str) -> bool: + """ A function to check if a flavour is pulled""" + + + try: + image = self.api.images.get(image) + return True + except docker.errors.ImageNotFound: + return False @@ -156,18 +206,12 @@ async def arate_flavour(self, flavour: Flavour) -> int: Returns: int: The rating of the flavour (-1 if it cannot be deployed) - Raises: + Raises:str RateError: If the flavour cannot be deployed """ count = 0 - - try: - self.api.images.get(flavour.image) - except docker.errors.ImageNotFound: - raise RateError(f"Image {flavour.image} not found") - for selector in flavour.get_selectors(): if isinstance(selector, selectors.CudaSelector): if not self.gpu_available: @@ -187,15 +231,51 @@ async def arate_flavour(self, flavour: Flavour) -> int: return count - async def aup_setup_with_flavour(self, setup: Setup, flavour: Flavour) -> Pod: + async def aget_status(self, pod: Pod) -> PodStatus: + try: + container = self.api.containers.get(pod.pod_id) + return container.status + except docker.errors.NotFound: + return "Not Found" + + + async def aget_logs(self, pod: Pod) -> str: + try: + container = self.api.containers.get(pod.pod_id) + return container.logs().decode("utf-8") + except docker.errors.NotFound: + return "Not Found" + + + + async def aup_setup(self, setup: Setup) -> Pod: + + + # Iterate over flavours and pull them + + + # Create a pod + flavour = await Flavour.objects.aget( + id=setup.flavour.id + ) + container_id = f"{setup.id}-{flavour.id}" + try: + # Lets see if a pod already exists + image = self.api.images.get(flavour.image) + except docker.errors.ImageNotFound: + raise Exception("Flavour not pulled. Please pull first") + + + + device_requests = [] @@ -251,59 +331,6 @@ async def aup_setup_with_flavour(self, setup: Setup, flavour: Flavour) -> Pod: return pod - async def aget_status(self, pod: Pod) -> str: - try: - container = self.api.containers.get(pod.pod_id) - return container.status - except docker.errors.NotFound: - return "Not Found" - - - async def aget_logs(self, pod: Pod) -> str: - try: - container = self.api.containers.get(pod.pod_id) - return container.logs().decode("utf-8") - except docker.errors.NotFound: - return "Not Found" - - - - async def aup_setup(self, setup: Setup) -> Pod: - - - # Iterate over flavours and pull them - - flavour_dict = {} - - error_dict = {} - - async for flavour in Flavour.objects.filter(release=setup.release).all(): - try: - rating = await self.arate_flavour(flavour) - flavour_dict[flavour] = rating - except RateError as e: - logger.warning(f"Flavour {flavour} cannot be deployed") - error_dict[flavour] = e - - - - # Sort flavours by rating - sorted_flavours = sorted(flavour_dict.items(), key=lambda x: x[1]) - - # Pull the best flavour - - try: - best_flavour = sorted_flavours[0][0] - except IndexError: - logger.error("No flavours available for setup") - - error_string = "\n".join([f"{k.name}: {v}" for k, v in error_dict.items()]) - - - raise Exception("No flavours available for setup: " +error_string) - - return await self.aup_setup_with_flavour(setup, best_flavour) - diff --git a/bridge/backends/messages.py b/bridge/backends/messages.py index 6a15a04..7553e77 100644 --- a/bridge/backends/messages.py +++ b/bridge/backends/messages.py @@ -11,6 +11,6 @@ class UpUpdate(PullUpdate): class FlavourUpdate(BaseModel): - flavour: str + id: str status: str progress: float \ No newline at end of file diff --git a/bridge/inputs.py b/bridge/inputs.py index aaae469..4ae76b0 100644 --- a/bridge/inputs.py +++ b/bridge/inputs.py @@ -1,6 +1,6 @@ from pydantic import BaseModel from strawberry.experimental import pydantic - +import strawberry class ScanRepoInputModel(BaseModel): """Create a dask cluster input model""" @@ -20,6 +20,7 @@ class CreateGithupRepoInputModel(BaseModel): user: str branch: str repo: str + auto_scan: bool = True @pydantic.input(CreateGithupRepoInputModel, description="Create a new Github repository input") class CreateGithupRepoInput: @@ -28,6 +29,7 @@ class CreateGithupRepoInput: user: str branch: str repo: str + auto_scan: bool | None = True @@ -39,24 +41,28 @@ class PullFlavourInputModel(BaseModel): @pydantic.input(PullFlavourInputModel, description="Create a new Github repository input") class PullFlavourInput: """ Create a new Github repository input""" - id: str + id: strawberry.ID class CreateSetupInputModel(BaseModel): """ Create a new Github repository input model""" release: str + flavour: str | None = None fakts_url: str | None = "lok:80" fakts_token: str | None = None command: str | None = "arkitekt prod run" + auto_pull: bool = True @pydantic.input(CreateSetupInputModel, description="Create a new Github repository input") class CreateSetupInput: """ Create a new Github repository input""" - release: str + release: strawberry.ID + flavour: strawberry.ID | None = None fakts_url: str | None = None fakts_token: str | None = None command: str | None = "arkitekt prod run" + auto_pull: bool | None = True class DeploySetupInputModel(BaseModel): """ Create a new Github repository input model""" @@ -65,4 +71,4 @@ class DeploySetupInputModel(BaseModel): @pydantic.input(DeploySetupInputModel, description="Create a new Github repository input") class DeploySetupInput: """ Create a new Github repository input""" - setup: str \ No newline at end of file + setup: strawberry.ID \ No newline at end of file diff --git a/bridge/migrations/0008_collection_protocol_remove_flavour_definitions_and_more.py b/bridge/migrations/0008_collection_protocol_remove_flavour_definitions_and_more.py new file mode 100644 index 0000000..f10fbd5 --- /dev/null +++ b/bridge/migrations/0008_collection_protocol_remove_flavour_definitions_and_more.py @@ -0,0 +1,209 @@ +# Generated by Django 4.2.9 on 2024-01-22 10:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0007_remove_pod_creator"), + ] + + operations = [ + migrations.CreateModel( + name="Collection", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "defined_at", + models.DateTimeField(auto_created=True, auto_now_add=True), + ), + ( + "name", + models.CharField( + help_text="The name of this Collection", + max_length=1000, + unique=True, + ), + ), + ( + "description", + models.TextField(help_text="A description for the Collection"), + ), + ], + ), + migrations.CreateModel( + name="Protocol", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField( + help_text="The name of this Protocol", + max_length=1000, + unique=True, + ), + ), + ( + "description", + models.TextField(help_text="A description for the Protocol"), + ), + ], + ), + migrations.RemoveField( + model_name="flavour", + name="definitions", + ), + migrations.AddField( + model_name="flavour", + name="inspection", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="setup", + name="api_token", + field=models.CharField(default="Fake Token", max_length=400), + ), + migrations.CreateModel( + name="Definition", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "defined_at", + models.DateTimeField(auto_created=True, auto_now_add=True), + ), + ( + "definition_version", + models.CharField( + default="0.0.1", + help_text="The version of the Node definition", + max_length=1000, + ), + ), + ( + "pure", + models.BooleanField( + default=False, + help_text="Is this function pure. e.g can we cache the result?", + ), + ), + ( + "idempotent", + models.BooleanField( + default=False, + help_text="Is this function pure. e.g can we cache the result?", + ), + ), + ( + "kind", + models.CharField( + help_text="The kind of this Node. e.g. is it a function or a generator?", + max_length=1000, + ), + ), + ( + "interfaces", + models.JSONField( + default=list, + help_text="Intercae that we use to interpret the meta data", + ), + ), + ( + "port_groups", + models.JSONField( + default=list, + help_text="Intercae that we use to interpret the meta data", + ), + ), + ( + "name", + models.CharField( + help_text="The cleartext name of this Node", max_length=1000 + ), + ), + ( + "meta", + models.JSONField( + blank=True, help_text="Meta data about this Node", null=True + ), + ), + ( + "description", + models.TextField(help_text="A description for the Node"), + ), + ( + "scope", + models.CharField( + default="GLOBAL", + help_text="The scope of this Node. e.g. does the data it needs or produce live only in the scope of this Node or is it global or does it bridge data?", + max_length=1000, + ), + ), + ( + "hash", + models.CharField( + help_text="The hash of the Node (completely unique)", + max_length=1000, + unique=True, + ), + ), + ( + "args", + models.JSONField(default=list, help_text="Inputs for this Node"), + ), + ( + "returns", + models.JSONField(default=list, help_text="Outputs for this Node"), + ), + ( + "collections", + models.ManyToManyField( + help_text="The collections this Node belongs to", + related_name="nodes", + to="bridge.collection", + ), + ), + ( + "is_test_for", + models.ManyToManyField( + blank=True, + help_text="The users that have pinned the position", + related_name="tests", + to="bridge.definition", + ), + ), + ( + "protocols", + models.ManyToManyField( + blank=True, + help_text="The protocols this Node implements (e.g. Predicate)", + related_name="nodes", + to="bridge.protocol", + ), + ), + ], + ), + ] diff --git a/bridge/migrations/0009_definition_flavous.py b/bridge/migrations/0009_definition_flavous.py new file mode 100644 index 0000000..b89b953 --- /dev/null +++ b/bridge/migrations/0009_definition_flavous.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.9 on 2024-01-22 10:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0008_collection_protocol_remove_flavour_definitions_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="definition", + name="flavous", + field=models.ManyToManyField( + help_text="The flavours this Definition belongs to", + related_name="definitions", + to="bridge.flavour", + ), + ), + ] diff --git a/bridge/migrations/0010_rename_flavous_definition_flavours.py b/bridge/migrations/0010_rename_flavous_definition_flavours.py new file mode 100644 index 0000000..eb33831 --- /dev/null +++ b/bridge/migrations/0010_rename_flavous_definition_flavours.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.9 on 2024-01-22 10:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0009_definition_flavous"), + ] + + operations = [ + migrations.RenameField( + model_name="definition", + old_name="flavous", + new_name="flavours", + ), + ] diff --git a/bridge/migrations/0011_remove_setup_release_setup_flavour.py b/bridge/migrations/0011_remove_setup_release_setup_flavour.py new file mode 100644 index 0000000..eeb5195 --- /dev/null +++ b/bridge/migrations/0011_remove_setup_release_setup_flavour.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.9 on 2024-01-22 13:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0010_rename_flavous_definition_flavours"), + ] + + operations = [ + migrations.RemoveField( + model_name="setup", + name="release", + ), + migrations.AddField( + model_name="setup", + name="flavour", + field=models.ForeignKey( + default=1, + on_delete=django.db.models.deletion.CASCADE, + related_name="setups", + to="bridge.flavour", + ), + preserve_default=False, + ), + ] diff --git a/bridge/models.py b/bridge/models.py index 134b294..74951c4 100644 --- a/bridge/models.py +++ b/bridge/models.py @@ -38,7 +38,12 @@ def manifest_url(self)-> str: @property def deployments_url(self)-> str: - return f"https://raw.githubusercontent.com/{self.user}/{self.repo}/{self.branch}/.arkitekt/deployments.yaml" + return self.build_deployments_url(self.user, self.repo, self.branch) + + + @classmethod + def build_deployments_url(cls, user: str, repo: str, branch: str) -> str: + return f"https://raw.githubusercontent.com/{user}/{repo}/{branch}/.arkitekt/deployments.yaml" @@ -67,26 +72,7 @@ class Meta: ] -class Setup(models.Model): - """ This model is represents - an authenticaated Release that is bound - to a user (the installer). Setups are created - when a user installs a release - return selectors(self.selectors)and authorizes - it to access their resources. - - - - """ - release = models.ForeignKey( - Release, on_delete=models.CASCADE, related_name="setups" - ) - installer = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) - created_at = models.DateTimeField(auto_now=True) - api_token = models.CharField(max_length=400, default="Fake Token") - fakts_url = models.CharField(max_length=400, default="lok:80") - instance = models.CharField(max_length=400, default="default") - command = models.CharField(max_length=400, default="arkitekt run prod") + class Flavour(models.Model): @@ -103,7 +89,7 @@ class Flavour(models.Model): ) image = models.CharField(max_length=400, default="jhnnsrs/fake:latest") builder = models.CharField(max_length=400) - definitions = models.JSONField(default=list) + inspection = models.JSONField(default=dict) created_at = models.DateTimeField(auto_now=True) deployed_at = models.DateTimeField(null=True) @@ -121,7 +107,131 @@ def get_selectors(self) -> List[rselectors.Selector]: return field_json.selectors +class Setup(models.Model): + """ This model is represents + an authenticaated Release that is bound + to a user (the installer). Setups are created + when a user installs a release + return selectors(self.selectors)and authorizes + it to access their resources. + + + + """ + flavour = models.ForeignKey( + Flavour, on_delete=models.CASCADE, related_name="setups" + ) + installer = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + created_at = models.DateTimeField(auto_now=True) + api_token = models.CharField(max_length=400, default="Fake Token") + fakts_url = models.CharField(max_length=400, default="lok:80") + instance = models.CharField(max_length=400, default="default") + command = models.CharField(max_length=400, default="arkitekt run prod") + + + +class Collection(models.Model): + name = models.CharField( + max_length=1000, unique=True, help_text="The name of this Collection" + ) + description = models.TextField(help_text="A description for the Collection") + defined_at = models.DateTimeField(auto_created=True, auto_now_add=True) + + +class Protocol(models.Model): + name = models.CharField( + max_length=1000, unique=True, help_text="The name of this Protocol" + ) + description = models.TextField(help_text="A description for the Protocol") + + def __str__(self) -> str: + return self.name + + + +class Definition(models.Model): + """Nodes are abstraction of RPC Tasks. They provide a common API to deal with creating tasks. + + See online Documentation""" + + flavours = models.ManyToManyField( + Flavour, + related_name="definitions", + help_text="The flavours this Definition belongs to", + ) + + definition_version = models.CharField( + max_length=1000, + help_text="The version of the Node definition", + default="0.0.1", + ) + hash = models.CharField( + max_length=1000, + help_text="The hash of the Node (completely unique)", + unique=True, + ) + + collections = models.ManyToManyField( + Collection, + related_name="nodes", + help_text="The collections this Node belongs to", + ) + pure = models.BooleanField( + default=False, help_text="Is this function pure. e.g can we cache the result?" + ) + idempotent = models.BooleanField( + default=False, help_text="Is this function pure. e.g can we cache the result?" + ) + kind = models.CharField( + max_length=1000, + help_text="The kind of this Node. e.g. is it a function or a generator?", + ) + interfaces = models.JSONField( + default=list, help_text="Intercae that we use to interpret the meta data" + ) + port_groups = models.JSONField( + default=list, help_text="Intercae that we use to interpret the meta data" + ) + name = models.CharField( + max_length=1000, help_text="The cleartext name of this Node" + ) + meta = models.JSONField( + null=True, blank=True, help_text="Meta data about this Node" + ) + + description = models.TextField(help_text="A description for the Node") + scope = models.CharField( + max_length=1000, + default="GLOBAL", + help_text="The scope of this Node. e.g. does the data it needs or produce live only in the scope of this Node or is it global or does it bridge data?", + ) + is_test_for = models.ManyToManyField( + "self", + related_name="tests", + blank=True, + symmetrical=False, + help_text="The users that have pinned the position", + ) + protocols = models.ManyToManyField( + Protocol, + related_name="nodes", + blank=True, + help_text="The protocols this Node implements (e.g. Predicate)", + ) + + hash = models.CharField( + max_length=1000, + help_text="The hash of the Node (completely unique)", + unique=True, + ) + defined_at = models.DateTimeField(auto_created=True, auto_now_add=True) + + args = models.JSONField(default=list, help_text="Inputs for this Node") + returns = models.JSONField(default=list, help_text="Outputs for this Node") + + def __str__(self) -> str: + return f"{self.name}" diff --git a/bridge/mutations/__init__.py b/bridge/mutations/__init__.py index 633ac4d..e57aa64 100644 --- a/bridge/mutations/__init__.py +++ b/bridge/mutations/__init__.py @@ -1,5 +1,5 @@ """Mutations for the bridge app.""" -from .repo import scan_repo, create_github_repo +from .repo import scan_repo, create_github_repo, rescan_repos from .flavour import pull_flavour from .setup import create_setup, deploy_setup @@ -8,5 +8,6 @@ "create_github_repo", "pull_flavour", "create_setup", - "deploy_setup" + "deploy_setup", + "rescan_repos", ] diff --git a/bridge/mutations/flavour.py b/bridge/mutations/flavour.py index ae71485..bb1c97d 100644 --- a/bridge/mutations/flavour.py +++ b/bridge/mutations/flavour.py @@ -12,6 +12,7 @@ async def pull_flavour( """ Create a new dask cluster on a bridge server""" backend = get_backend() + flavour = await models.Flavour.objects.aget( id=input.id diff --git a/bridge/mutations/repo.py b/bridge/mutations/repo.py index b019fa9..580adb1 100644 --- a/bridge/mutations/repo.py +++ b/bridge/mutations/repo.py @@ -3,6 +3,7 @@ import logging import aiohttp import yaml +from bridge.repo.db import parse_config from bridge.repo.models import DeploymentsConfigFile from django.core.files import File from django.core.files.temp import NamedTemporaryFile @@ -19,19 +20,13 @@ async def adownload_logo(url: str) -> File: img_tmp.flush() return File(img_tmp) - -async def scan_repo( - info: Info, input: inputs.ScanRepoInput -) -> types.GithubRepo: - """ Create a new dask cluster on a bridge server""" - repo = await models.GithubRepo.objects.aget( - id=input.id - ) + +async def aget_deployment(deployments_url: str) -> DeploymentsConfigFile: async with aiohttp.ClientSession(headers={ "Cache-Control": "no-cache" }) as session: - async with session.get(repo.deployments_url) as response: + async with session.get(deployments_url) as response: z = await response.text() z = yaml.safe_load(z) @@ -40,44 +35,26 @@ async def scan_repo( config = DeploymentsConfigFile(**z) + return config + + + + +async def scan_repo( + info: Info, input: inputs.ScanRepoInput +) -> types.GithubRepo: + """ Create a new dask cluster on a bridge server""" + repo = await models.GithubRepo.objects.aget( + id=input.id + ) + + + config = await aget_deployment(repo.deployments_url) + + - deps = [] try: - for deployment in config.deployments: - manifest = deployment.manifest - - app, _ = await models.App.objects.aget_or_create( - identifier=manifest.identifier, - ) - - release, _ = await models.Release.objects.aupdate_or_create( - version=manifest.version, - app=app, - defaults=dict( - scopes=manifest.scopes, - ), - ) - - if manifest.logo: - logo_file = await adownload_logo(manifest.logo) - release.logo.save(f"logo{release.id}.png", logo_file) - await release.asave() - - - dep, _ = await models.Flavour.objects.aupdate_or_create( - release=release, - name=deployment.flavour , - defaults=dict( - deployment_id=deployment.deployment_id, - build_id=deployment.build_id, - flavour=deployment.flavour, - selectors=[d.dict() for d in deployment.selectors], - repo=repo, - image=deployment.image, - ), - ) - - deps.append(dep) + parsed = await parse_config(config, repo) except KeyError as e: logger.error(e, exc_info=True) pass @@ -94,10 +71,44 @@ async def create_github_repo( info: Info, input: inputs.CreateGithupRepoInput ) -> types.GithubRepo: - return await models.GithubRepo.objects.acreate( + dep_url = models.GithubRepo.build_deployments_url(input.user, input.repo, input.branch) + + config = await aget_deployment(dep_url) + + + repo = await models.GithubRepo.objects.acreate( name=input.name, user=input.user, branch=input.branch, repo=input.repo, creator=info.context.request.user, ) + + + if input.auto_scan: + try: + parsed = await parse_config(config, repo) + except KeyError as e: + logger.error(e, exc_info=True) + pass + + + return repo + + +async def rescan_repos( + info: Info +) -> list[types.GithubRepo]: + repos = models.GithubRepo.objects.all() + + async for repo in repos: + config = await aget_deployment(repo.deployments_url) + + try: + parsed = await parse_config(config, repo) + except KeyError as e: + logger.error(e, exc_info=True) + pass + + + return repos \ No newline at end of file diff --git a/bridge/mutations/setup.py b/bridge/mutations/setup.py index 5d16835..b5311c1 100644 --- a/bridge/mutations/setup.py +++ b/bridge/mutations/setup.py @@ -13,8 +13,24 @@ async def create_setup( installer = info.context.request.user + backend = get_backend() + + + if input.flavour: + flavour = await models.Flavour.objects.aget( + id=input.flavour + ) + else: + release = await models.Release.objects.aget( + id=input.release + ) + + flavour = await backend.aget_fitting_flavour(release) + + + setup = await models.Setup.objects.acreate( - release_id=input.release, + flavour=flavour, installer=installer, command=input.command, api_token=input.fakts_token, diff --git a/bridge/repo/db.py b/bridge/repo/db.py new file mode 100644 index 0000000..fd4375d --- /dev/null +++ b/bridge/repo/db.py @@ -0,0 +1,85 @@ +from rekuest_core.hash import hash_definition +from .models import DeploymentsConfigFile +from bridge import models +from .errors import DBError +import aiohttp +from django.core.files import File +from django.core.files.temp import NamedTemporaryFile + + + +async def adownload_logo(url: str) -> File: #type: ignore + """Downloads a logo from a url and returns a django file""" + img_tmp = NamedTemporaryFile(delete=True) + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + img_tmp.write(await response.read()) + img_tmp.flush() + + return File(img_tmp) + + + +async def parse_config(config: DeploymentsConfigFile, repo: models.GithubRepo) -> list[models.Flavour]: + """Parse a deployments config file and create models""" + + deps = [] + try: + for deployment in config.deployments: + manifest = deployment.manifest + + app, _ = await models.App.objects.aget_or_create( + identifier=manifest.identifier, + ) + + release, _ = await models.Release.objects.aupdate_or_create( + version=manifest.version, + app=app, + defaults=dict( + scopes=manifest.scopes, + ), + ) + + if manifest.logo: + logo_file = await adownload_logo(manifest.logo) + release.logo.save(f"logo{release.id}.png", logo_file) + await release.asave() + + + flavour, _ = await models.Flavour.objects.aupdate_or_create( + release=release, + name=deployment.flavour , + defaults=dict( + deployment_id=deployment.deployment_id, + build_id=deployment.build_id, + flavour=deployment.flavour, + selectors=[d.dict() for d in deployment.selectors], + repo=repo, + image=deployment.image, + ), + ) + + + if deployment.inspection: + inspection = deployment.inspection + for definition in inspection.definitions: + def_model, _ = await models.Definition.objects.aupdate_or_create( + hash=hash_definition(definition), + defaults=dict( + description=definition.description, + args=[d.dict() for d in definition.args], + returns=[d.dict() for d in definition.returns], + name=definition.name, + ), + ) + await def_model.flavours.aadd(flavour) + + + + deps.append(flavour) + except Exception as e: + raise DBError("Could not create models from deployments") from e + + return deps + + diff --git a/bridge/repo/errors.py b/bridge/repo/errors.py new file mode 100644 index 0000000..76dff98 --- /dev/null +++ b/bridge/repo/errors.py @@ -0,0 +1,7 @@ + +class RepoError(Exception): + pass + + +class DBError(RepoError): + pass \ No newline at end of file diff --git a/bridge/repo/models.py b/bridge/repo/models.py index 8f4b91c..e956528 100644 --- a/bridge/repo/models.py +++ b/bridge/repo/models.py @@ -5,6 +5,7 @@ import semver from bridge.repo.selectors import Selector import uuid +from rekuest_core.inputs.models import DefinitionInputModel class Manifest(BaseModel): identifier: str @@ -33,6 +34,15 @@ class Config: validate_assignment = True + + +class Inspection(BaseModel): + size: int + definitions: List[DefinitionInputModel] + + + + class Deployment(BaseModel): """A deployment is a Release of a Build. It contains the build_id, the manifest, the builder, the definitions, the image and the deployed_at timestamp. @@ -69,6 +79,10 @@ class Deployment(BaseModel): default_factory=datetime.datetime.now, description="The timestamp of the deployment", ) + inspection: Optional[Inspection] = Field( + description="The inspection of the deployment", + default=None, + ) class DeploymentsConfigFile(BaseModel): diff --git a/bridge/subscriptions/__init__.py b/bridge/subscriptions/__init__.py index 0c54021..8b66034 100644 --- a/bridge/subscriptions/__init__.py +++ b/bridge/subscriptions/__init__.py @@ -1,5 +1,6 @@ -from .flavour import flavour +from .flavour import flavour, flavours __all__ = [ "flavour", + "flavours" ] \ No newline at end of file diff --git a/bridge/subscriptions/flavour.py b/bridge/subscriptions/flavour.py index e5f61a6..128a852 100644 --- a/bridge/subscriptions/flavour.py +++ b/bridge/subscriptions/flavour.py @@ -15,4 +15,16 @@ async def flavour( flavour = await models.Flavour.objects.aget(id=flavour_id) async for message in backend.awatch_flavour(info,flavour): - yield message \ No newline at end of file + yield message + + + +async def flavours( + info: Info, +) -> AsyncGenerator[types.FlavourUpdate, None]: + """Join and subscribe to message sent to the given rooms.""" + + backend = get_backend() + + async for message in backend.awatch_flavours(info): + yield message diff --git a/bridge/types.py b/bridge/types.py index f31a0c9..01344d1 100644 --- a/bridge/types.py +++ b/bridge/types.py @@ -1,3 +1,4 @@ +import datetime from typing import Optional import strawberry @@ -11,6 +12,11 @@ from kante.types import Info from bridge.backend import get_backend from bridge.backends import messages +from rekuest_core import enums as rkenums +from rekuest_core import scalars as rkscalars +from rekuest_core.objects import models as rmodels +from rekuest_core.objects import types as rtypes +from asgiref.sync import async_to_sync @strawberry_django.type(get_user_model(), description="A user of the bridge server. Maps to an authentikate user") @@ -49,17 +55,33 @@ class App: class Release: id: auto version: str + app: App scopes: List[str] logo: Optional[str] original_logo: Optional[str] entrypoint: str - setups: List["Setup"] + flavours: List["Flavour"] + + @strawberry_django.field(description="Is this release deployed") + def installed(self, info: Info) -> bool: + return self.flavours.filter(setups__installer=info.context.request.user).first() is not None + + @strawberry_django.field(description="Is this release deployed") + def setups(self, info: Info) -> List["Setup"]: + return models.Setup.objects.filter(flavour__release=self, installer=info.context.request.user).all() + @strawberry_django.field(description="Is this release deployed") + def description(self, info: Info) -> str: + return "This is a basic app. That allows a few extra things" + @strawberry_django.field(description="Is this release deployed") + def colour(self, info: Info) -> str: + return "#254d11" + @strawberry_django.type(models.Setup, description="A user of the bridge server. Maps to an authentikate user") class Setup: id: auto - release: Release + flavour: "Flavour" installer: User api_token: str @@ -70,6 +92,7 @@ class FlavourUpdate: """ A selector is a way to select a release""" status: str progress: float + id: strawberry.ID @@ -102,10 +125,64 @@ class Flavour: entrypoint: str image: str release: Release + setups: List[Setup] @strawberry_django.field() def selectors(self, info: Info) -> List[types.Selector]: return self.get_selectors() + + @strawberry_django.field() + def pulled(self, info: Info) -> bool: + backend = get_backend() + print(self.image) + return async_to_sync(backend.ais_image_pulled)(self.image) + + @strawberry_django.field() + def latest_update(self, info: Info) -> FlavourUpdate: + return FlavourUpdate(status="Pulled", progress=1, id=self.id) + + + +@strawberry_django.type(models.Collection, description="A user of the bridge server. Maps to an authentikate user") +class Collection: + id: auto + name: str + description: str + defined_at: datetime.datetime + +@strawberry_django.type(models.Protocol, description="A user of the bridge server. Maps to an authentikate user") +class Protocol: + id: auto + name: str + description: str + + + + +@strawberry_django.type( + models.Definition, pagination=True +) +class Definition: + id: strawberry.ID + hash: rkscalars.NodeHash + name: str + kind: rkenums.NodeKind + description: str | None + collections: list["Collection"] + flavours: list["Flavour"] + scope: rkenums.NodeScope + is_test_for: list["Definition"] + tests: list["Definition"] + protocols: list["Protocol"] + defined_at: datetime.datetime + + @strawberry_django.field() + def args(self) -> list[rtypes.Port]: + return [rmodels.PortModel(**i) for i in self.args] + + @strawberry_django.field() + def returns(self) -> list[rtypes.Port]: + return [rmodels.PortModel(**i) for i in self.returns] diff --git a/mydatabase b/mydatabase index 04805cfd6316e96fa2fc79f7a491d4016d219d97..b8e282280d67516fcf50c4fa7b520a7a1c83b13e 100644 GIT binary patch delta 12558 zcmcIL33MCRar<`{1ObXzP?Q8w6eWX8R2L&yF1qliSKpZ?o zGw{+%;+Lyd@sf@6lGt(6_@!yw(rUsqhnqgPaay-d;^yVqdGd>$#!}nZjhv*;*|dKz zU_p?w;>41;+?|;}e`fy7{Q0x1`(LiQ|Lcy2YOA^t1i1izDEI^5&!?{*U~0RO=F{CK z_+%n(!Yguy_zm$U@doi4adTNr5OrPi5O_EaFZ)99vZWbbHrG_bYrXks&TPNCCSyHd z`H|(bmMKe1`LpF8Deo)aX1;FzkooAc_-0wQr{09robv^eks<(=Lz8>p8R&vOAa;j`Ws{&#WHpX=UBbjMwF6yv&Z= z=56(xWGNwAs+FUg-ENAdJjd&D-P@RAgo~m)9%g&)>bCl9(FoSfxx9heGNeW{6(sTa zgYjro2r8N+lJRsLKI^8_%w!NY<>zA||5Q9FtWs8$ieea=J+w_zHuv7GyNZN!x!qnD z*IJW%YU{pYgo|Za&bu}D>#ci>5v-S?+$S74u7)i}Fdmv^I=18%Yq(+r<)vv)@8;Y$ zY7TBR8o))eUXsfF!?yY&Fz2CI#+&=aw)SE$!+BYEuBX;p45nC`;c`+fRSYIMFT>=% zU+XLavtGB0q;ng#@2xUQ!g^dJnLD<UfiW$2olBRQ8 z>fFU(j-k9{&R^#$2D5HBWA5>~VuP?O?WW1x&+CeHVHr1EjGTK1UCe}GNQ%xa>>!K5 zwA({*xv%YDR)J}bp>ya?PcfKs(NOiCojZ%cBy5Ry}oR0`&c-pWd>zFeb9#6!duY`6ZXu5faaG)3iB)%6^w^UmpMYS!45dTd)OMIU=LNpTFY#+772(#@K+Y`1c zw&!fWuzihqf|wzOiHDcPqqgj3yry2$tB}{KtMQsT4XP)GjRwj>-l($SHPt$0JY=`y zH4g1jQHa=x*VJkNEf3oajK;6MevboSy_TnP=A+S;_f=#nE>`qb9Im*#!ckFX{iXG$ z^?TN5tWR1$W?iyIt!J$5)+TF>6}P-*`L5-uWpTR&DggF_>Y0czYgo!&2M$_5bqHpD zxQYeEdQ^CDf{%?G;U_oYpb=E_p^$;NyBY_3!A6C(ZiOyox{VrR2+b;4vr`3Ye0AcE zjX2m1Hp*Oe&Nlau(})wU^wc%68W_-Eqt z#7Bt?F+|)$>?3N3GTWQBmu>%Q`>O5pHp%vYZE9IOV#{u-#*vzOHO>l)zi}gk)~Qf^ zJvLPpC<~UXv_ojMMhU}|fR7IKQBe_kJyKJv0yM!lRTdasd+owj3hTAGbLNL~<+d(c zyX~;ey)1sXQo0pK@lDmeCJcb;9_4ycxprUc`{WmVSTFWQ&irz;<(!!@SCri>`=_!$ zFB8l7vW~L7WoFZlO;4LXY#KKmHr3!i!T%9|1fRvb@LKGRWw8&-?&~!nU_Y}u6x#Wi zQ(g;a!4L;}AZ3fYG(2Qlb&-tJCsk@~p$cqdY85C`XSJGW53KY+Beg1)WLmhW2DW!~ zLAt#y#o?6~R$rfeoseojV+c4U-VB23l-Ye9Dyb%?4mU`tgBngi%H2&xfvF3q)9gF0 za9bUzc>xQ~a^M)GY;k8x!`C3CfxKNIuhPUUAPj;QfP2~?bt9#OdI72nTYzlyWg%hR z?tJv;1%=zpj-wD)-)Kk$2FYan5f!gnE*>(-0`Xh#f%w{8dZN&0mq#8}M#@PehrhbT zd^&D|H9xvOtH)ajvm zGoB&eneJp_B9!V6Gou6D0uvfxhX)d8JX3v>uCCF^mf6F``j%>CxK@g7W?*^6GSV~w zwkx%@3MZR67Yhp;KOhi4Qogscva;ObgS&A3X`%P{Xms9n#uL0eA+&|(L#{~tG?kpc zG?-}{>5F7KFZ08}!FzqNcx-+uo=J_sQo}g0UhaDs4?L8}J3C6eiV!~`UR~K>2fu8= zHX=79^HK1FmCp`>L4f+UN`IF{x37%0fhQ_)0z8LE)_$=$N^YAM6Zh9 z1J7O+pIN!DA2gvWzZn9Se|a2t;`SB%7C`Oz^9b?$wW^EXBs>Rz3EAxfZ9*Cr87b!= ztZ9tmDf-h~Ge=P*#iH_I4ZzbKaTAVyiI8sk(Yh6!#Q&GzElH#b{SJbiLBC^t+WH>L z@60#MfwEUkzcYOW|26)V4d1f-z2yqN7yCGJ3%P)P8)=8EivOTbfm;Z?{8$BJ)#$r) z0?7PNuZkbSElKdoviK5^xGefCCYA@Kr)JUjfq>LChh8l^e4)%SuwQz44&B+Q2e_Y7 z^LKu0_KNq6nH~H0gC!=-2cp6%^r}aXbpJfMxm@Qa`MLooRSW_fMv>wB90W=CTq_<%#Y zFpt)Q))i>fG>{qxfm4btpx>=%h{RF}^@u9Hv54Nar|{g+=OT7 zp`Bm6AH|)I;}4dnq)VZ;9WBtiU zEbSah!(ZdW6OoiNoQWx?Y-d8oq@3`C;f%x*nY1&d5INIvs|*XqW2tyl=;-Nd^Yvtd zJnId?V|9RY3v_^S2~0Rl27)vlaCzN=H^{p^ZnwblY>1_Va3DaDA==|5S&AhCVU`V^ zTWGi>%r~^kHiC0C9B?+Ig7JhPL)0l`P?Qe{(ITQtkyr=<=_UwxH!ZYle4^p$0#{B>Opcg`%^D4w;hep`)`Nk|wFVqx%vG zAr^9;jwhp`Vn>I22E?SCvoZ?0cZtiZ#ueBg$%VtN03G0XH{*dfu5%u<6(s`S6IRcAsUSrEnw;THROrjY6+o>XNzO>kLTET z=-1hI%CSrBvZh~WWsO$_{=$M{!No;&&EHX*WdFFq{-7Id(z7{*hzFsYKe%wkw^llb z?Ks$I%I4j?Lx>4U*bZ`@(vo_r*xAQ5e9F175QDZ+lR9+%NP5D_Ltr{GCB!pnh?;`Q zUwv=}g!HT+#GC>joY01-D?_>YKrlpSnV?4yye?h{(!sFHLwo6fHxL%s5bP`SUJf=j z-C++C3Itpv#XwY$hd~tJ-E1K!Vs8}?3V|LBU8jZ(Yo1jK7cJYzGE)H|>5PY+k+f_J zSkt5qI0N(YflkiZuswYT&X7)K-Xif+WIj5w?@-!5;LPuf+ET1-q-W4Wi*~fkUfv@2om+^^vRF@KkKPv)aL~9HqQ{^x z7A7aOtM$?VoI0Q6@>a|LdxwP-?D5^2POeHqzs zaznZGBbM#vzbX5;>1q7GurJGh;VJXRRguOk{saB$Rq-Jd{qC~pLxFt2%q}&>%N^qW z4zQGh<*_iQoxPO9mHI9)lJcj8RN5boC;j1Me9F*C_D4egbbJ*6mO8YrJsLtM&>i3_t!P$4p*>GE?@&PihOjdjA841Z@yIP@!MsYM|c4Rt$xVVmq z3J24a@!cb|j^K9lR4D8&6~-_Iv`O(Sx;bkwx&gZm$MuvnjFgouJi^72Rhe2>mE&?w zGt5@>Mi}bC3RIXPJ-64zcUx5q8X6ehp(;g#8yc53N1^A|LeCv4S(Y=Z>lTz>txEwT z`!EF+6UZRHj>{H^Q%7O50Bl~vVGT5^;-P>knd)sR4qbGlh$l3KaLMdwZwFTfj13io zz4UfYs?-njs3U5VWsOnN(Qe?f77$YZAgGs!`%tr414>tC(Z&iWna-_1QfMVr4S+o- z8<%D$&5n)^aAj&8Sq=H7R9XY_9!1_Q1+n}Fq|&z{@)52EctNNlx$Ucier>wDyO-h- zRrIlSL@zAM8iqC?Sw(c%SkHfOp(3S3A837RLraf#0aLmDiGx8}Z3=lc?nZqRYA?<( zi{7&;fVlVU8CQC7?_XG3r;;ULR=dd>y*Ohgn-P-oE8+$xY;wS!d2%1@Ba)5zmZ9u3 znWWI*48-G6BNTQSrV{Zq?6{QTlwx&#h7ofN7V}{~DE(v+-NV4vl=38`@~yQzpwL>A z1&Jrqex){&ZY`p_hYQd8NF$_ZsQe+#ZB(3s-Db5{D*a{=-F8WNR{MM^D&OIQVi?b+ z#r;)?R6AYk>psD!CW-`6dbSv!99MMO1#fg^l-nXL)OL|VxLdu~Y4xW9c_XY|K5$ih z@Ui>ta$R)|zxdmok0tD@{ymuh_Py3GAy)7vLTt6UDi2t0!t!K~sTnt;FM>BKFKzfV zGFfrcdZXget702&{Sx@VviKmdiWle2j)g9;WRV^n!S0eKdw^Z~w*yN{-AF2r&4AIY}SjJ^=0=eZyZ-|jGO?QQ5>v-m5It+SgZROHOzfX(Fz`D@;+)nu63f&x!$2(t11g+u_~{D1m;{P5yHIqWCFLB0o!%Nc70iV z-j-de2d3T!27I##uWLS^jGIFzo}6qL_+%2d8`#QD|$;^W7v^1qEDA*-OW)#iz*9Ys@WYEN#4Ecd=WvnjLW{_4I9}W|Jk`D8sYxEoWeT z$j&mdNcg~dctip&&(`0IW~!y6!l|=;k*66@MELcK@pW-+HaR;2CyBb)1pfR z(qsMDJ}@dh+mC$^7ri1PJvxAOfDyU4bV&aHHwp35I1*jr7zYb`HEt+BSrp z216><;S^dY^~iPQ6GIpc7Nu8)urV;0M{brb4P&RkebSeQF+UhEz?EutbOdVy3(~_Q z5ZNyyo1}_i%qFp;n6LaYtUbG6?WtD3zZ=D>q$fwQ9iUGpOxWZMhY_zM@P8cqoOm5( zxgQYKWjL?^3{!rgD2`h!md{*7?HcV*5#qPRFNmK)+8+@%9D}6t1;sFn<%^`;0DcW2 eeog$0cx|Q1iM|X!a7BOxGokV?6bCqLk^VoMOsfO{ delta 5457 zcmcIodr(~0dB5jA*az&M1%v=0#Dc&GN4wnj-n;0vyb&OYmyk&0)jsaR!m|hdp=ft7Ov3Nyo{giS5)hA~T5-calzLlFD{GZ6|h9TkcGxN$k|gOe!U{r|G$O z7YUF(asMdG-8tv?JW^vpfIo7;L4a}OV+ zP>+DWL*JMPKZ3X6%2#e2Ayy91jRnQKx)8e6jlN3CZ0Nhv{HSVjJXmZR*W(eoer8Il;|HoX5j;l&kJ-f+1kN9PMR!WFgQVskcm>-PV*7 zu%egY+$_&{WQi+Px0h9KVLVPo^a`?9VBC@@^P;D1tJ+t_Z4?MHBhy@odZnzoXoCeW z3ZkU`c-tnJ;TYbd-r3f&LEEDbET}yd@_%-4ZMUqR+CguGSq~$N>MJ|AO|ZV#DqbaR zgc-pjF=|g$)kc_>Sea2D*`zJVJS__9KUUSV>mwpaqQJ2>_$F2Wi*fipcnf|3ehX@F z8rHxz_^(ie=V0+l;mHr|3TnNHZNY2~qS_oxM#H0ue<~3!vJed**NwyiGl@(Rw-pf8 zmK+wCnkumpjUbRok16qVbRv*e{F#)J#ES{ya|Z06Qj(L=R4ST?cd65j#l`SV2#LBi z4=E*`nNqbzZ$%|KbR^6tcCHPWRHhO{DNb-e3-p3wSCJKI6xo+B_@D5n@NKvsHo$Gr zWPi{875kF?75g3gQ}#FDi*N=G!Y`s#4cQ;EljX%Zclv4^WO-?>Ym8f1L6*C6Ks268 z2VyZLoK#{;Af-5p$nuJuEC<1L_OoMw*vh;6>HyJYwW+29vHA=7^ZAkdq5PJ7cYb+( zp7nj}b?Zx3)q2J{Wj$f-vF@|FtlO#IQT!0l znFT^)fp~T!I&37U07ZdtSQob&;zGmRSV&tD3F<&$&Vd27W}%!518RXJr~-w^$)eu? znkz`)0S?2@=Joq3NzeowIhUKI2eQ(f_s!D%+ejb)hd$fQqJ74+7NId_F0OmGke~s; z+)z1iPZrFLkOP|vNw5<@-Ej`_8lkWo?({uTWMcq(&Y1y8St#d9ha8H>h=pDEWNhV^ zy2r^~MfTGe{0)2y{sg`NufPl(fcxMsSPt{-x9vCVuiL+Gf6ji{ej1T$(C)Vtkyv?g z_9UQ9frY4Bn(Z3V-R3}mD+_E6Lm@=6imWWR5@^3?Z%pakhw2!j%YIljJ*6Vne8;B! z_9cp38ThNe^$}g<8P#+nR_`}8nM(8S@q4|P1gPp05q ze|Mwj%*9GRfXtfOK8>BaVU_^S`%U*}O$|0oDEARmx6Lx;ep_xIvK97!BC&NlJu>$V zShE)y)w!VhAzJ$$)NWv)T70Y8)Rb#C-Kt*nqV~?utta-Hzuby<$Vj8QVI9$R1!MON zj@L*?Ugz4tdtD%=G%4E6=+=JY^Ix39G}AZCsW^V0OO#MERW*f~5QMUn$Uo}-P)_G1e(d@3@- zrHALD-OZg7ds&+0T{Pp$)=z@qWn?!kp(2UarXkqAs9$3qFU@#a+Kq}TTB7{`Vt&kA zZ?oCV79Y9^s(pjOxvts3aB9}mJLn5XLVZejq>Gp0ho=)mOiLi#e7w75l3(z}6Y=@U zL?$(aE~dK!uPc#cR`M`{e!JlCml*sI{&Kaz0shcP6ylO*Iso1!&0)}w&d?!{=PS!A z9ctA6K8=^H9&H9M+IBmzhcWyc_H(xD1@9L~`ODTZ%SH2Zc~_|4P$T4J;v=9Th27|b z<;B!KN6+eOeZYmU-aG*M!Rki?!17O40)B|0di1~GhbXhW2aR)X!wHH20LMuJSnzAF zQh5&ZB@A7Ae+Mrit@=K`fo@(*C$a27sh77pPM0^-I)NZM+9Kv@6SG3+U8-u}sbZkM`aODAC5xkXJ#6 zW_^sz03i$Ax=8KNPF$eO+TO>=I_<_|2LE5 zwmAoV%{^_}pDt0gnz;)w+Rx9D7qyd*lds_BR_$sEe*&~<52o>_^O~CTN)I>ayr|N? zF^`v;_$J>yXqW z+jXy>s%dZ@?swL#N1R;)&O<{zJq-=o8_U#A?ee4eKWmYP@nY@zBF?S;&piGDuKg^D z|Mc2voOGITcn5phw_nS1=^fEA6z9KaDM*8T{fea zE5XZ&UJkv+aDpI z029`y6b^+0Y((JDJB$~_}7D$&ugbd7h>$)H|(eo8^I8eP2x^&`505)aKIFrJCU5HzVwCX(y= zfn;bbnpQ&TOj0+^X@wG#QyEnGg#uH7U^Es@>lQVfThP7g-uvt?yVzEdOpgZaNP1JTXlyiGAAh=c!%b zfcDyX>SA6~|JqS3)0WRuI|_UzA7(~m!j|MCn*Rx^1MJr`O1bvx6VwPetW`Wob%K4_ zBy;LX>I68Xz4atD4EE|!9Q10>T|nR-y|-I?@FH~x^l0-JDL-h+LPkz1xkNRayG^CD zU0@kJyG+( zo|ieda&YeGcx&qL?6|M58I^Y(v3Q4H9G+O1n@q{arVsh%qr=klL|>Q>cXbDvl1pBeCs1o}D$7gC9^Fqb)Ybg?_a#N)l8SY)EFKP1S#Jx4@W zr>k?YKg1>@lfzv{=3^6MN28<1hlV>@c5o~{K09)V?HKRz^z}@Swj_^7Goxd^!;=R` z7ns>l`;ktUuXm*55YKy=WZyvYn3oyqj?72n{<-*kVr;Z|Ve*K-MM))mEu%?U?pX9k zJ6l72f*9-{Y4K$fB#R0t{nw{l^(3Qre{s167-5P74I16?4DQR z9kGMa`Oen)aR2o0_zRSOz6pnFiC*yMxP|W|dM$r0 zvRT(Z!r*_vA0gMPaPJPn2R?nr@JhmG`Sh7ybK<1q>*Tj_M;?IR#o%|5&llkr+JOc0 zl>kd)MfsX-u9WuZr8N3U&Li;cfn*vFBqz{?Gm&;qB@m2(9?jPS~S83@T;dC{|le<5efhR diff --git a/port_server/schema.py b/port_server/schema.py index 6a6669b..61496bb 100644 --- a/port_server/schema.py +++ b/port_server/schema.py @@ -8,6 +8,8 @@ import strawberry_django from koherent.strawberry.extension import KoherentExtension from typing import List +from rekuest_core.constants import interface_types + @strawberry.type class Query: @@ -20,6 +22,8 @@ class Query: resolver=queries.me, description="Return the currently logged in user" ) flavours: List[types.Flavour] = strawberry_django.field() + releases: List[types.Release] = strawberry_django.field() + definitions: List[types.Definition] = strawberry_django.field() pods: List[types.Pod] = strawberry_django.field() @@ -47,6 +51,10 @@ class Mutation: resolver=mutations.deploy_setup, description="Create a new dask cluster on a bridge server", ) + rescan_repos: list[types.GithubRepo] = strawberry_django.mutation( + resolver=mutations.rescan_repos, + description="Create a new dask cluster on a bridge server", + ) @strawberry.type class Subscription: @@ -56,6 +64,10 @@ class Subscription: resolver=subscriptions.flavour, description="Create a new dask cluster on a bridge server", ) + flavours: types.FlavourUpdate = strawberry.subscription( + resolver=subscriptions.flavours, + description="Create a new dask cluster on a bridge server", + ) schema = strawberry.Schema( @@ -64,5 +76,5 @@ class Subscription: subscription=Subscription, directives=[upper, replace, relation], extensions=[DjangoOptimizerExtension, KoherentExtension], - types=[types.Selector, types.CudaSelector, types.CPUSelector] + types=[types.Selector, types.CudaSelector, types.CPUSelector] + interface_types ) diff --git a/pyproject.toml b/pyproject.toml index 82b858b..4fb09c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ django_settings_module = "port_server.settings_test" [tool.pytest.ini_options] DJANGO_SETTINGS_MODULE = "port_server.settings_test" + [tool.poetry.group.dev.dependencies] ruff = "^0.0.280" black = "^23.7.0" diff --git a/rekuest_core/__init__.py b/rekuest_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/constants.py b/rekuest_core/constants.py new file mode 100644 index 0000000..2683d07 --- /dev/null +++ b/rekuest_core/constants.py @@ -0,0 +1,15 @@ +from rekuest_core.objects import types + + + +interface_types = [ + types.SliderAssignWidget, + types.ChoiceAssignWidget, + types.SearchAssignWidget, + types.CustomReturnWidget, + types.ChoiceReturnWidget, + types.StringAssignWidget, + types.CustomAssignWidget, + types.CustomEffect, + types.MessageEffect, +] \ No newline at end of file diff --git a/rekuest_core/enums.py b/rekuest_core/enums.py new file mode 100644 index 0000000..eb66888 --- /dev/null +++ b/rekuest_core/enums.py @@ -0,0 +1,63 @@ +import strawberry +from enum import Enum + +@strawberry.enum +class NodeKind(str, Enum): + FUNCTION = "FUNCTION" + GENERATOR = "GENERATOR" + + + +@strawberry.enum +class PortKind(str, Enum): + INT = "INT" + STRING = "STRING" + STRUCTURE = "STRUCTURE" + LIST = "LIST" + BOOL = "BOOL" + DICT = "DICT" + FLOAT = "FLOAT" + DATE = "DATE" + UNION = "UNION" + + +@strawberry.enum +class AssignWidgetKind(str, Enum): + SEARCH = "SEARCH" + CHOICE = "CHOICE" + SLIDER = "SLIDER" + CUSTOM = "CUSTOM" + STRING = "STRING" + + +@strawberry.enum +class ReturnWidgetKind(str, Enum): + CHOICE = "CHOICE" + CUSTOM = "CUSTOM" + + +@strawberry.enum +class EffectKind(str, Enum): + MESSAGE = "MESSAGE" + CUSTOM = "CUSTOM" + + +@strawberry.enum +class LogicalCondition(str, Enum): + IS = "IS" + IS_NOT = "IS_NOT" + IN = "IN" + + +@strawberry.enum +class PortScope(str, Enum): + GLOBAL = "GLOBAL" + LOCAL = "LOCAL" + + +@strawberry.enum +class NodeScope(str, Enum): + GLOBAL = "GLOBAL" + LOCAL = "LOCAL" + BRIDGE_GLOBAL_TO_LOCAL = "BRIDGE_GLOBAL_TO_LOCAL" + BRIDGE_LOCAL_TO_GLOBAL = "BRIDGE_LOCAL_TO_GLOBAL" diff --git a/rekuest_core/hash.py b/rekuest_core/hash.py new file mode 100644 index 0000000..32d8274 --- /dev/null +++ b/rekuest_core/hash.py @@ -0,0 +1,10 @@ +from .inputs.models import DefinitionInputModel +import hashlib +import json + +def hash_definition(definition: DefinitionInputModel) -> str: + """Hash a definition""" + hash = hashlib.sha256( + json.dumps(definition.dict(), sort_keys=True).encode() + ).hexdigest() + return hash \ No newline at end of file diff --git a/rekuest_core/inputs.py b/rekuest_core/inputs.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/inputs/__init__.py b/rekuest_core/inputs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/inputs/models.py b/rekuest_core/inputs/models.py new file mode 100644 index 0000000..d35d132 --- /dev/null +++ b/rekuest_core/inputs/models.py @@ -0,0 +1,104 @@ +from typing import Any, Optional +from rekuest_core import enums +from pydantic import BaseModel, Field + + +class BindsInputModel(BaseModel): + templates: Optional[list[str]] + clients: Optional[list[str]] + desired_instances: int = 1 + + + + +class EffectDependencyInputModel(BaseModel): + key: str + condition: str + value: str + + + +class EffectInputModel(BaseModel): + dependencies: list[EffectDependencyInputModel] + kind: enums.EffectKind + +class ChoiceInputModel(BaseModel): + value: str + label: str + description: str | None + + + +class AssignWidgetInputModel(BaseModel): + kind: enums.AssignWidgetKind + query: str | None = None + choices: list[ChoiceInputModel] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + hook: str | None = None + ward: str | None = None + + +class ReturnWidgetInputModel(BaseModel): + kind: enums.ReturnWidgetKind + query: str | None = None + choices: list[ChoiceInputModel] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + hook: str | None = None + ward: str | None = None + + + +class ChildPortInputModel(BaseModel): + label: str | None + kind: enums.PortKind + scope: enums.PortScope + description: str | None = None + child: Optional["ChildPortInputModel"] = None + identifier: str | None = None + nullable: bool + variants: list["ChildPortInputModel"] | None = None + effects: list[EffectInputModel] | None = None + assign_widget: Optional["AssignWidgetInputModel"] = None + return_widget: Optional["ReturnWidgetInputModel"] = None + + + +class PortInputModel(BaseModel): + key: str + scope: enums.PortScope + label: str | None = None + kind: enums.PortKind + description: str | None = None + identifier: str | None = None + nullable: bool + effects: list[EffectInputModel] | None + default: Any | None = None + child: ChildPortInputModel | None = None + variants: list["ChildPortInputModel"] | None + assign_widget: Optional["AssignWidgetInputModel"] = None + return_widget: Optional["ReturnWidgetInputModel"] = None + groups: list[str] | None + + +class PortGroupInputModel(BaseModel): + key: str + hidden: bool + + +class DefinitionInputModel(BaseModel): + """A definition for a template""" + description: str = "No description provided" + collections: list[str] = Field(default_factory=list) + name: str + port_groups: list[PortGroupInputModel] = Field(default_factory=list) + args: list[PortInputModel]= Field(default_factory=list) + returns: list[PortInputModel] = Field(default_factory=list) + kind: enums.NodeKind + is_test_for: list[str]= Field(default_factory=list) + interfaces: list[str]= Field(default_factory=list) diff --git a/rekuest_core/inputs/types.py b/rekuest_core/inputs/types.py new file mode 100644 index 0000000..49331a3 --- /dev/null +++ b/rekuest_core/inputs/types.py @@ -0,0 +1,218 @@ +from typing import Optional +from pydantic import BaseModel +from strawberry.experimental import pydantic +from typing import Any +from strawberry import LazyType +from rekuest_core.inputs import models +import strawberry +from rekuest_core import enums, scalars + + + +@pydantic.input(models.EffectDependencyInputModel) +class EffectDependencyInput: + """ An effect dependency is a dependency that is used to determine + whether or not an effect should be applied to a port. For example, + you could have an effect dependency that checks whether or not + a port is null, and if it is, then the effect is applied. + + It is composed of a key, a condition, and a value. The key is the + name of the port that the effect dependency is checking. The condition + is the logical condition that the value should be checked against. + + + + """ + key: str + condition: enums.LogicalCondition + value: scalars.AnyDefault + + + + +@pydantic.input(models.EffectInputModel) +class EffectInput: + """ An effect is a way to modify a port based on a condition. For example, + you could have an effect that sets a port to null if another port is null. + + Or, you could have an effect that hides the port if another port meets a condition. + E.g when the user selects a certain option in a dropdown, another port is hidden. + + + """ + dependencies: list[EffectDependencyInput] + label: str + description: str | None + + +@pydantic.input(models.ChoiceInputModel) +class ChoiceInput: + """ A choice is a value that can be selected in a dropdown. + + It is composed of a value, a label, and a description. The value is the + value that is returned when the choice is selected. The label is the + text that is displayed in the dropdown. The description is the text + that is displayed when the user hovers over the choice. + + """ + value: scalars.AnyDefault + label: str + description: str | None + + + + +@pydantic.input(models.AssignWidgetInputModel) +class AssignWidgetInput: + """ An Assign Widget is a UI element that is used to assign a value to a port. + + It gets displayed if we intend to assign to a node, and represents the Widget + that gets displayed in the UI. For example, a dropdown, a text input, a slider, + etc. + + This input type composes elements of all the different kinds of assign widgets. + Please refere to each subtype for more information. + + + + """ + kind: enums.AssignWidgetKind + query: scalars.SearchQuery | None = None + choices: list[ChoiceInput] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + as_paragraph: bool | None = None + hook: str | None = None + ward: str | None = None + + +@pydantic.input(models.ReturnWidgetInputModel) +class ReturnWidgetInput: + """ A Return Widget is a UI element that is used to display the value of a port. + + Return Widgets get displayed both if we show the return values of an assignment, + but also when we inspect the given arguments of a previous run task. Their primary + usecase is to adequately display the value of a port, in a user readable way. + + Return Widgets are often overwriten by the underlying UI framework (e.g. Orkestrator) + to provide a better user experience. For example, a return widget that displays a + date could be overwriten to display a calendar widget. + + Return Widgets provide more a way to customize this overwriten behavior. + + """ + + + kind: enums.ReturnWidgetKind + query: scalars.SearchQuery | None = None + choices: list[ChoiceInput] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + hook: str | None = None + ward: str | None = None + + + + +@pydantic.input(models.ChildPortInputModel) +class ChildPortInput: + """ A child port is a port that is nested inside another port. For example, + + a List of Integers has a governing port that is a list, and a child port that + is of kind integer. + + """ + label: str | None + kind: enums.PortKind + scope: enums.PortScope + description: str | None = None + child: Optional[LazyType["ChildPortInput", __name__]] = None + identifier: scalars.Identifier | None = None + nullable: bool + default: scalars.AnyDefault | None = None + variants: list[LazyType["ChildPortInput", __name__]] | None = strawberry.field( + default_factory=list + ) + effects: list[EffectInput] | None = strawberry.field(default_factory=list) + assign_widget: Optional["AssignWidgetInput"] = None + return_widget: Optional["ReturnWidgetInput"] = None + + + + +@pydantic.input(models.PortInputModel) +class PortInput: + """ Port + + A Port is a single input or output of a node. It is composed of a key and a kind + which are used to uniquely identify the port. + + If the Port is a structure, we need to define a identifier and scope, + Identifiers uniquely identify a specific type of model for the scopes (e.g + all the ports that have the identifier "@mikro/image" are of the same type, and + are hence compatible with each other). Scopes are used to define in which context + the identifier is valid (e.g. a port with the identifier "@mikro/image" and the + scope "local", can only be wired to other ports that have the same identifier and + are running in the same app). Global ports are ports that have the scope "global", + and can be wired to any other port that has the same identifier, as there exists a + mechanism to resolve and retrieve the object for each app. Please check the rekuest + documentation for more information on how this works. + + + """ + key: str + scope: enums.PortScope + label: str | None = None + kind: enums.PortKind + description: str | None = None + identifier: str | None = None + nullable: bool + effects: list[EffectInput] | None = strawberry.field(default_factory=list) + default: scalars.AnyDefault | None = None + child: Optional[LazyType["ChildPortInput", __name__]] = None + variants: list[LazyType["ChildPortInput", __name__]] | None = strawberry.field( + default_factory=list + ) + assign_widget: Optional["AssignWidgetInput"] = None + return_widget: Optional["ReturnWidgetInput"] = None + groups: list[str] | None = strawberry.field(default_factory=list) + + +@strawberry.input() +class PortGroupInput: + key: str + hidden: bool + + +@strawberry.input() +class DefinitionInput: + """A definition + + Definitions are the building template for Nodes and provide the + information needed to create a node. They are primarly composed of a name, + a description, and a list of ports. + + Definitions provide a protocol of input and output, and do not contain + any information about the actual implementation of the node ( this is handled + by a template that implements a node). + + + + + """ + + description: str | None = None + collections: list[str] | None = strawberry.field(default_factory=list) + name: str + port_groups: list[PortGroupInput] | None = strawberry.field(default_factory=list) + args: list[PortInput] | None = strawberry.field(default_factory=list) + returns: list[PortInput] | None = strawberry.field(default_factory=list) + kind: enums.NodeKind + is_test_for: list[str] | None = strawberry.field(default_factory=list) + interfaces: list[str] | None = strawberry.field(default_factory=list) + + diff --git a/rekuest_core/objects/__init__.py b/rekuest_core/objects/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/objects/models.py b/rekuest_core/objects/models.py new file mode 100644 index 0000000..b6df31a --- /dev/null +++ b/rekuest_core/objects/models.py @@ -0,0 +1,167 @@ +import strawberry_django +import strawberry +from typing import Optional +from pydantic import BaseModel +from strawberry.experimental import pydantic +from typing import Literal, Union +import datetime +from rekuest_core import enums + +class ChoiceModel(BaseModel): + label: str + value: str + description: str | None + + +class AssignWidgetModel(BaseModel): + kind: str + + +class SliderAssignWidgetModel(AssignWidgetModel): + kind: Literal["SLIDER"] + min: int | None + max: int | None + + +class ChoiceAssignWidgetModel(AssignWidgetModel): + kind: Literal["CHOICE"] + choices: list[ChoiceModel] | None + + +class CustomAssignWidgetModel(AssignWidgetModel): + kind: Literal["CUSTOM"] + hook: str + ward: str + + +class SearchAssignWidgetModel(AssignWidgetModel): + kind: Literal["SEARCH"] + query: str # TODO: Validators + ward: str + + +class StringWidgetModel(AssignWidgetModel): + kind: Literal["STRING"] + placeholder: str + as_paragraph: bool + + +AssignWidgetModelUnion = Union[ + SliderAssignWidgetModel, ChoiceAssignWidgetModel, SearchAssignWidgetModel +] + + +class ReturnWidgetModel(BaseModel): + kind: str + + +class CustomReturnWidgetModel(ReturnWidgetModel): + hook: str + ward: str + + +class ChoiceReturnWidgetModel(ReturnWidgetModel): + kind: Literal["CHOICE"] + choices: list[ChoiceModel] | None + + +ReturnWidgetModelUnion = Union[CustomReturnWidgetModel, ChoiceReturnWidgetModel] + + + +class EffectDependencyModel(BaseModel): + condition: str + key: str + value: str + + + +class EffectModel(BaseModel): + dependencies: list[EffectDependencyModel] + kind: str + + +class MessageEffectModel(EffectModel): + kind: Literal["MESSAGE"] + message: str + + +class CustomEffectModel(EffectModel): + kind: Literal["CUSTOM"] + hook: str + ward: str + + + + +EffectModelUnion = Union[MessageEffectModel, CustomEffectModel] + + +class ChildPortModel(BaseModel): + label: str | None + scope: str + kind: str + child: Optional["ChildPortModel"] = None + description: str | None + identifier: str | None + nullable: bool + default: str | None + variants: list["ChildPortModel"] | None + assign_widget: AssignWidgetModelUnion | None + return_widget: ReturnWidgetModelUnion | None + + +class BindsModel(BaseModel): + templates: Optional[list[str]] = None + clients: Optional[list[str]] = None + desired_instances: int = 1 + minimum_instances: int = 1 + + + + +class PortGroupModel(BaseModel): + key: str + hidden: bool + + +@pydantic.type(PortGroupModel) +class PortGroup: + key: str + hidden: bool + + +class PortModel(BaseModel): + key: str + scope: str + label: str | None = None + kind: str + description: str | None = None + identifier: str | None = None + nullable: bool + effects: list[EffectModelUnion] | None + default: str | None + variants: list[ChildPortModel] | None + assign_widget: AssignWidgetModelUnion | None + return_widget: ReturnWidgetModelUnion | None + child: Optional[ChildPortModel] = None + groups: list[str] | None + + + + +class DefinitionModel(BaseModel): + id: strawberry.ID + hash: str + name: str + kind: enums.NodeKind + description: str | None + port_groups: list[PortGroup] + collections: list[str] + scope: enums.NodeScope + is_test_for: list[str] + tests: list[str] + protocols: list[str] + defined_at: datetime.datetime + + diff --git a/rekuest_core/objects/types.py b/rekuest_core/objects/types.py new file mode 100644 index 0000000..8bcc626 --- /dev/null +++ b/rekuest_core/objects/types.py @@ -0,0 +1,180 @@ +import datetime +from typing import Any, ForwardRef, Literal, Optional, Union + +import strawberry +import strawberry_django +from authentikate.strawberry.types import App, User +from pydantic import BaseModel +from strawberry import LazyType +from strawberry.experimental import pydantic + +from rekuest_core.objects import models +from rekuest_core import enums, scalars + +class ChoiceModel(BaseModel): + label: str + value: str + description: str | None + + +@pydantic.type(models.ChoiceModel, fields=["label", "value", "description"]) +class Choice: + pass + + + + +@pydantic.interface(models.AssignWidgetModel) +class AssignWidget: + kind: enums.AssignWidgetKind + + +@pydantic.type(models.SliderAssignWidgetModel) +class SliderAssignWidget(AssignWidget): + min: int | None + max: int | None + + +@pydantic.type(models.ChoiceAssignWidgetModel) +class ChoiceAssignWidget(AssignWidget): + choices: strawberry.auto + + +@pydantic.type(models.CustomAssignWidgetModel) +class CustomAssignWidget(AssignWidget): + hook: str + ward: str + + +@pydantic.type(models.SearchAssignWidgetModel) +class SearchAssignWidget(AssignWidget): + query: str + ward: str + + +@pydantic.type(models.StringWidgetModel) +class StringAssignWidget(AssignWidget): + placeholder: str + as_paragraph: bool + + +@pydantic.interface(models.ReturnWidgetModel) +class ReturnWidget: + kind: enums.ReturnWidgetKind + + +@pydantic.type(models.CustomReturnWidgetModel) +class CustomReturnWidget(ReturnWidget): + hook: str + ward: str + + +@pydantic.type(models.ChoiceReturnWidgetModel) +class ChoiceReturnWidget(ReturnWidget): + choices: strawberry.auto + + + +@pydantic.type(models.EffectDependencyModel) +class EffectDependency: + condition: enums.LogicalCondition + key: str + value: str + + + + +@pydantic.interface(models.EffectModel) +class Effect: + kind: str + dependencies: list[EffectDependency] + pass + + +@pydantic.type(models.MessageEffectModel) +class MessageEffect(Effect): + message: str + + +@pydantic.type(models.CustomEffectModel) +class CustomEffect(Effect): + ward: str + hook: str + + +@pydantic.type(models.BindsModel) +class Binds: + templates: list[strawberry.ID] + clients: list[strawberry.ID] + desired_instances: int + + +@pydantic.type(models.ChildPortModel) +class ChildPort: + label: strawberry.auto + identifier: scalars.Identifier | None + default: scalars.AnyDefault | None + scope: enums.PortScope + kind: enums.PortKind + nullable: bool + child: Optional[ + LazyType["ChildPort", __name__] + ] = None # this took me a while to figure out should be more obvious + variants: Optional[ + list[LazyType["ChildPort", __name__]] + ] = None # this took me a while to figure out should be more obvious + assign_widget: AssignWidget | None + return_widget: ReturnWidget | None + + + + +@pydantic.type(models.PortGroupModel) +class PortGroup: + key: str + hidden: bool + + +@pydantic.type(models.PortModel) +class Port: + identifier: scalars.Identifier | None + default: scalars.AnyDefault | None + scope: enums.PortScope + kind: enums.PortKind + key: str + nullable: bool + label: str | None + description: str | None + effects: list[Effect] | None + child: Optional[ChildPort] = None + variants: list[ChildPort] | None = None + assign_widget: AssignWidget | None + return_widget: ReturnWidget | None + groups: list[str] | None + + + +@pydantic.type( + models.DefinitionModel +) +class Definition: + hash: scalars.NodeHash + name: str + kind: enums.NodeKind + description: str | None + port_groups: list[PortGroup] + collections: list[str] + scope: enums.NodeScope + is_test_for: list[str] + tests: list[str] + protocols: list[str] + defined_at: datetime.datetime + + @strawberry_django.field() + def args(self) -> list[Port]: + return [models.PortModel(**i) for i in self.args] + + @strawberry_django.field() + def returns(self) -> list[Port]: + return [models.PortModel(**i) for i in self.returns] + diff --git a/rekuest_core/scalars.py b/rekuest_core/scalars.py new file mode 100644 index 0000000..f3823c9 --- /dev/null +++ b/rekuest_core/scalars.py @@ -0,0 +1,49 @@ +from typing import NewType +import strawberry + +Identifier = strawberry.scalar( + NewType("Identifier", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +AnyDefault = strawberry.scalar( + NewType("AnyDefault", object), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +Arg = strawberry.scalar( + NewType("Arg", object), + description="The `Arg` scalar type represents a an Argument in a Node assignment", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +SearchQuery = strawberry.scalar( + NewType("SearchQuery", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +InstanceID = strawberry.scalar( + NewType("InstanceId", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +NodeHash = strawberry.scalar( + NewType("NodeHash", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..df21bf5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,18 @@ +import pytest +import typing +from bridge.models import GithubRepo +from django.contrib.auth import get_user_model + +@pytest.fixture +def githup_repo(db: typing.Any) -> GithubRepo: + user, _ = get_user_model().objects.get_or_create( + username="test", + defaults=dict(email="test@gmail.com") + ) + + repo = GithubRepo.objects.create( + name="test", + creator=user, + ) + + return repo \ No newline at end of file diff --git a/tests/deployments/deployments.yaml b/tests/deployments/deployments.yaml new file mode 100644 index 0000000..78f8478 --- /dev/null +++ b/tests/deployments/deployments.yaml @@ -0,0 +1,88 @@ +deployments: +- build_id: 0e663dde-7027-42d4-bd81-ebcdc3013969 + deployed_at: '2024-01-17T16:12:05.578619' + deployment_id: 7d7c2d32-057f-4870-8df8-f31faea2a3fb + deployment_run: ac6b92ca-9ab7-4f60-b0f1-ee64a193c59f + flavour: vanilla + image: jhnnsrs/testing:0.0.1-vanilla + manifest: + author: jhnnsrs + created_at: '2024-01-17T16:11:59.996067' + entrypoint: app + identifier: testing + requirements: [] + scopes: + - read + version: 0.0.1 + selectors: [] +- build_id: d381ec90-90f1-4ba0-9516-dffa8d34e037 + deployed_at: '2024-01-17T16:13:43.661059' + deployment_id: 83df05c5-2f5a-480f-9f1a-94fd57872eb7 + deployment_run: ef0e07d4-83bd-41b4-bc7d-0045988ebfb7 + flavour: cuda + image: jhnnsrs/testing:0.0.1-cuda + manifest: + author: jhnnsrs + created_at: '2024-01-17T16:13:30.177467' + entrypoint: app + identifier: testing + requirements: [] + scopes: + - read + version: 0.0.1 + selectors: + - compute_capability: '3.5' + cuda_cores: 6000 + cuda_version: '10.2' + required: true + type: cuda +- build_id: ec981a3b-0e64-43a3-8c8b-41accfea5bb2 + deployed_at: '2024-01-22T10:33:56.827211' + deployment_id: ec60879f-5455-46fc-9483-9cf5a2d66131 + deployment_run: c3a9190d-de9d-4ca7-9da9-3d70d9c1f281 + flavour: cuda + image: jhnnsrs/testing:0.0.2-cuda + inspection: + definitions: + - args: + - annotations: [] + description: 'The input string + + + Returns' + key: input + kind: STRING + label: input + nullable: false + scope: GLOBAL + collections: [] + description: 'This function prints the input string to + + the console' + interfaces: [] + kind: FUNCTION + name: Print String + port_groups: [] + returns: + - annotations: [] + key: return0 + kind: STRING + nullable: false + scope: GLOBAL + size: 606452033 + manifest: + author: jhnnsrs + created_at: '2024-01-22T10:30:52.268973' + entrypoint: app + identifier: testing + requirements: [] + scopes: + - read + version: 0.0.2 + selectors: + - compute_capability: '3.5' + cuda_cores: 6000 + cuda_version: '10.2' + required: true + type: cuda +latest_deployment_run: 11f43a7b-7b6c-4612-9580-08fbd0234f9a diff --git a/tests/test_init.py b/tests/test_init.py index 72e410e..2a16c6c 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -12,6 +12,7 @@ def test_init() -> None: assert backend.channel_layer is not None, "Backend should have a channel layer" @pytest.mark.asyncio +@pytest.mark.docker async def test_pull_flavour(db: typing.Any) -> None: backend = DockerBackend() @@ -52,6 +53,7 @@ async def test_pull_flavour(db: typing.Any) -> None: @pytest.mark.asyncio +@pytest.mark.docker async def test_up_setup(db: typing.Any) -> None: backend = DockerBackend() diff --git a/tests/test_retrieve_repo.py b/tests/test_retrieve_repo.py new file mode 100644 index 0000000..8562a11 --- /dev/null +++ b/tests/test_retrieve_repo.py @@ -0,0 +1,75 @@ +import pytest +from bridge.backends.docker import DockerBackend +from bridge.models import Flavour, App, Release, GithubRepo, Setup +from django.contrib.auth import get_user_model +import typing +from bridge.repo.models import DeploymentsConfigFile +import yaml +from tests.utils import build_relative_dir +from rekuest_core.enums import PortKind +from bridge.repo.db import parse_config + +@pytest.mark.asyncio +async def test_parse_deployments(db: typing.Any) -> None: + + + with open(build_relative_dir("deployments/deployments.yaml"), "r") as deployment_file: + deployment = yaml.safe_load(deployment_file) + + config = DeploymentsConfigFile(**deployment) + + + assert len(config.deployments) == 3, "Should have one deployment" + assert config.deployments[0].flavour == "vanilla", "First deployment should be vanilla" + assert config.deployments[2].flavour == "cuda", "Third deployment should be vanilla" + + third_deployment = config.deployments[2] + + assert third_deployment.inspection, "Should have an inspection" + + inspection = third_deployment.inspection + + assert inspection.definitions, "Should have definitions" + + definitions = inspection.definitions + + assert len(definitions) == 1, "Should have one definition" + + definition = definitions[0] + + assert definition.name == "Print String", "First definition should be named Print String" + + assert definition.args, "Should have args" + + assert len(definition.args) == 1, "Should have one arg" + + arg = definition.args[0] + + assert arg.key == "input", "Arg should be named string" + assert arg.kind == PortKind.STRING, "Arg should be an string" + + + + +@pytest.mark.asyncio +async def test_db_deployments(db: typing.Any) -> None: + + user, _ = await get_user_model().objects.aget_or_create( + username="test", + defaults=dict(email="test@gmail.com") + ) + + github_repo = await GithubRepo.objects.acreate( + name="test", + creator=user, + ) + + with open(build_relative_dir("deployments/deployments.yaml"), "r") as deployment_file: + deployment = yaml.safe_load(deployment_file) + + config = DeploymentsConfigFile(**deployment) + + + flavours = await parse_config(config, github_repo) + + assert len(flavours) == 3, "Should have three flavours" \ No newline at end of file diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..0eb6c2a --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,16 @@ +import os + +def build_relative_dir(path: str) -> str: + """Build a relative directory from a path + + Parameters + ---------- + path : str + The path to build a relative directory from + + Returns + ------- + str + The relative directory + """ + return os.path.join( os.path.dirname(__file__), path) \ No newline at end of file