diff --git a/bridge/backends/base.py b/bridge/backends/base.py index 4a7e370..5073cb4 100644 --- a/bridge/backends/base.py +++ b/bridge/backends/base.py @@ -1,4 +1,4 @@ -from bridge.models import Flavour, Setup, Pod +from bridge.models import Flavour, Setup, Pod, Release from bridge.backends.errors import NotImplementedByBackendError from typing import AsyncGenerator from channels.layers import get_channel_layer, InMemoryChannelLayer @@ -46,6 +46,27 @@ async def apull_flavour(self, flavour: Flavour) -> None: raise NotImplementedByBackendError("Subclasses should implement this!") + + async def aget_fitting_flavour(self, release: Release) -> Flavour: + """ A function to get the flavour that best fits a release + + This function is called by the API when a user requests a release to be run. This should + cause the flavour that best fits the release to be returned. This function + should return immediately + + Args: + release (Release): The release to get the flavour for + + Raises: + NotImplementedError: This function should be implemented by subclasses + + """ + raise NotImplementedByBackendError("Subclasses should implement this!") + + + async def ais_image_pulled(self,image: str) -> bool: + """ A function to check if a flavour is pulled""" + async def aup_setup(self, setup: Setup) -> Pod: @@ -123,6 +144,27 @@ def awatch_flavour(self, info: Info, flavour: Flavour) -> AsyncGenerator[message raise NotImplementedByBackendError("Subclasses should implement this!") pass + def awatch_flavours(self, info: Info) -> AsyncGenerator[messages.FlavourUpdate, None]: + """ A async generator that yields updates to all flavours + + This function is called by the API when a user requests to watch all flavours within a graphql + subscription. This function should then yield updates to all flavours, which will be sent to + the user. + + Most like this should delegate to the alisten function, which is implemented by the backend + base class and allows for easy listening to channels. + + Args: + info (Info): The info object from the graphql query + + Raises: + NotImplementedError: This function should be implemented by subclasses + + + + """ + raise NotImplementedByBackendError("Subclasses should implement this!") + pass async def asend_background(self, function: str, *args: typing.Any, **kwargs: typing.Any) -> None: diff --git a/bridge/backends/docker/backend.py b/bridge/backends/docker/backend.py index b074d65..5b0da9f 100644 --- a/bridge/backends/docker/backend.py +++ b/bridge/backends/docker/backend.py @@ -2,8 +2,9 @@ from bridge.backends.base import ContainerBackend from channels.layers import get_channel_layer from bridge.backends.messages import PullUpdate, UpUpdate, FlavourUpdate -from bridge.models import Flavour, Setup, Pod +from bridge.models import Flavour, Release, Setup, Pod from bridge.repo import selectors +from bridge.enums import PodStatus import docker import os from kante.types import Info @@ -84,6 +85,45 @@ async def awatch_flavour(self, info: Info, flavour: Flavour) -> AsyncGenerator[F """ Watches a flavour for updates and sends them to the client """ async for i in self.alisten(info, "whale_pull", FlavourUpdate, groups=[flavour.id]): yield i + + async def awatch_flavours(self, info: Info): + """ Watches a flavour for updates and sends them to the client """ + async for i in self.alisten(info, "whale_pull", FlavourUpdate, groups=["all"]): + yield i + + + async def aget_fitting_flavour(self, release: Release) -> Flavour: + + flavour_dict = {} + + error_dict = {} + + async for flavour in Flavour.objects.filter(release=release).all(): + try: + rating = await self.arate_flavour(flavour) + flavour_dict[flavour] = rating + except RateError as e: + logger.warning(f"Flavour {flavour} cannot be deployed") + error_dict[flavour] = e + + + + # Sort flavours by rating + sorted_flavours = sorted(flavour_dict.items(), key=lambda x: x[1]) + + # Pull the best flavour + + try: + best_flavour = sorted_flavours[0][0] + return best_flavour + except IndexError: + logger.error("No flavours available for setup") + + error_string = "\n".join([f"{k.name}: {v}" for k, v in error_dict.items()]) + + + raise Exception("No flavours available for setup: " +error_string) + @@ -111,7 +151,7 @@ async def abackground_pull_flavour(self, flavour_id: str) -> None: finished = [] - await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=0.5, flavour=flavour.id), groups=[flavour.id]) + await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=0.1, id=flavour.id), groups=[flavour.id, "all"]) for f in s: @@ -132,16 +172,26 @@ async def abackground_pull_flavour(self, flavour_id: str) -> None: "Progress: " + flavour.image + " " + str(progress) ) - await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=progress, flavour=flavour.id), groups=[flavour.id]) + await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=progress, id=flavour.id), groups=[flavour.id, "all"]) - await self.abroadcast("whale_pull", FlavourUpdate(status="Pulling", progress=1, flavour=flavour.id), groups=[flavour.id]) + await self.abroadcast("whale_pull", FlavourUpdate(status="Pulled", progress=1, id=flavour.id), groups=[flavour.id, "all"]) async def apull_flavour(self, flavour: Flavour) -> None: """ A function to pull a flavour""" await self.asend_background("abackground_pull_flavour", flavour.id) return None + + async def ais_image_pulled(self, image: str) -> bool: + """ A function to check if a flavour is pulled""" + + + try: + image = self.api.images.get(image) + return True + except docker.errors.ImageNotFound: + return False @@ -156,18 +206,12 @@ async def arate_flavour(self, flavour: Flavour) -> int: Returns: int: The rating of the flavour (-1 if it cannot be deployed) - Raises: + Raises:str RateError: If the flavour cannot be deployed """ count = 0 - - try: - self.api.images.get(flavour.image) - except docker.errors.ImageNotFound: - raise RateError(f"Image {flavour.image} not found") - for selector in flavour.get_selectors(): if isinstance(selector, selectors.CudaSelector): if not self.gpu_available: @@ -187,15 +231,51 @@ async def arate_flavour(self, flavour: Flavour) -> int: return count - async def aup_setup_with_flavour(self, setup: Setup, flavour: Flavour) -> Pod: + async def aget_status(self, pod: Pod) -> PodStatus: + try: + container = self.api.containers.get(pod.pod_id) + return container.status + except docker.errors.NotFound: + return "Not Found" + + + async def aget_logs(self, pod: Pod) -> str: + try: + container = self.api.containers.get(pod.pod_id) + return container.logs().decode("utf-8") + except docker.errors.NotFound: + return "Not Found" + + + + async def aup_setup(self, setup: Setup) -> Pod: + + + # Iterate over flavours and pull them + + + # Create a pod + flavour = await Flavour.objects.aget( + id=setup.flavour.id + ) + container_id = f"{setup.id}-{flavour.id}" + try: + # Lets see if a pod already exists + image = self.api.images.get(flavour.image) + except docker.errors.ImageNotFound: + raise Exception("Flavour not pulled. Please pull first") + + + + device_requests = [] @@ -251,59 +331,6 @@ async def aup_setup_with_flavour(self, setup: Setup, flavour: Flavour) -> Pod: return pod - async def aget_status(self, pod: Pod) -> str: - try: - container = self.api.containers.get(pod.pod_id) - return container.status - except docker.errors.NotFound: - return "Not Found" - - - async def aget_logs(self, pod: Pod) -> str: - try: - container = self.api.containers.get(pod.pod_id) - return container.logs().decode("utf-8") - except docker.errors.NotFound: - return "Not Found" - - - - async def aup_setup(self, setup: Setup) -> Pod: - - - # Iterate over flavours and pull them - - flavour_dict = {} - - error_dict = {} - - async for flavour in Flavour.objects.filter(release=setup.release).all(): - try: - rating = await self.arate_flavour(flavour) - flavour_dict[flavour] = rating - except RateError as e: - logger.warning(f"Flavour {flavour} cannot be deployed") - error_dict[flavour] = e - - - - # Sort flavours by rating - sorted_flavours = sorted(flavour_dict.items(), key=lambda x: x[1]) - - # Pull the best flavour - - try: - best_flavour = sorted_flavours[0][0] - except IndexError: - logger.error("No flavours available for setup") - - error_string = "\n".join([f"{k.name}: {v}" for k, v in error_dict.items()]) - - - raise Exception("No flavours available for setup: " +error_string) - - return await self.aup_setup_with_flavour(setup, best_flavour) - diff --git a/bridge/backends/messages.py b/bridge/backends/messages.py index 6a15a04..7553e77 100644 --- a/bridge/backends/messages.py +++ b/bridge/backends/messages.py @@ -11,6 +11,6 @@ class UpUpdate(PullUpdate): class FlavourUpdate(BaseModel): - flavour: str + id: str status: str progress: float \ No newline at end of file diff --git a/bridge/inputs.py b/bridge/inputs.py index aaae469..4ae76b0 100644 --- a/bridge/inputs.py +++ b/bridge/inputs.py @@ -1,6 +1,6 @@ from pydantic import BaseModel from strawberry.experimental import pydantic - +import strawberry class ScanRepoInputModel(BaseModel): """Create a dask cluster input model""" @@ -20,6 +20,7 @@ class CreateGithupRepoInputModel(BaseModel): user: str branch: str repo: str + auto_scan: bool = True @pydantic.input(CreateGithupRepoInputModel, description="Create a new Github repository input") class CreateGithupRepoInput: @@ -28,6 +29,7 @@ class CreateGithupRepoInput: user: str branch: str repo: str + auto_scan: bool | None = True @@ -39,24 +41,28 @@ class PullFlavourInputModel(BaseModel): @pydantic.input(PullFlavourInputModel, description="Create a new Github repository input") class PullFlavourInput: """ Create a new Github repository input""" - id: str + id: strawberry.ID class CreateSetupInputModel(BaseModel): """ Create a new Github repository input model""" release: str + flavour: str | None = None fakts_url: str | None = "lok:80" fakts_token: str | None = None command: str | None = "arkitekt prod run" + auto_pull: bool = True @pydantic.input(CreateSetupInputModel, description="Create a new Github repository input") class CreateSetupInput: """ Create a new Github repository input""" - release: str + release: strawberry.ID + flavour: strawberry.ID | None = None fakts_url: str | None = None fakts_token: str | None = None command: str | None = "arkitekt prod run" + auto_pull: bool | None = True class DeploySetupInputModel(BaseModel): """ Create a new Github repository input model""" @@ -65,4 +71,4 @@ class DeploySetupInputModel(BaseModel): @pydantic.input(DeploySetupInputModel, description="Create a new Github repository input") class DeploySetupInput: """ Create a new Github repository input""" - setup: str \ No newline at end of file + setup: strawberry.ID \ No newline at end of file diff --git a/bridge/migrations/0008_collection_protocol_remove_flavour_definitions_and_more.py b/bridge/migrations/0008_collection_protocol_remove_flavour_definitions_and_more.py new file mode 100644 index 0000000..f10fbd5 --- /dev/null +++ b/bridge/migrations/0008_collection_protocol_remove_flavour_definitions_and_more.py @@ -0,0 +1,209 @@ +# Generated by Django 4.2.9 on 2024-01-22 10:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0007_remove_pod_creator"), + ] + + operations = [ + migrations.CreateModel( + name="Collection", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "defined_at", + models.DateTimeField(auto_created=True, auto_now_add=True), + ), + ( + "name", + models.CharField( + help_text="The name of this Collection", + max_length=1000, + unique=True, + ), + ), + ( + "description", + models.TextField(help_text="A description for the Collection"), + ), + ], + ), + migrations.CreateModel( + name="Protocol", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField( + help_text="The name of this Protocol", + max_length=1000, + unique=True, + ), + ), + ( + "description", + models.TextField(help_text="A description for the Protocol"), + ), + ], + ), + migrations.RemoveField( + model_name="flavour", + name="definitions", + ), + migrations.AddField( + model_name="flavour", + name="inspection", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="setup", + name="api_token", + field=models.CharField(default="Fake Token", max_length=400), + ), + migrations.CreateModel( + name="Definition", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "defined_at", + models.DateTimeField(auto_created=True, auto_now_add=True), + ), + ( + "definition_version", + models.CharField( + default="0.0.1", + help_text="The version of the Node definition", + max_length=1000, + ), + ), + ( + "pure", + models.BooleanField( + default=False, + help_text="Is this function pure. e.g can we cache the result?", + ), + ), + ( + "idempotent", + models.BooleanField( + default=False, + help_text="Is this function pure. e.g can we cache the result?", + ), + ), + ( + "kind", + models.CharField( + help_text="The kind of this Node. e.g. is it a function or a generator?", + max_length=1000, + ), + ), + ( + "interfaces", + models.JSONField( + default=list, + help_text="Intercae that we use to interpret the meta data", + ), + ), + ( + "port_groups", + models.JSONField( + default=list, + help_text="Intercae that we use to interpret the meta data", + ), + ), + ( + "name", + models.CharField( + help_text="The cleartext name of this Node", max_length=1000 + ), + ), + ( + "meta", + models.JSONField( + blank=True, help_text="Meta data about this Node", null=True + ), + ), + ( + "description", + models.TextField(help_text="A description for the Node"), + ), + ( + "scope", + models.CharField( + default="GLOBAL", + help_text="The scope of this Node. e.g. does the data it needs or produce live only in the scope of this Node or is it global or does it bridge data?", + max_length=1000, + ), + ), + ( + "hash", + models.CharField( + help_text="The hash of the Node (completely unique)", + max_length=1000, + unique=True, + ), + ), + ( + "args", + models.JSONField(default=list, help_text="Inputs for this Node"), + ), + ( + "returns", + models.JSONField(default=list, help_text="Outputs for this Node"), + ), + ( + "collections", + models.ManyToManyField( + help_text="The collections this Node belongs to", + related_name="nodes", + to="bridge.collection", + ), + ), + ( + "is_test_for", + models.ManyToManyField( + blank=True, + help_text="The users that have pinned the position", + related_name="tests", + to="bridge.definition", + ), + ), + ( + "protocols", + models.ManyToManyField( + blank=True, + help_text="The protocols this Node implements (e.g. Predicate)", + related_name="nodes", + to="bridge.protocol", + ), + ), + ], + ), + ] diff --git a/bridge/migrations/0009_definition_flavous.py b/bridge/migrations/0009_definition_flavous.py new file mode 100644 index 0000000..b89b953 --- /dev/null +++ b/bridge/migrations/0009_definition_flavous.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.9 on 2024-01-22 10:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0008_collection_protocol_remove_flavour_definitions_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="definition", + name="flavous", + field=models.ManyToManyField( + help_text="The flavours this Definition belongs to", + related_name="definitions", + to="bridge.flavour", + ), + ), + ] diff --git a/bridge/migrations/0010_rename_flavous_definition_flavours.py b/bridge/migrations/0010_rename_flavous_definition_flavours.py new file mode 100644 index 0000000..eb33831 --- /dev/null +++ b/bridge/migrations/0010_rename_flavous_definition_flavours.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.9 on 2024-01-22 10:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0009_definition_flavous"), + ] + + operations = [ + migrations.RenameField( + model_name="definition", + old_name="flavous", + new_name="flavours", + ), + ] diff --git a/bridge/migrations/0011_remove_setup_release_setup_flavour.py b/bridge/migrations/0011_remove_setup_release_setup_flavour.py new file mode 100644 index 0000000..eeb5195 --- /dev/null +++ b/bridge/migrations/0011_remove_setup_release_setup_flavour.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.9 on 2024-01-22 13:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("bridge", "0010_rename_flavous_definition_flavours"), + ] + + operations = [ + migrations.RemoveField( + model_name="setup", + name="release", + ), + migrations.AddField( + model_name="setup", + name="flavour", + field=models.ForeignKey( + default=1, + on_delete=django.db.models.deletion.CASCADE, + related_name="setups", + to="bridge.flavour", + ), + preserve_default=False, + ), + ] diff --git a/bridge/models.py b/bridge/models.py index 134b294..74951c4 100644 --- a/bridge/models.py +++ b/bridge/models.py @@ -38,7 +38,12 @@ def manifest_url(self)-> str: @property def deployments_url(self)-> str: - return f"https://raw.githubusercontent.com/{self.user}/{self.repo}/{self.branch}/.arkitekt/deployments.yaml" + return self.build_deployments_url(self.user, self.repo, self.branch) + + + @classmethod + def build_deployments_url(cls, user: str, repo: str, branch: str) -> str: + return f"https://raw.githubusercontent.com/{user}/{repo}/{branch}/.arkitekt/deployments.yaml" @@ -67,26 +72,7 @@ class Meta: ] -class Setup(models.Model): - """ This model is represents - an authenticaated Release that is bound - to a user (the installer). Setups are created - when a user installs a release - return selectors(self.selectors)and authorizes - it to access their resources. - - - - """ - release = models.ForeignKey( - Release, on_delete=models.CASCADE, related_name="setups" - ) - installer = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) - created_at = models.DateTimeField(auto_now=True) - api_token = models.CharField(max_length=400, default="Fake Token") - fakts_url = models.CharField(max_length=400, default="lok:80") - instance = models.CharField(max_length=400, default="default") - command = models.CharField(max_length=400, default="arkitekt run prod") + class Flavour(models.Model): @@ -103,7 +89,7 @@ class Flavour(models.Model): ) image = models.CharField(max_length=400, default="jhnnsrs/fake:latest") builder = models.CharField(max_length=400) - definitions = models.JSONField(default=list) + inspection = models.JSONField(default=dict) created_at = models.DateTimeField(auto_now=True) deployed_at = models.DateTimeField(null=True) @@ -121,7 +107,131 @@ def get_selectors(self) -> List[rselectors.Selector]: return field_json.selectors +class Setup(models.Model): + """ This model is represents + an authenticaated Release that is bound + to a user (the installer). Setups are created + when a user installs a release + return selectors(self.selectors)and authorizes + it to access their resources. + + + + """ + flavour = models.ForeignKey( + Flavour, on_delete=models.CASCADE, related_name="setups" + ) + installer = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + created_at = models.DateTimeField(auto_now=True) + api_token = models.CharField(max_length=400, default="Fake Token") + fakts_url = models.CharField(max_length=400, default="lok:80") + instance = models.CharField(max_length=400, default="default") + command = models.CharField(max_length=400, default="arkitekt run prod") + + + +class Collection(models.Model): + name = models.CharField( + max_length=1000, unique=True, help_text="The name of this Collection" + ) + description = models.TextField(help_text="A description for the Collection") + defined_at = models.DateTimeField(auto_created=True, auto_now_add=True) + + +class Protocol(models.Model): + name = models.CharField( + max_length=1000, unique=True, help_text="The name of this Protocol" + ) + description = models.TextField(help_text="A description for the Protocol") + + def __str__(self) -> str: + return self.name + + + +class Definition(models.Model): + """Nodes are abstraction of RPC Tasks. They provide a common API to deal with creating tasks. + + See online Documentation""" + + flavours = models.ManyToManyField( + Flavour, + related_name="definitions", + help_text="The flavours this Definition belongs to", + ) + + definition_version = models.CharField( + max_length=1000, + help_text="The version of the Node definition", + default="0.0.1", + ) + hash = models.CharField( + max_length=1000, + help_text="The hash of the Node (completely unique)", + unique=True, + ) + + collections = models.ManyToManyField( + Collection, + related_name="nodes", + help_text="The collections this Node belongs to", + ) + pure = models.BooleanField( + default=False, help_text="Is this function pure. e.g can we cache the result?" + ) + idempotent = models.BooleanField( + default=False, help_text="Is this function pure. e.g can we cache the result?" + ) + kind = models.CharField( + max_length=1000, + help_text="The kind of this Node. e.g. is it a function or a generator?", + ) + interfaces = models.JSONField( + default=list, help_text="Intercae that we use to interpret the meta data" + ) + port_groups = models.JSONField( + default=list, help_text="Intercae that we use to interpret the meta data" + ) + name = models.CharField( + max_length=1000, help_text="The cleartext name of this Node" + ) + meta = models.JSONField( + null=True, blank=True, help_text="Meta data about this Node" + ) + + description = models.TextField(help_text="A description for the Node") + scope = models.CharField( + max_length=1000, + default="GLOBAL", + help_text="The scope of this Node. e.g. does the data it needs or produce live only in the scope of this Node or is it global or does it bridge data?", + ) + is_test_for = models.ManyToManyField( + "self", + related_name="tests", + blank=True, + symmetrical=False, + help_text="The users that have pinned the position", + ) + protocols = models.ManyToManyField( + Protocol, + related_name="nodes", + blank=True, + help_text="The protocols this Node implements (e.g. Predicate)", + ) + + hash = models.CharField( + max_length=1000, + help_text="The hash of the Node (completely unique)", + unique=True, + ) + defined_at = models.DateTimeField(auto_created=True, auto_now_add=True) + + args = models.JSONField(default=list, help_text="Inputs for this Node") + returns = models.JSONField(default=list, help_text="Outputs for this Node") + + def __str__(self) -> str: + return f"{self.name}" diff --git a/bridge/mutations/__init__.py b/bridge/mutations/__init__.py index 633ac4d..e57aa64 100644 --- a/bridge/mutations/__init__.py +++ b/bridge/mutations/__init__.py @@ -1,5 +1,5 @@ """Mutations for the bridge app.""" -from .repo import scan_repo, create_github_repo +from .repo import scan_repo, create_github_repo, rescan_repos from .flavour import pull_flavour from .setup import create_setup, deploy_setup @@ -8,5 +8,6 @@ "create_github_repo", "pull_flavour", "create_setup", - "deploy_setup" + "deploy_setup", + "rescan_repos", ] diff --git a/bridge/mutations/flavour.py b/bridge/mutations/flavour.py index ae71485..bb1c97d 100644 --- a/bridge/mutations/flavour.py +++ b/bridge/mutations/flavour.py @@ -12,6 +12,7 @@ async def pull_flavour( """ Create a new dask cluster on a bridge server""" backend = get_backend() + flavour = await models.Flavour.objects.aget( id=input.id diff --git a/bridge/mutations/repo.py b/bridge/mutations/repo.py index b019fa9..580adb1 100644 --- a/bridge/mutations/repo.py +++ b/bridge/mutations/repo.py @@ -3,6 +3,7 @@ import logging import aiohttp import yaml +from bridge.repo.db import parse_config from bridge.repo.models import DeploymentsConfigFile from django.core.files import File from django.core.files.temp import NamedTemporaryFile @@ -19,19 +20,13 @@ async def adownload_logo(url: str) -> File: img_tmp.flush() return File(img_tmp) - -async def scan_repo( - info: Info, input: inputs.ScanRepoInput -) -> types.GithubRepo: - """ Create a new dask cluster on a bridge server""" - repo = await models.GithubRepo.objects.aget( - id=input.id - ) + +async def aget_deployment(deployments_url: str) -> DeploymentsConfigFile: async with aiohttp.ClientSession(headers={ "Cache-Control": "no-cache" }) as session: - async with session.get(repo.deployments_url) as response: + async with session.get(deployments_url) as response: z = await response.text() z = yaml.safe_load(z) @@ -40,44 +35,26 @@ async def scan_repo( config = DeploymentsConfigFile(**z) + return config + + + + +async def scan_repo( + info: Info, input: inputs.ScanRepoInput +) -> types.GithubRepo: + """ Create a new dask cluster on a bridge server""" + repo = await models.GithubRepo.objects.aget( + id=input.id + ) + + + config = await aget_deployment(repo.deployments_url) + + - deps = [] try: - for deployment in config.deployments: - manifest = deployment.manifest - - app, _ = await models.App.objects.aget_or_create( - identifier=manifest.identifier, - ) - - release, _ = await models.Release.objects.aupdate_or_create( - version=manifest.version, - app=app, - defaults=dict( - scopes=manifest.scopes, - ), - ) - - if manifest.logo: - logo_file = await adownload_logo(manifest.logo) - release.logo.save(f"logo{release.id}.png", logo_file) - await release.asave() - - - dep, _ = await models.Flavour.objects.aupdate_or_create( - release=release, - name=deployment.flavour , - defaults=dict( - deployment_id=deployment.deployment_id, - build_id=deployment.build_id, - flavour=deployment.flavour, - selectors=[d.dict() for d in deployment.selectors], - repo=repo, - image=deployment.image, - ), - ) - - deps.append(dep) + parsed = await parse_config(config, repo) except KeyError as e: logger.error(e, exc_info=True) pass @@ -94,10 +71,44 @@ async def create_github_repo( info: Info, input: inputs.CreateGithupRepoInput ) -> types.GithubRepo: - return await models.GithubRepo.objects.acreate( + dep_url = models.GithubRepo.build_deployments_url(input.user, input.repo, input.branch) + + config = await aget_deployment(dep_url) + + + repo = await models.GithubRepo.objects.acreate( name=input.name, user=input.user, branch=input.branch, repo=input.repo, creator=info.context.request.user, ) + + + if input.auto_scan: + try: + parsed = await parse_config(config, repo) + except KeyError as e: + logger.error(e, exc_info=True) + pass + + + return repo + + +async def rescan_repos( + info: Info +) -> list[types.GithubRepo]: + repos = models.GithubRepo.objects.all() + + async for repo in repos: + config = await aget_deployment(repo.deployments_url) + + try: + parsed = await parse_config(config, repo) + except KeyError as e: + logger.error(e, exc_info=True) + pass + + + return repos \ No newline at end of file diff --git a/bridge/mutations/setup.py b/bridge/mutations/setup.py index 5d16835..b5311c1 100644 --- a/bridge/mutations/setup.py +++ b/bridge/mutations/setup.py @@ -13,8 +13,24 @@ async def create_setup( installer = info.context.request.user + backend = get_backend() + + + if input.flavour: + flavour = await models.Flavour.objects.aget( + id=input.flavour + ) + else: + release = await models.Release.objects.aget( + id=input.release + ) + + flavour = await backend.aget_fitting_flavour(release) + + + setup = await models.Setup.objects.acreate( - release_id=input.release, + flavour=flavour, installer=installer, command=input.command, api_token=input.fakts_token, diff --git a/bridge/repo/db.py b/bridge/repo/db.py new file mode 100644 index 0000000..fd4375d --- /dev/null +++ b/bridge/repo/db.py @@ -0,0 +1,85 @@ +from rekuest_core.hash import hash_definition +from .models import DeploymentsConfigFile +from bridge import models +from .errors import DBError +import aiohttp +from django.core.files import File +from django.core.files.temp import NamedTemporaryFile + + + +async def adownload_logo(url: str) -> File: #type: ignore + """Downloads a logo from a url and returns a django file""" + img_tmp = NamedTemporaryFile(delete=True) + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + img_tmp.write(await response.read()) + img_tmp.flush() + + return File(img_tmp) + + + +async def parse_config(config: DeploymentsConfigFile, repo: models.GithubRepo) -> list[models.Flavour]: + """Parse a deployments config file and create models""" + + deps = [] + try: + for deployment in config.deployments: + manifest = deployment.manifest + + app, _ = await models.App.objects.aget_or_create( + identifier=manifest.identifier, + ) + + release, _ = await models.Release.objects.aupdate_or_create( + version=manifest.version, + app=app, + defaults=dict( + scopes=manifest.scopes, + ), + ) + + if manifest.logo: + logo_file = await adownload_logo(manifest.logo) + release.logo.save(f"logo{release.id}.png", logo_file) + await release.asave() + + + flavour, _ = await models.Flavour.objects.aupdate_or_create( + release=release, + name=deployment.flavour , + defaults=dict( + deployment_id=deployment.deployment_id, + build_id=deployment.build_id, + flavour=deployment.flavour, + selectors=[d.dict() for d in deployment.selectors], + repo=repo, + image=deployment.image, + ), + ) + + + if deployment.inspection: + inspection = deployment.inspection + for definition in inspection.definitions: + def_model, _ = await models.Definition.objects.aupdate_or_create( + hash=hash_definition(definition), + defaults=dict( + description=definition.description, + args=[d.dict() for d in definition.args], + returns=[d.dict() for d in definition.returns], + name=definition.name, + ), + ) + await def_model.flavours.aadd(flavour) + + + + deps.append(flavour) + except Exception as e: + raise DBError("Could not create models from deployments") from e + + return deps + + diff --git a/bridge/repo/errors.py b/bridge/repo/errors.py new file mode 100644 index 0000000..76dff98 --- /dev/null +++ b/bridge/repo/errors.py @@ -0,0 +1,7 @@ + +class RepoError(Exception): + pass + + +class DBError(RepoError): + pass \ No newline at end of file diff --git a/bridge/repo/models.py b/bridge/repo/models.py index 8f4b91c..e956528 100644 --- a/bridge/repo/models.py +++ b/bridge/repo/models.py @@ -5,6 +5,7 @@ import semver from bridge.repo.selectors import Selector import uuid +from rekuest_core.inputs.models import DefinitionInputModel class Manifest(BaseModel): identifier: str @@ -33,6 +34,15 @@ class Config: validate_assignment = True + + +class Inspection(BaseModel): + size: int + definitions: List[DefinitionInputModel] + + + + class Deployment(BaseModel): """A deployment is a Release of a Build. It contains the build_id, the manifest, the builder, the definitions, the image and the deployed_at timestamp. @@ -69,6 +79,10 @@ class Deployment(BaseModel): default_factory=datetime.datetime.now, description="The timestamp of the deployment", ) + inspection: Optional[Inspection] = Field( + description="The inspection of the deployment", + default=None, + ) class DeploymentsConfigFile(BaseModel): diff --git a/bridge/subscriptions/__init__.py b/bridge/subscriptions/__init__.py index 0c54021..8b66034 100644 --- a/bridge/subscriptions/__init__.py +++ b/bridge/subscriptions/__init__.py @@ -1,5 +1,6 @@ -from .flavour import flavour +from .flavour import flavour, flavours __all__ = [ "flavour", + "flavours" ] \ No newline at end of file diff --git a/bridge/subscriptions/flavour.py b/bridge/subscriptions/flavour.py index e5f61a6..128a852 100644 --- a/bridge/subscriptions/flavour.py +++ b/bridge/subscriptions/flavour.py @@ -15,4 +15,16 @@ async def flavour( flavour = await models.Flavour.objects.aget(id=flavour_id) async for message in backend.awatch_flavour(info,flavour): - yield message \ No newline at end of file + yield message + + + +async def flavours( + info: Info, +) -> AsyncGenerator[types.FlavourUpdate, None]: + """Join and subscribe to message sent to the given rooms.""" + + backend = get_backend() + + async for message in backend.awatch_flavours(info): + yield message diff --git a/bridge/types.py b/bridge/types.py index f31a0c9..01344d1 100644 --- a/bridge/types.py +++ b/bridge/types.py @@ -1,3 +1,4 @@ +import datetime from typing import Optional import strawberry @@ -11,6 +12,11 @@ from kante.types import Info from bridge.backend import get_backend from bridge.backends import messages +from rekuest_core import enums as rkenums +from rekuest_core import scalars as rkscalars +from rekuest_core.objects import models as rmodels +from rekuest_core.objects import types as rtypes +from asgiref.sync import async_to_sync @strawberry_django.type(get_user_model(), description="A user of the bridge server. Maps to an authentikate user") @@ -49,17 +55,33 @@ class App: class Release: id: auto version: str + app: App scopes: List[str] logo: Optional[str] original_logo: Optional[str] entrypoint: str - setups: List["Setup"] + flavours: List["Flavour"] + + @strawberry_django.field(description="Is this release deployed") + def installed(self, info: Info) -> bool: + return self.flavours.filter(setups__installer=info.context.request.user).first() is not None + + @strawberry_django.field(description="Is this release deployed") + def setups(self, info: Info) -> List["Setup"]: + return models.Setup.objects.filter(flavour__release=self, installer=info.context.request.user).all() + @strawberry_django.field(description="Is this release deployed") + def description(self, info: Info) -> str: + return "This is a basic app. That allows a few extra things" + @strawberry_django.field(description="Is this release deployed") + def colour(self, info: Info) -> str: + return "#254d11" + @strawberry_django.type(models.Setup, description="A user of the bridge server. Maps to an authentikate user") class Setup: id: auto - release: Release + flavour: "Flavour" installer: User api_token: str @@ -70,6 +92,7 @@ class FlavourUpdate: """ A selector is a way to select a release""" status: str progress: float + id: strawberry.ID @@ -102,10 +125,64 @@ class Flavour: entrypoint: str image: str release: Release + setups: List[Setup] @strawberry_django.field() def selectors(self, info: Info) -> List[types.Selector]: return self.get_selectors() + + @strawberry_django.field() + def pulled(self, info: Info) -> bool: + backend = get_backend() + print(self.image) + return async_to_sync(backend.ais_image_pulled)(self.image) + + @strawberry_django.field() + def latest_update(self, info: Info) -> FlavourUpdate: + return FlavourUpdate(status="Pulled", progress=1, id=self.id) + + + +@strawberry_django.type(models.Collection, description="A user of the bridge server. Maps to an authentikate user") +class Collection: + id: auto + name: str + description: str + defined_at: datetime.datetime + +@strawberry_django.type(models.Protocol, description="A user of the bridge server. Maps to an authentikate user") +class Protocol: + id: auto + name: str + description: str + + + + +@strawberry_django.type( + models.Definition, pagination=True +) +class Definition: + id: strawberry.ID + hash: rkscalars.NodeHash + name: str + kind: rkenums.NodeKind + description: str | None + collections: list["Collection"] + flavours: list["Flavour"] + scope: rkenums.NodeScope + is_test_for: list["Definition"] + tests: list["Definition"] + protocols: list["Protocol"] + defined_at: datetime.datetime + + @strawberry_django.field() + def args(self) -> list[rtypes.Port]: + return [rmodels.PortModel(**i) for i in self.args] + + @strawberry_django.field() + def returns(self) -> list[rtypes.Port]: + return [rmodels.PortModel(**i) for i in self.returns] diff --git a/mydatabase b/mydatabase index 04805cf..b8e2822 100644 Binary files a/mydatabase and b/mydatabase differ diff --git a/port_server/schema.py b/port_server/schema.py index 6a6669b..61496bb 100644 --- a/port_server/schema.py +++ b/port_server/schema.py @@ -8,6 +8,8 @@ import strawberry_django from koherent.strawberry.extension import KoherentExtension from typing import List +from rekuest_core.constants import interface_types + @strawberry.type class Query: @@ -20,6 +22,8 @@ class Query: resolver=queries.me, description="Return the currently logged in user" ) flavours: List[types.Flavour] = strawberry_django.field() + releases: List[types.Release] = strawberry_django.field() + definitions: List[types.Definition] = strawberry_django.field() pods: List[types.Pod] = strawberry_django.field() @@ -47,6 +51,10 @@ class Mutation: resolver=mutations.deploy_setup, description="Create a new dask cluster on a bridge server", ) + rescan_repos: list[types.GithubRepo] = strawberry_django.mutation( + resolver=mutations.rescan_repos, + description="Create a new dask cluster on a bridge server", + ) @strawberry.type class Subscription: @@ -56,6 +64,10 @@ class Subscription: resolver=subscriptions.flavour, description="Create a new dask cluster on a bridge server", ) + flavours: types.FlavourUpdate = strawberry.subscription( + resolver=subscriptions.flavours, + description="Create a new dask cluster on a bridge server", + ) schema = strawberry.Schema( @@ -64,5 +76,5 @@ class Subscription: subscription=Subscription, directives=[upper, replace, relation], extensions=[DjangoOptimizerExtension, KoherentExtension], - types=[types.Selector, types.CudaSelector, types.CPUSelector] + types=[types.Selector, types.CudaSelector, types.CPUSelector] + interface_types ) diff --git a/pyproject.toml b/pyproject.toml index 82b858b..4fb09c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ django_settings_module = "port_server.settings_test" [tool.pytest.ini_options] DJANGO_SETTINGS_MODULE = "port_server.settings_test" + [tool.poetry.group.dev.dependencies] ruff = "^0.0.280" black = "^23.7.0" diff --git a/rekuest_core/__init__.py b/rekuest_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/constants.py b/rekuest_core/constants.py new file mode 100644 index 0000000..2683d07 --- /dev/null +++ b/rekuest_core/constants.py @@ -0,0 +1,15 @@ +from rekuest_core.objects import types + + + +interface_types = [ + types.SliderAssignWidget, + types.ChoiceAssignWidget, + types.SearchAssignWidget, + types.CustomReturnWidget, + types.ChoiceReturnWidget, + types.StringAssignWidget, + types.CustomAssignWidget, + types.CustomEffect, + types.MessageEffect, +] \ No newline at end of file diff --git a/rekuest_core/enums.py b/rekuest_core/enums.py new file mode 100644 index 0000000..eb66888 --- /dev/null +++ b/rekuest_core/enums.py @@ -0,0 +1,63 @@ +import strawberry +from enum import Enum + +@strawberry.enum +class NodeKind(str, Enum): + FUNCTION = "FUNCTION" + GENERATOR = "GENERATOR" + + + +@strawberry.enum +class PortKind(str, Enum): + INT = "INT" + STRING = "STRING" + STRUCTURE = "STRUCTURE" + LIST = "LIST" + BOOL = "BOOL" + DICT = "DICT" + FLOAT = "FLOAT" + DATE = "DATE" + UNION = "UNION" + + +@strawberry.enum +class AssignWidgetKind(str, Enum): + SEARCH = "SEARCH" + CHOICE = "CHOICE" + SLIDER = "SLIDER" + CUSTOM = "CUSTOM" + STRING = "STRING" + + +@strawberry.enum +class ReturnWidgetKind(str, Enum): + CHOICE = "CHOICE" + CUSTOM = "CUSTOM" + + +@strawberry.enum +class EffectKind(str, Enum): + MESSAGE = "MESSAGE" + CUSTOM = "CUSTOM" + + +@strawberry.enum +class LogicalCondition(str, Enum): + IS = "IS" + IS_NOT = "IS_NOT" + IN = "IN" + + +@strawberry.enum +class PortScope(str, Enum): + GLOBAL = "GLOBAL" + LOCAL = "LOCAL" + + +@strawberry.enum +class NodeScope(str, Enum): + GLOBAL = "GLOBAL" + LOCAL = "LOCAL" + BRIDGE_GLOBAL_TO_LOCAL = "BRIDGE_GLOBAL_TO_LOCAL" + BRIDGE_LOCAL_TO_GLOBAL = "BRIDGE_LOCAL_TO_GLOBAL" diff --git a/rekuest_core/hash.py b/rekuest_core/hash.py new file mode 100644 index 0000000..32d8274 --- /dev/null +++ b/rekuest_core/hash.py @@ -0,0 +1,10 @@ +from .inputs.models import DefinitionInputModel +import hashlib +import json + +def hash_definition(definition: DefinitionInputModel) -> str: + """Hash a definition""" + hash = hashlib.sha256( + json.dumps(definition.dict(), sort_keys=True).encode() + ).hexdigest() + return hash \ No newline at end of file diff --git a/rekuest_core/inputs.py b/rekuest_core/inputs.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/inputs/__init__.py b/rekuest_core/inputs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/inputs/models.py b/rekuest_core/inputs/models.py new file mode 100644 index 0000000..d35d132 --- /dev/null +++ b/rekuest_core/inputs/models.py @@ -0,0 +1,104 @@ +from typing import Any, Optional +from rekuest_core import enums +from pydantic import BaseModel, Field + + +class BindsInputModel(BaseModel): + templates: Optional[list[str]] + clients: Optional[list[str]] + desired_instances: int = 1 + + + + +class EffectDependencyInputModel(BaseModel): + key: str + condition: str + value: str + + + +class EffectInputModel(BaseModel): + dependencies: list[EffectDependencyInputModel] + kind: enums.EffectKind + +class ChoiceInputModel(BaseModel): + value: str + label: str + description: str | None + + + +class AssignWidgetInputModel(BaseModel): + kind: enums.AssignWidgetKind + query: str | None = None + choices: list[ChoiceInputModel] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + hook: str | None = None + ward: str | None = None + + +class ReturnWidgetInputModel(BaseModel): + kind: enums.ReturnWidgetKind + query: str | None = None + choices: list[ChoiceInputModel] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + hook: str | None = None + ward: str | None = None + + + +class ChildPortInputModel(BaseModel): + label: str | None + kind: enums.PortKind + scope: enums.PortScope + description: str | None = None + child: Optional["ChildPortInputModel"] = None + identifier: str | None = None + nullable: bool + variants: list["ChildPortInputModel"] | None = None + effects: list[EffectInputModel] | None = None + assign_widget: Optional["AssignWidgetInputModel"] = None + return_widget: Optional["ReturnWidgetInputModel"] = None + + + +class PortInputModel(BaseModel): + key: str + scope: enums.PortScope + label: str | None = None + kind: enums.PortKind + description: str | None = None + identifier: str | None = None + nullable: bool + effects: list[EffectInputModel] | None + default: Any | None = None + child: ChildPortInputModel | None = None + variants: list["ChildPortInputModel"] | None + assign_widget: Optional["AssignWidgetInputModel"] = None + return_widget: Optional["ReturnWidgetInputModel"] = None + groups: list[str] | None + + +class PortGroupInputModel(BaseModel): + key: str + hidden: bool + + +class DefinitionInputModel(BaseModel): + """A definition for a template""" + description: str = "No description provided" + collections: list[str] = Field(default_factory=list) + name: str + port_groups: list[PortGroupInputModel] = Field(default_factory=list) + args: list[PortInputModel]= Field(default_factory=list) + returns: list[PortInputModel] = Field(default_factory=list) + kind: enums.NodeKind + is_test_for: list[str]= Field(default_factory=list) + interfaces: list[str]= Field(default_factory=list) diff --git a/rekuest_core/inputs/types.py b/rekuest_core/inputs/types.py new file mode 100644 index 0000000..49331a3 --- /dev/null +++ b/rekuest_core/inputs/types.py @@ -0,0 +1,218 @@ +from typing import Optional +from pydantic import BaseModel +from strawberry.experimental import pydantic +from typing import Any +from strawberry import LazyType +from rekuest_core.inputs import models +import strawberry +from rekuest_core import enums, scalars + + + +@pydantic.input(models.EffectDependencyInputModel) +class EffectDependencyInput: + """ An effect dependency is a dependency that is used to determine + whether or not an effect should be applied to a port. For example, + you could have an effect dependency that checks whether or not + a port is null, and if it is, then the effect is applied. + + It is composed of a key, a condition, and a value. The key is the + name of the port that the effect dependency is checking. The condition + is the logical condition that the value should be checked against. + + + + """ + key: str + condition: enums.LogicalCondition + value: scalars.AnyDefault + + + + +@pydantic.input(models.EffectInputModel) +class EffectInput: + """ An effect is a way to modify a port based on a condition. For example, + you could have an effect that sets a port to null if another port is null. + + Or, you could have an effect that hides the port if another port meets a condition. + E.g when the user selects a certain option in a dropdown, another port is hidden. + + + """ + dependencies: list[EffectDependencyInput] + label: str + description: str | None + + +@pydantic.input(models.ChoiceInputModel) +class ChoiceInput: + """ A choice is a value that can be selected in a dropdown. + + It is composed of a value, a label, and a description. The value is the + value that is returned when the choice is selected. The label is the + text that is displayed in the dropdown. The description is the text + that is displayed when the user hovers over the choice. + + """ + value: scalars.AnyDefault + label: str + description: str | None + + + + +@pydantic.input(models.AssignWidgetInputModel) +class AssignWidgetInput: + """ An Assign Widget is a UI element that is used to assign a value to a port. + + It gets displayed if we intend to assign to a node, and represents the Widget + that gets displayed in the UI. For example, a dropdown, a text input, a slider, + etc. + + This input type composes elements of all the different kinds of assign widgets. + Please refere to each subtype for more information. + + + + """ + kind: enums.AssignWidgetKind + query: scalars.SearchQuery | None = None + choices: list[ChoiceInput] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + as_paragraph: bool | None = None + hook: str | None = None + ward: str | None = None + + +@pydantic.input(models.ReturnWidgetInputModel) +class ReturnWidgetInput: + """ A Return Widget is a UI element that is used to display the value of a port. + + Return Widgets get displayed both if we show the return values of an assignment, + but also when we inspect the given arguments of a previous run task. Their primary + usecase is to adequately display the value of a port, in a user readable way. + + Return Widgets are often overwriten by the underlying UI framework (e.g. Orkestrator) + to provide a better user experience. For example, a return widget that displays a + date could be overwriten to display a calendar widget. + + Return Widgets provide more a way to customize this overwriten behavior. + + """ + + + kind: enums.ReturnWidgetKind + query: scalars.SearchQuery | None = None + choices: list[ChoiceInput] | None = None + min: int | None = None + max: int | None = None + step: int | None = None + placeholder: str | None = None + hook: str | None = None + ward: str | None = None + + + + +@pydantic.input(models.ChildPortInputModel) +class ChildPortInput: + """ A child port is a port that is nested inside another port. For example, + + a List of Integers has a governing port that is a list, and a child port that + is of kind integer. + + """ + label: str | None + kind: enums.PortKind + scope: enums.PortScope + description: str | None = None + child: Optional[LazyType["ChildPortInput", __name__]] = None + identifier: scalars.Identifier | None = None + nullable: bool + default: scalars.AnyDefault | None = None + variants: list[LazyType["ChildPortInput", __name__]] | None = strawberry.field( + default_factory=list + ) + effects: list[EffectInput] | None = strawberry.field(default_factory=list) + assign_widget: Optional["AssignWidgetInput"] = None + return_widget: Optional["ReturnWidgetInput"] = None + + + + +@pydantic.input(models.PortInputModel) +class PortInput: + """ Port + + A Port is a single input or output of a node. It is composed of a key and a kind + which are used to uniquely identify the port. + + If the Port is a structure, we need to define a identifier and scope, + Identifiers uniquely identify a specific type of model for the scopes (e.g + all the ports that have the identifier "@mikro/image" are of the same type, and + are hence compatible with each other). Scopes are used to define in which context + the identifier is valid (e.g. a port with the identifier "@mikro/image" and the + scope "local", can only be wired to other ports that have the same identifier and + are running in the same app). Global ports are ports that have the scope "global", + and can be wired to any other port that has the same identifier, as there exists a + mechanism to resolve and retrieve the object for each app. Please check the rekuest + documentation for more information on how this works. + + + """ + key: str + scope: enums.PortScope + label: str | None = None + kind: enums.PortKind + description: str | None = None + identifier: str | None = None + nullable: bool + effects: list[EffectInput] | None = strawberry.field(default_factory=list) + default: scalars.AnyDefault | None = None + child: Optional[LazyType["ChildPortInput", __name__]] = None + variants: list[LazyType["ChildPortInput", __name__]] | None = strawberry.field( + default_factory=list + ) + assign_widget: Optional["AssignWidgetInput"] = None + return_widget: Optional["ReturnWidgetInput"] = None + groups: list[str] | None = strawberry.field(default_factory=list) + + +@strawberry.input() +class PortGroupInput: + key: str + hidden: bool + + +@strawberry.input() +class DefinitionInput: + """A definition + + Definitions are the building template for Nodes and provide the + information needed to create a node. They are primarly composed of a name, + a description, and a list of ports. + + Definitions provide a protocol of input and output, and do not contain + any information about the actual implementation of the node ( this is handled + by a template that implements a node). + + + + + """ + + description: str | None = None + collections: list[str] | None = strawberry.field(default_factory=list) + name: str + port_groups: list[PortGroupInput] | None = strawberry.field(default_factory=list) + args: list[PortInput] | None = strawberry.field(default_factory=list) + returns: list[PortInput] | None = strawberry.field(default_factory=list) + kind: enums.NodeKind + is_test_for: list[str] | None = strawberry.field(default_factory=list) + interfaces: list[str] | None = strawberry.field(default_factory=list) + + diff --git a/rekuest_core/objects/__init__.py b/rekuest_core/objects/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rekuest_core/objects/models.py b/rekuest_core/objects/models.py new file mode 100644 index 0000000..b6df31a --- /dev/null +++ b/rekuest_core/objects/models.py @@ -0,0 +1,167 @@ +import strawberry_django +import strawberry +from typing import Optional +from pydantic import BaseModel +from strawberry.experimental import pydantic +from typing import Literal, Union +import datetime +from rekuest_core import enums + +class ChoiceModel(BaseModel): + label: str + value: str + description: str | None + + +class AssignWidgetModel(BaseModel): + kind: str + + +class SliderAssignWidgetModel(AssignWidgetModel): + kind: Literal["SLIDER"] + min: int | None + max: int | None + + +class ChoiceAssignWidgetModel(AssignWidgetModel): + kind: Literal["CHOICE"] + choices: list[ChoiceModel] | None + + +class CustomAssignWidgetModel(AssignWidgetModel): + kind: Literal["CUSTOM"] + hook: str + ward: str + + +class SearchAssignWidgetModel(AssignWidgetModel): + kind: Literal["SEARCH"] + query: str # TODO: Validators + ward: str + + +class StringWidgetModel(AssignWidgetModel): + kind: Literal["STRING"] + placeholder: str + as_paragraph: bool + + +AssignWidgetModelUnion = Union[ + SliderAssignWidgetModel, ChoiceAssignWidgetModel, SearchAssignWidgetModel +] + + +class ReturnWidgetModel(BaseModel): + kind: str + + +class CustomReturnWidgetModel(ReturnWidgetModel): + hook: str + ward: str + + +class ChoiceReturnWidgetModel(ReturnWidgetModel): + kind: Literal["CHOICE"] + choices: list[ChoiceModel] | None + + +ReturnWidgetModelUnion = Union[CustomReturnWidgetModel, ChoiceReturnWidgetModel] + + + +class EffectDependencyModel(BaseModel): + condition: str + key: str + value: str + + + +class EffectModel(BaseModel): + dependencies: list[EffectDependencyModel] + kind: str + + +class MessageEffectModel(EffectModel): + kind: Literal["MESSAGE"] + message: str + + +class CustomEffectModel(EffectModel): + kind: Literal["CUSTOM"] + hook: str + ward: str + + + + +EffectModelUnion = Union[MessageEffectModel, CustomEffectModel] + + +class ChildPortModel(BaseModel): + label: str | None + scope: str + kind: str + child: Optional["ChildPortModel"] = None + description: str | None + identifier: str | None + nullable: bool + default: str | None + variants: list["ChildPortModel"] | None + assign_widget: AssignWidgetModelUnion | None + return_widget: ReturnWidgetModelUnion | None + + +class BindsModel(BaseModel): + templates: Optional[list[str]] = None + clients: Optional[list[str]] = None + desired_instances: int = 1 + minimum_instances: int = 1 + + + + +class PortGroupModel(BaseModel): + key: str + hidden: bool + + +@pydantic.type(PortGroupModel) +class PortGroup: + key: str + hidden: bool + + +class PortModel(BaseModel): + key: str + scope: str + label: str | None = None + kind: str + description: str | None = None + identifier: str | None = None + nullable: bool + effects: list[EffectModelUnion] | None + default: str | None + variants: list[ChildPortModel] | None + assign_widget: AssignWidgetModelUnion | None + return_widget: ReturnWidgetModelUnion | None + child: Optional[ChildPortModel] = None + groups: list[str] | None + + + + +class DefinitionModel(BaseModel): + id: strawberry.ID + hash: str + name: str + kind: enums.NodeKind + description: str | None + port_groups: list[PortGroup] + collections: list[str] + scope: enums.NodeScope + is_test_for: list[str] + tests: list[str] + protocols: list[str] + defined_at: datetime.datetime + + diff --git a/rekuest_core/objects/types.py b/rekuest_core/objects/types.py new file mode 100644 index 0000000..8bcc626 --- /dev/null +++ b/rekuest_core/objects/types.py @@ -0,0 +1,180 @@ +import datetime +from typing import Any, ForwardRef, Literal, Optional, Union + +import strawberry +import strawberry_django +from authentikate.strawberry.types import App, User +from pydantic import BaseModel +from strawberry import LazyType +from strawberry.experimental import pydantic + +from rekuest_core.objects import models +from rekuest_core import enums, scalars + +class ChoiceModel(BaseModel): + label: str + value: str + description: str | None + + +@pydantic.type(models.ChoiceModel, fields=["label", "value", "description"]) +class Choice: + pass + + + + +@pydantic.interface(models.AssignWidgetModel) +class AssignWidget: + kind: enums.AssignWidgetKind + + +@pydantic.type(models.SliderAssignWidgetModel) +class SliderAssignWidget(AssignWidget): + min: int | None + max: int | None + + +@pydantic.type(models.ChoiceAssignWidgetModel) +class ChoiceAssignWidget(AssignWidget): + choices: strawberry.auto + + +@pydantic.type(models.CustomAssignWidgetModel) +class CustomAssignWidget(AssignWidget): + hook: str + ward: str + + +@pydantic.type(models.SearchAssignWidgetModel) +class SearchAssignWidget(AssignWidget): + query: str + ward: str + + +@pydantic.type(models.StringWidgetModel) +class StringAssignWidget(AssignWidget): + placeholder: str + as_paragraph: bool + + +@pydantic.interface(models.ReturnWidgetModel) +class ReturnWidget: + kind: enums.ReturnWidgetKind + + +@pydantic.type(models.CustomReturnWidgetModel) +class CustomReturnWidget(ReturnWidget): + hook: str + ward: str + + +@pydantic.type(models.ChoiceReturnWidgetModel) +class ChoiceReturnWidget(ReturnWidget): + choices: strawberry.auto + + + +@pydantic.type(models.EffectDependencyModel) +class EffectDependency: + condition: enums.LogicalCondition + key: str + value: str + + + + +@pydantic.interface(models.EffectModel) +class Effect: + kind: str + dependencies: list[EffectDependency] + pass + + +@pydantic.type(models.MessageEffectModel) +class MessageEffect(Effect): + message: str + + +@pydantic.type(models.CustomEffectModel) +class CustomEffect(Effect): + ward: str + hook: str + + +@pydantic.type(models.BindsModel) +class Binds: + templates: list[strawberry.ID] + clients: list[strawberry.ID] + desired_instances: int + + +@pydantic.type(models.ChildPortModel) +class ChildPort: + label: strawberry.auto + identifier: scalars.Identifier | None + default: scalars.AnyDefault | None + scope: enums.PortScope + kind: enums.PortKind + nullable: bool + child: Optional[ + LazyType["ChildPort", __name__] + ] = None # this took me a while to figure out should be more obvious + variants: Optional[ + list[LazyType["ChildPort", __name__]] + ] = None # this took me a while to figure out should be more obvious + assign_widget: AssignWidget | None + return_widget: ReturnWidget | None + + + + +@pydantic.type(models.PortGroupModel) +class PortGroup: + key: str + hidden: bool + + +@pydantic.type(models.PortModel) +class Port: + identifier: scalars.Identifier | None + default: scalars.AnyDefault | None + scope: enums.PortScope + kind: enums.PortKind + key: str + nullable: bool + label: str | None + description: str | None + effects: list[Effect] | None + child: Optional[ChildPort] = None + variants: list[ChildPort] | None = None + assign_widget: AssignWidget | None + return_widget: ReturnWidget | None + groups: list[str] | None + + + +@pydantic.type( + models.DefinitionModel +) +class Definition: + hash: scalars.NodeHash + name: str + kind: enums.NodeKind + description: str | None + port_groups: list[PortGroup] + collections: list[str] + scope: enums.NodeScope + is_test_for: list[str] + tests: list[str] + protocols: list[str] + defined_at: datetime.datetime + + @strawberry_django.field() + def args(self) -> list[Port]: + return [models.PortModel(**i) for i in self.args] + + @strawberry_django.field() + def returns(self) -> list[Port]: + return [models.PortModel(**i) for i in self.returns] + diff --git a/rekuest_core/scalars.py b/rekuest_core/scalars.py new file mode 100644 index 0000000..f3823c9 --- /dev/null +++ b/rekuest_core/scalars.py @@ -0,0 +1,49 @@ +from typing import NewType +import strawberry + +Identifier = strawberry.scalar( + NewType("Identifier", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +AnyDefault = strawberry.scalar( + NewType("AnyDefault", object), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +Arg = strawberry.scalar( + NewType("Arg", object), + description="The `Arg` scalar type represents a an Argument in a Node assignment", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +SearchQuery = strawberry.scalar( + NewType("SearchQuery", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +InstanceID = strawberry.scalar( + NewType("InstanceId", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) + +NodeHash = strawberry.scalar( + NewType("NodeHash", str), + description="The `ArrayLike` scalar type represents a reference to a store " + "previously created by the user n a datalayer", + serialize=lambda v: v, + parse_value=lambda v: v, +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..df21bf5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,18 @@ +import pytest +import typing +from bridge.models import GithubRepo +from django.contrib.auth import get_user_model + +@pytest.fixture +def githup_repo(db: typing.Any) -> GithubRepo: + user, _ = get_user_model().objects.get_or_create( + username="test", + defaults=dict(email="test@gmail.com") + ) + + repo = GithubRepo.objects.create( + name="test", + creator=user, + ) + + return repo \ No newline at end of file diff --git a/tests/deployments/deployments.yaml b/tests/deployments/deployments.yaml new file mode 100644 index 0000000..78f8478 --- /dev/null +++ b/tests/deployments/deployments.yaml @@ -0,0 +1,88 @@ +deployments: +- build_id: 0e663dde-7027-42d4-bd81-ebcdc3013969 + deployed_at: '2024-01-17T16:12:05.578619' + deployment_id: 7d7c2d32-057f-4870-8df8-f31faea2a3fb + deployment_run: ac6b92ca-9ab7-4f60-b0f1-ee64a193c59f + flavour: vanilla + image: jhnnsrs/testing:0.0.1-vanilla + manifest: + author: jhnnsrs + created_at: '2024-01-17T16:11:59.996067' + entrypoint: app + identifier: testing + requirements: [] + scopes: + - read + version: 0.0.1 + selectors: [] +- build_id: d381ec90-90f1-4ba0-9516-dffa8d34e037 + deployed_at: '2024-01-17T16:13:43.661059' + deployment_id: 83df05c5-2f5a-480f-9f1a-94fd57872eb7 + deployment_run: ef0e07d4-83bd-41b4-bc7d-0045988ebfb7 + flavour: cuda + image: jhnnsrs/testing:0.0.1-cuda + manifest: + author: jhnnsrs + created_at: '2024-01-17T16:13:30.177467' + entrypoint: app + identifier: testing + requirements: [] + scopes: + - read + version: 0.0.1 + selectors: + - compute_capability: '3.5' + cuda_cores: 6000 + cuda_version: '10.2' + required: true + type: cuda +- build_id: ec981a3b-0e64-43a3-8c8b-41accfea5bb2 + deployed_at: '2024-01-22T10:33:56.827211' + deployment_id: ec60879f-5455-46fc-9483-9cf5a2d66131 + deployment_run: c3a9190d-de9d-4ca7-9da9-3d70d9c1f281 + flavour: cuda + image: jhnnsrs/testing:0.0.2-cuda + inspection: + definitions: + - args: + - annotations: [] + description: 'The input string + + + Returns' + key: input + kind: STRING + label: input + nullable: false + scope: GLOBAL + collections: [] + description: 'This function prints the input string to + + the console' + interfaces: [] + kind: FUNCTION + name: Print String + port_groups: [] + returns: + - annotations: [] + key: return0 + kind: STRING + nullable: false + scope: GLOBAL + size: 606452033 + manifest: + author: jhnnsrs + created_at: '2024-01-22T10:30:52.268973' + entrypoint: app + identifier: testing + requirements: [] + scopes: + - read + version: 0.0.2 + selectors: + - compute_capability: '3.5' + cuda_cores: 6000 + cuda_version: '10.2' + required: true + type: cuda +latest_deployment_run: 11f43a7b-7b6c-4612-9580-08fbd0234f9a diff --git a/tests/test_init.py b/tests/test_init.py index 72e410e..2a16c6c 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -12,6 +12,7 @@ def test_init() -> None: assert backend.channel_layer is not None, "Backend should have a channel layer" @pytest.mark.asyncio +@pytest.mark.docker async def test_pull_flavour(db: typing.Any) -> None: backend = DockerBackend() @@ -52,6 +53,7 @@ async def test_pull_flavour(db: typing.Any) -> None: @pytest.mark.asyncio +@pytest.mark.docker async def test_up_setup(db: typing.Any) -> None: backend = DockerBackend() diff --git a/tests/test_retrieve_repo.py b/tests/test_retrieve_repo.py new file mode 100644 index 0000000..8562a11 --- /dev/null +++ b/tests/test_retrieve_repo.py @@ -0,0 +1,75 @@ +import pytest +from bridge.backends.docker import DockerBackend +from bridge.models import Flavour, App, Release, GithubRepo, Setup +from django.contrib.auth import get_user_model +import typing +from bridge.repo.models import DeploymentsConfigFile +import yaml +from tests.utils import build_relative_dir +from rekuest_core.enums import PortKind +from bridge.repo.db import parse_config + +@pytest.mark.asyncio +async def test_parse_deployments(db: typing.Any) -> None: + + + with open(build_relative_dir("deployments/deployments.yaml"), "r") as deployment_file: + deployment = yaml.safe_load(deployment_file) + + config = DeploymentsConfigFile(**deployment) + + + assert len(config.deployments) == 3, "Should have one deployment" + assert config.deployments[0].flavour == "vanilla", "First deployment should be vanilla" + assert config.deployments[2].flavour == "cuda", "Third deployment should be vanilla" + + third_deployment = config.deployments[2] + + assert third_deployment.inspection, "Should have an inspection" + + inspection = third_deployment.inspection + + assert inspection.definitions, "Should have definitions" + + definitions = inspection.definitions + + assert len(definitions) == 1, "Should have one definition" + + definition = definitions[0] + + assert definition.name == "Print String", "First definition should be named Print String" + + assert definition.args, "Should have args" + + assert len(definition.args) == 1, "Should have one arg" + + arg = definition.args[0] + + assert arg.key == "input", "Arg should be named string" + assert arg.kind == PortKind.STRING, "Arg should be an string" + + + + +@pytest.mark.asyncio +async def test_db_deployments(db: typing.Any) -> None: + + user, _ = await get_user_model().objects.aget_or_create( + username="test", + defaults=dict(email="test@gmail.com") + ) + + github_repo = await GithubRepo.objects.acreate( + name="test", + creator=user, + ) + + with open(build_relative_dir("deployments/deployments.yaml"), "r") as deployment_file: + deployment = yaml.safe_load(deployment_file) + + config = DeploymentsConfigFile(**deployment) + + + flavours = await parse_config(config, github_repo) + + assert len(flavours) == 3, "Should have three flavours" \ No newline at end of file diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..0eb6c2a --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,16 @@ +import os + +def build_relative_dir(path: str) -> str: + """Build a relative directory from a path + + Parameters + ---------- + path : str + The path to build a relative directory from + + Returns + ------- + str + The relative directory + """ + return os.path.join( os.path.dirname(__file__), path) \ No newline at end of file