From 751d694215491bcab281682cec1b3f3920de6560 Mon Sep 17 00:00:00 2001 From: David Gauldie Date: Mon, 23 Sep 2024 15:56:17 -0400 Subject: [PATCH] added equations-extraction taskrunner --- architecture/system-arch.py | 7 +++++ ...tion-extraction-taskrunner-deployment.yaml | 31 +++++++++++++++++++ .../kustomization.yaml | 6 ++++ .../overlays/prod/base/kustomization.yaml | 12 ++++--- ...tion-extraction-taskrunner-deployment.yaml | 31 +++++++++++++++++++ .../askem-dev/check-latest/images.txt | 1 + .../overlays/askem-dev/kustomization.yaml | 5 +++ .../askem-production/kustomization.yaml | 15 ++++++--- .../askem-staging/check-latest/images.txt | 3 +- .../overlays/askem-staging/kustomization.yaml | 5 +++ 10 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 kubernetes/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml create mode 100644 kubernetes/base/services/equation-extraction-taskrunner/kustomization.yaml create mode 100644 kubernetes/overlays/prod/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml diff --git a/architecture/system-arch.py b/architecture/system-arch.py index 6745d36b..454acebf 100644 --- a/architecture/system-arch.py +++ b/architecture/system-arch.py @@ -74,6 +74,9 @@ gollm = Custom("Gollm", "./resources/uncharted.png") mira_taskrunner = Custom("MIRA Taskrunner", "./resources/uncharted.png") mira_local = Custom("MIRA", "./resources/uncharted.png") + equation_extraction_cpu_taskrunner = Custom("Equation Extraction (CPU) Taskrunner", "./resources/uncharted.png") + equation_extraction_cpu = Custom("Equation Extraction (CPU)", "./resources/uncharted.png") + funman_taskrunner = Custom("Funman Taskrunner", "./resources/uncharted.png") with Cluster("Data Sources"): spicedb = Database("SpiceDB") @@ -130,6 +133,10 @@ climate_data_worker >> Edge() >> openai climate_data_worker >> Edge() >> redis climate_data_worker >> Edge() >> s3 + equation_extraction_cpu_taskrunner >> Edge() >> equation_extraction_cpu + equation_extraction_cpu_taskrunner >> Edge() >> message_queue + funman_taskrunner >> Edge() >> funman + funman_taskrunner >> Edge() >> message_queue gollm_taskrunner >> Edge() >> gollm gollm_taskrunner >> Edge() >> message_queue grafana >> Edge() >> es diff --git a/kubernetes/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml b/kubernetes/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml new file mode 100644 index 00000000..68c63603 --- /dev/null +++ b/kubernetes/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: equation-extraction-taskrunner + labels: + software.uncharted.terarium/name: equation-extraction-taskrunner + software.uncharted.terarium/component: taskrunner + software.uncharted.terarium/service: equation-extraction + software.uncharted.terarium/part-of: services +spec: + replicas: 1 + selector: + matchLabels: + software.uncharted.terarium/name: equation-extraction-taskrunner + strategy: + type: Recreate + template: + metadata: + labels: + software.uncharted.terarium/name: equation-extraction-taskrunner + spec: + containers: + - name: equation-extraction-taskrunner + image: equation-extraction-taskrunner-image + imagePullPolicy: Always + resources: {} + imagePullSecrets: + - name: ghcr-cred + restartPolicy: Always +status: {} diff --git a/kubernetes/base/services/equation-extraction-taskrunner/kustomization.yaml b/kubernetes/base/services/equation-extraction-taskrunner/kustomization.yaml new file mode 100644 index 00000000..d3777fc1 --- /dev/null +++ b/kubernetes/base/services/equation-extraction-taskrunner/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: terarium +resources: + - equation-extraction-taskrunner-deployment.yaml diff --git a/kubernetes/overlays/prod/base/kustomization.yaml b/kubernetes/overlays/prod/base/kustomization.yaml index ba256cac..1f85b3a6 100644 --- a/kubernetes/overlays/prod/base/kustomization.yaml +++ b/kubernetes/overlays/prod/base/kustomization.yaml @@ -6,13 +6,14 @@ resources: - ../../../base/hmi/client - ../../../base/hmi/documentation - ../../../base/hmi/server + - ../../../base/services/beaker - ../../../base/services/climate-data - ../../../base/services/cosmos - - ../../../base/services/beaker - ../../../base/services/data-service + - ../../../base/services/equation-extraction-taskrunner + - ../../../base/services/funman-taskrunner - ../../../base/services/gollm-taskrunner - ../../../base/services/mira-taskrunner - - ../../../base/services/funman-taskrunner - ../../../base/services/mit - ../../../base/services/pyciemss-service - ../../../base/services/redis @@ -28,14 +29,15 @@ patches: - path: hmi/server/hmi-server-service.yaml - path: hmi/server/spicedb-deployment.yaml - path: hmi/server/spicedb-service.yaml - - path: services/cosmos/cosmos-deployment.yaml - path: services/beaker/beaker-deployment.yaml - - path: services/data-service/data-service-graphdb-deployment.yaml - path: services/climate-data/climate-data-deployment.yaml - path: services/climate-data/climate-data-worker-deployment.yaml + - path: services/cosmos/cosmos-deployment.yaml + - path: services/data-service/data-service-graphdb-deployment.yaml + - path: services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml + - path: services/funman-taskrunner/funman-taskrunner-deployment.yaml - path: services/gollm-taskrunner/gollm-taskrunner-deployment.yaml - path: services/mira-taskrunner/mira-taskrunner-deployment.yaml - - path: services/funman-taskrunner/funman-taskrunner-deployment.yaml - path: services/mit/mit-tr-deployment.yaml - path: services/pyciemss-service/pyciemss-api-deployment.yaml - path: services/pyciemss-service/pyciemss-worker-deployment.yaml diff --git a/kubernetes/overlays/prod/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml b/kubernetes/overlays/prod/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml new file mode 100644 index 00000000..d8efbb1c --- /dev/null +++ b/kubernetes/overlays/prod/base/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: equation-extraction-taskrunner +spec: + strategy: + type: RollingUpdate + template: + spec: + containers: + - name: equation-extraction-taskrunner + imagePullPolicy: Always + env: + - name: SPRING_RABBITMQ_ADDRESSES + valueFrom: + secretKeyRef: + name: mq-creds + key: address + - name: TERARIUM_MQ-USERNAME + valueFrom: + secretKeyRef: + name: mq-creds + key: username + - name: TERARIUM_MQ-PASSWORD + valueFrom: + secretKeyRef: + name: mq-creds + key: password + - name: TERARIUM_TASKRUNNER_REQUEST-TYPE + value: 'equation-extraction' diff --git a/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt b/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt index df60a382..ac23b484 100644 --- a/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt +++ b/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt @@ -1,6 +1,7 @@ software.uncharted.terarium/name=beaker ghcr.io/darpa-askem/beaker-kernel:latest software.uncharted.terarium/name=climate-data ghcr.io/darpa-askem/climate-data-service:latest software.uncharted.terarium/name=climate-data-worker ghcr.io/darpa-askem/climate-data-service:latest +software.uncharted.terarium/name=equation-extraction-taskrunner ghcr.io/darpa-askem/equation-extraction-taskrunner:latest software.uncharted.terarium/name=funman-taskrunner ghcr.io/darpa-askem/funman-taskrunner:latest software.uncharted.terarium/name=gollm-taskrunner ghcr.io/darpa-askem/gollm-taskrunner:latest software.uncharted.terarium/name=hmi-client ghcr.io/darpa-askem/hmi-client:latest diff --git a/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml index 491dc531..2f8203a9 100644 --- a/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml +++ b/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml @@ -115,6 +115,11 @@ images: newName: ghcr.io/darpa-askem/gollm-taskrunner newTag: 'latest' + # Equation extraction (CPU) + - name: equation-extraction-taskrunner-image + newName: ghcr.io/darpa-askem/equation-extraction-taskrunner + newTag: 'latest' + # climate data - name: climate-data-image newName: ghcr.io/darpa-askem/climate-data-service diff --git a/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml index da3f5046..be0f9f67 100644 --- a/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml +++ b/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml @@ -36,13 +36,13 @@ images: # Terarium HMI - name: hmi-client-image newName: ghcr.io/darpa-askem/hmi-client - newTag: '2.20.0' + newTag: '2.21.1' - name: terarium-docs-image newName: ghcr.io/darpa-askem/terarium-docs newTag: 'latest' - name: hmi-server-image newName: ghcr.io/darpa-askem/hmi-server - newTag: '2.20.0' + newTag: '2.21.1' - name: spicedb-image newName: ghcr.io/authzed/spicedb newTag: 'v1.24.0' @@ -89,7 +89,12 @@ images: # GoLLM service - name: gollm-taskrunner-image newName: ghcr.io/darpa-askem/gollm-taskrunner - newTag: '2.20.0' + newTag: '2.21.1' + + # Equation extraction (CPU) + - name: equation-extraction-taskrunner-image + newName: ghcr.io/darpa-askem/equation-extraction-taskrunner + newTag: '2.21.1' # Integration Dashboard - name: integration-dashboard-image @@ -104,7 +109,7 @@ images: # mira taskrunner - name: mira-taskrunner-image newName: ghcr.io/darpa-askem/mira-taskrunner - newTag: '2.20.0' + newTag: '2.21.1' # cosmos (xdd) - name: cosmos-image @@ -114,7 +119,7 @@ images: # funman taskrunner - name: funman-taskrunner-image newName: ghcr.io/darpa-askem/funman-taskrunner - newTag: '2.20.0' + newTag: '2.21.1' # Redis - name: redis-image diff --git a/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt b/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt index 7d673071..ac23b484 100644 --- a/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt +++ b/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt @@ -1,11 +1,12 @@ software.uncharted.terarium/name=beaker ghcr.io/darpa-askem/beaker-kernel:latest software.uncharted.terarium/name=climate-data ghcr.io/darpa-askem/climate-data-service:latest software.uncharted.terarium/name=climate-data-worker ghcr.io/darpa-askem/climate-data-service:latest +software.uncharted.terarium/name=equation-extraction-taskrunner ghcr.io/darpa-askem/equation-extraction-taskrunner:latest +software.uncharted.terarium/name=funman-taskrunner ghcr.io/darpa-askem/funman-taskrunner:latest software.uncharted.terarium/name=gollm-taskrunner ghcr.io/darpa-askem/gollm-taskrunner:latest software.uncharted.terarium/name=hmi-client ghcr.io/darpa-askem/hmi-client:latest software.uncharted.terarium/name=hmi-server ghcr.io/darpa-askem/hmi-server:latest software.uncharted.terarium/name=mira-taskrunner ghcr.io/darpa-askem/mira-taskrunner:latest -software.uncharted.terarium/name=funman-taskrunner ghcr.io/darpa-askem/funman-taskrunner:latest software.uncharted.terarium/name=mit-tr ghcr.io/darpa-askem/mit-tr:latest software.uncharted.terarium/name=pyciemss-api ghcr.io/darpa-askem/pyciemss-api:latest software.uncharted.terarium/name=pyciemss-worker ghcr.io/darpa-askem/pyciemss-worker:latest diff --git a/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml index 5f7ba3e4..ed4ed0d0 100644 --- a/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml +++ b/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml @@ -100,6 +100,11 @@ images: newName: ghcr.io/darpa-askem/gollm-taskrunner newTag: 'latest' + # Equation extraction (CPU) + - name: equation-extraction-taskrunner-image + newName: ghcr.io/darpa-askem/equation-extraction-taskrunner + newTag: 'latest' + # climate data - name: climate-data-image newName: ghcr.io/darpa-askem/climate-data-service