added equations-extraction taskrunner

DARPA-ASKEM · Sep 23, 2024 · 751d694 · 751d694
1 parent 945f56c
commit 751d694
Show file tree

Hide file tree

Showing 10 changed files with 105 additions and 11 deletions.
diff --git a/architecture/system-arch.py b/architecture/system-arch.py
@@ -74,6 +74,9 @@
  gollm = Custom("Gollm", "./resources/uncharted.png")
  mira_taskrunner = Custom("MIRA Taskrunner", "./resources/uncharted.png")
  mira_local = Custom("MIRA", "./resources/uncharted.png")
+ equation_extraction_cpu_taskrunner = Custom("Equation Extraction (CPU) Taskrunner", "./resources/uncharted.png")
+ equation_extraction_cpu = Custom("Equation Extraction (CPU)", "./resources/uncharted.png")
+ funman_taskrunner = Custom("Funman Taskrunner", "./resources/uncharted.png")
 
  with Cluster("Data Sources"):
  spicedb = Database("SpiceDB")
@@ -130,6 +133,10 @@
  climate_data_worker >> Edge() >> openai
  climate_data_worker >> Edge() >> redis
  climate_data_worker >> Edge() >> s3
+ equation_extraction_cpu_taskrunner >> Edge() >> equation_extraction_cpu
+ equation_extraction_cpu_taskrunner >> Edge() >> message_queue
+ funman_taskrunner >> Edge() >> funman
+ funman_taskrunner >> Edge() >> message_queue
  gollm_taskrunner >> Edge() >> gollm
  gollm_taskrunner >> Edge() >> message_queue
  grafana >> Edge() >> es

diff --git a/...se/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml b/...se/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml
@@ -0,0 +1,31 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: equation-extraction-taskrunner
+ labels:
+ software.uncharted.terarium/name: equation-extraction-taskrunner
+ software.uncharted.terarium/component: taskrunner
+ software.uncharted.terarium/service: equation-extraction
+ software.uncharted.terarium/part-of: services
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ software.uncharted.terarium/name: equation-extraction-taskrunner
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ software.uncharted.terarium/name: equation-extraction-taskrunner
+ spec:
+ containers:
+ - name: equation-extraction-taskrunner
+ image: equation-extraction-taskrunner-image
+ imagePullPolicy: Always
+ resources: {}
+ imagePullSecrets:
+ - name: ghcr-cred
+ restartPolicy: Always
+status: {}
diff --git a/kubernetes/base/services/equation-extraction-taskrunner/kustomization.yaml b/kubernetes/base/services/equation-extraction-taskrunner/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: terarium
+resources:
+ - equation-extraction-taskrunner-deployment.yaml
diff --git a/kubernetes/overlays/prod/base/kustomization.yaml b/kubernetes/overlays/prod/base/kustomization.yaml
@@ -6,13 +6,14 @@ resources:
  - ../../../base/hmi/client
  - ../../../base/hmi/documentation
  - ../../../base/hmi/server
+ - ../../../base/services/beaker
  - ../../../base/services/climate-data
  - ../../../base/services/cosmos
- - ../../../base/services/beaker
  - ../../../base/services/data-service
+ - ../../../base/services/equation-extraction-taskrunner
+ - ../../../base/services/funman-taskrunner
  - ../../../base/services/gollm-taskrunner
  - ../../../base/services/mira-taskrunner
- - ../../../base/services/funman-taskrunner
  - ../../../base/services/mit
  - ../../../base/services/pyciemss-service
  - ../../../base/services/redis
@@ -28,14 +29,15 @@ patches:
  - path: hmi/server/hmi-server-service.yaml
  - path: hmi/server/spicedb-deployment.yaml
  - path: hmi/server/spicedb-service.yaml
- - path: services/cosmos/cosmos-deployment.yaml
  - path: services/beaker/beaker-deployment.yaml
- - path: services/data-service/data-service-graphdb-deployment.yaml
  - path: services/climate-data/climate-data-deployment.yaml
  - path: services/climate-data/climate-data-worker-deployment.yaml
+ - path: services/cosmos/cosmos-deployment.yaml
+ - path: services/data-service/data-service-graphdb-deployment.yaml
+ - path: services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml
+ - path: services/funman-taskrunner/funman-taskrunner-deployment.yaml
  - path: services/gollm-taskrunner/gollm-taskrunner-deployment.yaml
  - path: services/mira-taskrunner/mira-taskrunner-deployment.yaml
- - path: services/funman-taskrunner/funman-taskrunner-deployment.yaml
  - path: services/mit/mit-tr-deployment.yaml
  - path: services/pyciemss-service/pyciemss-api-deployment.yaml
  - path: services/pyciemss-service/pyciemss-worker-deployment.yaml

diff --git a/...se/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml b/...se/services/equation-extraction-taskrunner/equation-extraction-taskrunner-deployment.yaml
@@ -0,0 +1,31 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: equation-extraction-taskrunner
+spec:
+ strategy:
+ type: RollingUpdate
+ template:
+ spec:
+ containers:
+ - name: equation-extraction-taskrunner
+ imagePullPolicy: Always
+ env:
+ - name: SPRING_RABBITMQ_ADDRESSES
+ valueFrom:
+ secretKeyRef:
+ name: mq-creds
+ key: address
+ - name: TERARIUM_MQ-USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: mq-creds
+ key: username
+ - name: TERARIUM_MQ-PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mq-creds
+ key: password
+ - name: TERARIUM_TASKRUNNER_REQUEST-TYPE
+ value: 'equation-extraction'
diff --git a/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt b/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt
@@ -1,6 +1,7 @@
 software.uncharted.terarium/name=beaker ghcr.io/darpa-askem/beaker-kernel:latest
 software.uncharted.terarium/name=climate-data ghcr.io/darpa-askem/climate-data-service:latest
 software.uncharted.terarium/name=climate-data-worker ghcr.io/darpa-askem/climate-data-service:latest
+software.uncharted.terarium/name=equation-extraction-taskrunner ghcr.io/darpa-askem/equation-extraction-taskrunner:latest
 software.uncharted.terarium/name=funman-taskrunner ghcr.io/darpa-askem/funman-taskrunner:latest
 software.uncharted.terarium/name=gollm-taskrunner ghcr.io/darpa-askem/gollm-taskrunner:latest
 software.uncharted.terarium/name=hmi-client ghcr.io/darpa-askem/hmi-client:latest

diff --git a/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml
@@ -115,6 +115,11 @@ images:
  newName: ghcr.io/darpa-askem/gollm-taskrunner
  newTag: 'latest'
 
+ # Equation extraction (CPU)
+ - name: equation-extraction-taskrunner-image
+ newName: ghcr.io/darpa-askem/equation-extraction-taskrunner
+ newTag: 'latest'
+
  # climate data
  - name: climate-data-image
  newName: ghcr.io/darpa-askem/climate-data-service

diff --git a/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml
@@ -36,13 +36,13 @@ images:
  # Terarium HMI
  - name: hmi-client-image
  newName: ghcr.io/darpa-askem/hmi-client
- newTag: '2.20.0'
+ newTag: '2.21.1'
  - name: terarium-docs-image
  newName: ghcr.io/darpa-askem/terarium-docs
  newTag: 'latest'
  - name: hmi-server-image
  newName: ghcr.io/darpa-askem/hmi-server
- newTag: '2.20.0'
+ newTag: '2.21.1'
  - name: spicedb-image
  newName: ghcr.io/authzed/spicedb
  newTag: 'v1.24.0'
@@ -89,7 +89,12 @@ images:
  # GoLLM service
  - name: gollm-taskrunner-image
  newName: ghcr.io/darpa-askem/gollm-taskrunner
- newTag: '2.20.0'
+ newTag: '2.21.1'
+
+ # Equation extraction (CPU)
+ - name: equation-extraction-taskrunner-image
+ newName: ghcr.io/darpa-askem/equation-extraction-taskrunner
+ newTag: '2.21.1'
 
  # Integration Dashboard
  - name: integration-dashboard-image
@@ -104,7 +109,7 @@ images:
  # mira taskrunner
  - name: mira-taskrunner-image
  newName: ghcr.io/darpa-askem/mira-taskrunner
- newTag: '2.20.0'
+ newTag: '2.21.1'
 
  # cosmos (xdd)
  - name: cosmos-image
@@ -114,7 +119,7 @@ images:
  # funman taskrunner
  - name: funman-taskrunner-image
  newName: ghcr.io/darpa-askem/funman-taskrunner
- newTag: '2.20.0'
+ newTag: '2.21.1'
 
  # Redis
  - name: redis-image

diff --git a/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt b/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt
@@ -1,11 +1,12 @@
 software.uncharted.terarium/name=beaker ghcr.io/darpa-askem/beaker-kernel:latest
 software.uncharted.terarium/name=climate-data ghcr.io/darpa-askem/climate-data-service:latest
 software.uncharted.terarium/name=climate-data-worker ghcr.io/darpa-askem/climate-data-service:latest
+software.uncharted.terarium/name=equation-extraction-taskrunner ghcr.io/darpa-askem/equation-extraction-taskrunner:latest
+software.uncharted.terarium/name=funman-taskrunner ghcr.io/darpa-askem/funman-taskrunner:latest
 software.uncharted.terarium/name=gollm-taskrunner ghcr.io/darpa-askem/gollm-taskrunner:latest
 software.uncharted.terarium/name=hmi-client ghcr.io/darpa-askem/hmi-client:latest
 software.uncharted.terarium/name=hmi-server ghcr.io/darpa-askem/hmi-server:latest
 software.uncharted.terarium/name=mira-taskrunner ghcr.io/darpa-askem/mira-taskrunner:latest
-software.uncharted.terarium/name=funman-taskrunner ghcr.io/darpa-askem/funman-taskrunner:latest
 software.uncharted.terarium/name=mit-tr ghcr.io/darpa-askem/mit-tr:latest
 software.uncharted.terarium/name=pyciemss-api ghcr.io/darpa-askem/pyciemss-api:latest
 software.uncharted.terarium/name=pyciemss-worker ghcr.io/darpa-askem/pyciemss-worker:latest

diff --git a/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml
@@ -100,6 +100,11 @@ images:
  newName: ghcr.io/darpa-askem/gollm-taskrunner
  newTag: 'latest'
 
+ # Equation extraction (CPU)
+ - name: equation-extraction-taskrunner-image
+ newName: ghcr.io/darpa-askem/equation-extraction-taskrunner
+ newTag: 'latest'
+
  # climate data
  - name: climate-data-image
  newName: ghcr.io/darpa-askem/climate-data-service