added text-extraction taskrunner

architecture/system-arch.py

@@ -11,7 +11,6 @@
 from diagrams.onprem.logging import Loki
 from diagrams.onprem.monitoring import Grafana
 from diagrams.onprem.monitoring import Prometheus
-from diagrams.onprem.network import Apache
 from diagrams.onprem.queue import Rabbitmq
 from diagrams.programming.flowchart import Database
 
@@ -66,17 +65,19 @@
 
  with Cluster("TA4 Kubernetes Cluster"):
  with Cluster("Middle Tier"):
+ equation_extraction = Custom("Equation Extraction", "./resources/uncharted.png")
+ equation_extraction_taskrunner = Custom("Equation Extraction Taskrunner", "./resources/uncharted.png")
+ funman_taskrunner = Custom("Funman Taskrunner", "./resources/uncharted.png")
+ gollm = Custom("Gollm", "./resources/uncharted.png")
+ gollm_taskrunner = Custom("Gollm Taskrunner", "./resources/uncharted.png")
  hmi_server = Custom("HMI Server", "./resources/uncharted.png")
- web_servers = [Custom("Web Server", "./resources/uncharted.png"), Custom("Docs Server", "./resources/uncharted.png")]
  keycloak = Custom("Keycloak", "./resources/keycloak.png")
  message_queue = Rabbitmq("Message Queue")
- gollm_taskrunner = Custom("Gollm Taskrunner", "./resources/uncharted.png")
- gollm = Custom("Gollm", "./resources/uncharted.png")
- mira_taskrunner = Custom("MIRA Taskrunner", "./resources/uncharted.png")
  mira_local = Custom("MIRA", "./resources/uncharted.png")
- equation_extraction_cpu_taskrunner = Custom("Equation Extraction (CPU) Taskrunner", "./resources/uncharted.png")
- equation_extraction_cpu = Custom("Equation Extraction (CPU)", "./resources/uncharted.png")
- funman_taskrunner = Custom("Funman Taskrunner", "./resources/uncharted.png")
+ mira_taskrunner = Custom("MIRA Taskrunner", "./resources/uncharted.png")
+ text_extraction = Custom("Text Extraction", "./resources/uncharted.png")
+ text_extraction_taskrunner = Custom("Text Extraction Taskrunner", "./resources/uncharted.png")
+ web_servers = [Custom("Web Server", "./resources/uncharted.png"), Custom("Docs Server", "./resources/uncharted.png")]
 
  with Cluster("Data Sources"):
  spicedb = Database("SpiceDB")
@@ -133,8 +134,8 @@
  climate_data_worker >> Edge() >> openai
  climate_data_worker >> Edge() >> redis
  climate_data_worker >> Edge() >> s3
- equation_extraction_cpu_taskrunner >> Edge() >> equation_extraction_cpu
- equation_extraction_cpu_taskrunner >> Edge() >> message_queue
+ equation_extraction_taskrunner >> Edge() >> equation_extraction
+ equation_extraction_taskrunner >> Edge() >> message_queue
  funman_taskrunner >> Edge() >> funman
  funman_taskrunner >> Edge() >> message_queue
  gollm_taskrunner >> Edge() >> gollm
@@ -148,6 +149,7 @@
  hmi_server >> Edge() >> climate_data_api
  hmi_server >> Edge() >> cosmos
  hmi_server >> Edge() >> dkg
+ hmi_server >> Edge() >> equation_extraction_taskrunner
  hmi_server >> Edge() >> es
  hmi_server >> Edge() >> funman
  hmi_server >> Edge() >> github
@@ -167,6 +169,7 @@
  hmi_server >> Edge() >> skema_tr
  hmi_server >> Edge() >> skema_unified
  hmi_server >> Edge() >> spicedb
+ hmi_server >> Edge() >> text_extraction_taskrunner
  hmi_server >> Edge() >> xdd
  ingress_app >> Edge() << hmi_client
  ingress_docs >> Edge() << docs
@@ -183,5 +186,7 @@
  sciml_service >> Edge() >> message_queue
  skema_unified >> Edge() >> mit_tr
  skema_unified >> Edge() >> skema_rs
+ text_extraction_taskrunner >> Edge() >> message_queue
+ text_extraction_taskrunner >> Edge() >> text_extraction
  web_servers >> Edge() << ingress_app
  web_servers >> Edge() << ingress_docs

kubernetes/base/services/text-extraction-taskrunner/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: terarium
+resources:
+ - text-extraction-taskrunner-deployment.yaml

kubernetes/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: text-extraction-taskrunner
+ labels:
+ software.uncharted.terarium/name: text-extraction-taskrunner
+ software.uncharted.terarium/component: taskrunner
+ software.uncharted.terarium/service: text-extraction
+ software.uncharted.terarium/part-of: services
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ software.uncharted.terarium/name: text-extraction-taskrunner
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ software.uncharted.terarium/name: text-extraction-taskrunner
+ spec:
+ containers:
+ - name: text-extraction-taskrunner
+ image: text-extraction-taskrunner-image
+ imagePullPolicy: Always
+ resources: {}
+ imagePullSecrets:
+ - name: ghcr-cred
+ restartPolicy: Always
+status: {}

kubernetes/overlays/prod/base/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
  - ../../../base/services/redis
  - ../../../base/services/sciml-service
  - ../../../base/services/skema
+ - ../../../base/services/text-extraction-taskrunner
  - services/beaker/beaker-persistent-volume.yaml
  - services/beaker/beaker-pv-claim.yaml
 patches:
@@ -47,6 +48,7 @@ patches:
  - path: services/skema/skema-memgraph-deployment.yaml
  - path: services/skema/skema-rs-deployment.yaml
  - path: services/skema/skema-unified-deployment.yaml
+ - path: services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml
 
 configMapGenerator:
  - files:

kubernetes/overlays/prod/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: text-extraction-taskrunner
+spec:
+ strategy:
+ type: RollingUpdate
+ template:
+ spec:
+ containers:
+ - name: text-extraction-taskrunner
+ imagePullPolicy: Always
+ env:
+ - name: SPRING_RABBITMQ_ADDRESSES
+ valueFrom:
+ secretKeyRef:
+ name: mq-creds
+ key: address
+ - name: TERARIUM_MQ-USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: mq-creds
+ key: username
+ - name: TERARIUM_MQ-PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mq-creds
+ key: password
+ - name: TERARIUM_TASKRUNNER_REQUEST-TYPE
+ value: 'text-extraction'

kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt

@@ -15,3 +15,4 @@ software.uncharted.terarium/name=skema-mathjax ghcr.io/darpa-askem/skema-py:late
 software.uncharted.terarium/name=skema-rs ghcr.io/darpa-askem/skema-rs:latest
 software.uncharted.terarium/name=skema-unified ghcr.io/darpa-askem/skema-py:latest
 software.uncharted.terarium/name=terarium-docs ghcr.io/darpa-askem/terarium-docs:latest
+software.uncharted.terarium/name=text-extraction-taskrunner ghcr.io/darpa-askem/text-extraction-taskrunner:latest

kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml

@@ -120,6 +120,11 @@ images:
  newName: ghcr.io/darpa-askem/equation-extraction-taskrunner
  newTag: 'latest'
 
+ # Text extraction (CPU)
+ - name: text-extraction-taskrunner-image
+ newName: ghcr.io/darpa-askem/text-extraction-taskrunner
+ newTag: 'latest'
+
  # climate data
  - name: climate-data-image
  newName: ghcr.io/darpa-askem/climate-data-service

kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml

@@ -36,13 +36,13 @@ images:
  # Terarium HMI
  - name: hmi-client-image
  newName: ghcr.io/darpa-askem/hmi-client
- newTag: '2.21.1'
+ newTag: '2.21.2'
  - name: terarium-docs-image
  newName: ghcr.io/darpa-askem/terarium-docs
  newTag: 'latest'
  - name: hmi-server-image
  newName: ghcr.io/darpa-askem/hmi-server
- newTag: '2.21.1'
+ newTag: '2.21.2'
  - name: spicedb-image
  newName: ghcr.io/authzed/spicedb
  newTag: 'v1.24.0'
@@ -89,12 +89,17 @@ images:
  # GoLLM service
  - name: gollm-taskrunner-image
  newName: ghcr.io/darpa-askem/gollm-taskrunner
- newTag: '2.21.1'
+ newTag: '2.21.2'
 
  # Equation extraction (CPU)
  - name: equation-extraction-taskrunner-image
  newName: ghcr.io/darpa-askem/equation-extraction-taskrunner
- newTag: '2.21.1'
+ newTag: '2.21.2'
+
+ # Text extraction (CPU)
+ - name: text-extraction-taskrunner-image
+ newName: ghcr.io/darpa-askem/text-extraction-taskrunner
+ newTag: '2.21.2'
 
  # Integration Dashboard
  - name: integration-dashboard-image
@@ -109,7 +114,7 @@ images:
  # mira taskrunner
  - name: mira-taskrunner-image
  newName: ghcr.io/darpa-askem/mira-taskrunner
- newTag: '2.21.1'
+ newTag: '2.21.2'
 
  # cosmos (xdd)
  - name: cosmos-image
@@ -119,7 +124,7 @@ images:
  # funman taskrunner
  - name: funman-taskrunner-image
  newName: ghcr.io/darpa-askem/funman-taskrunner
- newTag: '2.21.1'
+ newTag: '2.21.2'
 
  # Redis
  - name: redis-image

kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt

@@ -15,3 +15,4 @@ software.uncharted.terarium/name=skema-mathjax ghcr.io/darpa-askem/skema-py:late
 software.uncharted.terarium/name=skema-rs ghcr.io/darpa-askem/skema-rs:latest
 software.uncharted.terarium/name=skema-unified ghcr.io/darpa-askem/skema-py:latest
 software.uncharted.terarium/name=terarium-docs ghcr.io/darpa-askem/terarium-docs:latest
+software.uncharted.terarium/name=text-extraction-taskrunner ghcr.io/darpa-askem/text-extraction-taskrunner:latest

kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml

@@ -105,6 +105,11 @@ images:
  newName: ghcr.io/darpa-askem/equation-extraction-taskrunner
  newTag: 'latest'
 
+ # Text extraction (CPU)
+ - name: text-extraction-taskrunner-image
+ newName: ghcr.io/darpa-askem/text-extraction-taskrunner
+ newTag: 'latest'
+
  # climate data
  - name: climate-data-image
  newName: ghcr.io/darpa-askem/climate-data-service