diff --git a/architecture/system-arch.py b/architecture/system-arch.py index 454acebf..02944de7 100644 --- a/architecture/system-arch.py +++ b/architecture/system-arch.py @@ -11,7 +11,6 @@ from diagrams.onprem.logging import Loki from diagrams.onprem.monitoring import Grafana from diagrams.onprem.monitoring import Prometheus -from diagrams.onprem.network import Apache from diagrams.onprem.queue import Rabbitmq from diagrams.programming.flowchart import Database @@ -66,17 +65,19 @@ with Cluster("TA4 Kubernetes Cluster"): with Cluster("Middle Tier"): + equation_extraction = Custom("Equation Extraction", "./resources/uncharted.png") + equation_extraction_taskrunner = Custom("Equation Extraction Taskrunner", "./resources/uncharted.png") + funman_taskrunner = Custom("Funman Taskrunner", "./resources/uncharted.png") + gollm = Custom("Gollm", "./resources/uncharted.png") + gollm_taskrunner = Custom("Gollm Taskrunner", "./resources/uncharted.png") hmi_server = Custom("HMI Server", "./resources/uncharted.png") - web_servers = [Custom("Web Server", "./resources/uncharted.png"), Custom("Docs Server", "./resources/uncharted.png")] keycloak = Custom("Keycloak", "./resources/keycloak.png") message_queue = Rabbitmq("Message Queue") - gollm_taskrunner = Custom("Gollm Taskrunner", "./resources/uncharted.png") - gollm = Custom("Gollm", "./resources/uncharted.png") - mira_taskrunner = Custom("MIRA Taskrunner", "./resources/uncharted.png") mira_local = Custom("MIRA", "./resources/uncharted.png") - equation_extraction_cpu_taskrunner = Custom("Equation Extraction (CPU) Taskrunner", "./resources/uncharted.png") - equation_extraction_cpu = Custom("Equation Extraction (CPU)", "./resources/uncharted.png") - funman_taskrunner = Custom("Funman Taskrunner", "./resources/uncharted.png") + mira_taskrunner = Custom("MIRA Taskrunner", "./resources/uncharted.png") + text_extraction = Custom("Text Extraction", "./resources/uncharted.png") + text_extraction_taskrunner = Custom("Text Extraction Taskrunner", "./resources/uncharted.png") + web_servers = [Custom("Web Server", "./resources/uncharted.png"), Custom("Docs Server", "./resources/uncharted.png")] with Cluster("Data Sources"): spicedb = Database("SpiceDB") @@ -133,8 +134,8 @@ climate_data_worker >> Edge() >> openai climate_data_worker >> Edge() >> redis climate_data_worker >> Edge() >> s3 - equation_extraction_cpu_taskrunner >> Edge() >> equation_extraction_cpu - equation_extraction_cpu_taskrunner >> Edge() >> message_queue + equation_extraction_taskrunner >> Edge() >> equation_extraction + equation_extraction_taskrunner >> Edge() >> message_queue funman_taskrunner >> Edge() >> funman funman_taskrunner >> Edge() >> message_queue gollm_taskrunner >> Edge() >> gollm @@ -148,6 +149,7 @@ hmi_server >> Edge() >> climate_data_api hmi_server >> Edge() >> cosmos hmi_server >> Edge() >> dkg + hmi_server >> Edge() >> equation_extraction_taskrunner hmi_server >> Edge() >> es hmi_server >> Edge() >> funman hmi_server >> Edge() >> github @@ -167,6 +169,7 @@ hmi_server >> Edge() >> skema_tr hmi_server >> Edge() >> skema_unified hmi_server >> Edge() >> spicedb + hmi_server >> Edge() >> text_extraction_taskrunner hmi_server >> Edge() >> xdd ingress_app >> Edge() << hmi_client ingress_docs >> Edge() << docs @@ -183,5 +186,7 @@ sciml_service >> Edge() >> message_queue skema_unified >> Edge() >> mit_tr skema_unified >> Edge() >> skema_rs + text_extraction_taskrunner >> Edge() >> message_queue + text_extraction_taskrunner >> Edge() >> text_extraction web_servers >> Edge() << ingress_app web_servers >> Edge() << ingress_docs diff --git a/kubernetes/base/services/text-extraction-taskrunner/kustomization.yaml b/kubernetes/base/services/text-extraction-taskrunner/kustomization.yaml new file mode 100644 index 00000000..aaa8f319 --- /dev/null +++ b/kubernetes/base/services/text-extraction-taskrunner/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: terarium +resources: + - text-extraction-taskrunner-deployment.yaml diff --git a/kubernetes/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml b/kubernetes/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml new file mode 100644 index 00000000..fd7cf4c3 --- /dev/null +++ b/kubernetes/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: text-extraction-taskrunner + labels: + software.uncharted.terarium/name: text-extraction-taskrunner + software.uncharted.terarium/component: taskrunner + software.uncharted.terarium/service: text-extraction + software.uncharted.terarium/part-of: services +spec: + replicas: 1 + selector: + matchLabels: + software.uncharted.terarium/name: text-extraction-taskrunner + strategy: + type: Recreate + template: + metadata: + labels: + software.uncharted.terarium/name: text-extraction-taskrunner + spec: + containers: + - name: text-extraction-taskrunner + image: text-extraction-taskrunner-image + imagePullPolicy: Always + resources: {} + imagePullSecrets: + - name: ghcr-cred + restartPolicy: Always +status: {} diff --git a/kubernetes/overlays/prod/base/kustomization.yaml b/kubernetes/overlays/prod/base/kustomization.yaml index 1f85b3a6..9af8fdb3 100644 --- a/kubernetes/overlays/prod/base/kustomization.yaml +++ b/kubernetes/overlays/prod/base/kustomization.yaml @@ -19,6 +19,7 @@ resources: - ../../../base/services/redis - ../../../base/services/sciml-service - ../../../base/services/skema + - ../../../base/services/text-extraction-taskrunner - services/beaker/beaker-persistent-volume.yaml - services/beaker/beaker-pv-claim.yaml patches: @@ -47,6 +48,7 @@ patches: - path: services/skema/skema-memgraph-deployment.yaml - path: services/skema/skema-rs-deployment.yaml - path: services/skema/skema-unified-deployment.yaml + - path: services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml configMapGenerator: - files: diff --git a/kubernetes/overlays/prod/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml b/kubernetes/overlays/prod/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml new file mode 100644 index 00000000..03241703 --- /dev/null +++ b/kubernetes/overlays/prod/base/services/text-extraction-taskrunner/text-extraction-taskrunner-deployment.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: text-extraction-taskrunner +spec: + strategy: + type: RollingUpdate + template: + spec: + containers: + - name: text-extraction-taskrunner + imagePullPolicy: Always + env: + - name: SPRING_RABBITMQ_ADDRESSES + valueFrom: + secretKeyRef: + name: mq-creds + key: address + - name: TERARIUM_MQ-USERNAME + valueFrom: + secretKeyRef: + name: mq-creds + key: username + - name: TERARIUM_MQ-PASSWORD + valueFrom: + secretKeyRef: + name: mq-creds + key: password + - name: TERARIUM_TASKRUNNER_REQUEST-TYPE + value: 'text-extraction' diff --git a/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt b/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt index ac23b484..d497b123 100644 --- a/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt +++ b/kubernetes/overlays/prod/overlays/askem-dev/check-latest/images.txt @@ -15,3 +15,4 @@ software.uncharted.terarium/name=skema-mathjax ghcr.io/darpa-askem/skema-py:late software.uncharted.terarium/name=skema-rs ghcr.io/darpa-askem/skema-rs:latest software.uncharted.terarium/name=skema-unified ghcr.io/darpa-askem/skema-py:latest software.uncharted.terarium/name=terarium-docs ghcr.io/darpa-askem/terarium-docs:latest +software.uncharted.terarium/name=text-extraction-taskrunner ghcr.io/darpa-askem/text-extraction-taskrunner:latest diff --git a/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml index 2f8203a9..2879b92a 100644 --- a/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml +++ b/kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml @@ -120,6 +120,11 @@ images: newName: ghcr.io/darpa-askem/equation-extraction-taskrunner newTag: 'latest' + # Text extraction (CPU) + - name: text-extraction-taskrunner-image + newName: ghcr.io/darpa-askem/text-extraction-taskrunner + newTag: 'latest' + # climate data - name: climate-data-image newName: ghcr.io/darpa-askem/climate-data-service diff --git a/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml index be0f9f67..e76919a9 100644 --- a/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml +++ b/kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml @@ -36,13 +36,13 @@ images: # Terarium HMI - name: hmi-client-image newName: ghcr.io/darpa-askem/hmi-client - newTag: '2.21.1' + newTag: '2.21.2' - name: terarium-docs-image newName: ghcr.io/darpa-askem/terarium-docs newTag: 'latest' - name: hmi-server-image newName: ghcr.io/darpa-askem/hmi-server - newTag: '2.21.1' + newTag: '2.21.2' - name: spicedb-image newName: ghcr.io/authzed/spicedb newTag: 'v1.24.0' @@ -89,12 +89,17 @@ images: # GoLLM service - name: gollm-taskrunner-image newName: ghcr.io/darpa-askem/gollm-taskrunner - newTag: '2.21.1' + newTag: '2.21.2' # Equation extraction (CPU) - name: equation-extraction-taskrunner-image newName: ghcr.io/darpa-askem/equation-extraction-taskrunner - newTag: '2.21.1' + newTag: '2.21.2' + + # Text extraction (CPU) + - name: text-extraction-taskrunner-image + newName: ghcr.io/darpa-askem/text-extraction-taskrunner + newTag: '2.21.2' # Integration Dashboard - name: integration-dashboard-image @@ -109,7 +114,7 @@ images: # mira taskrunner - name: mira-taskrunner-image newName: ghcr.io/darpa-askem/mira-taskrunner - newTag: '2.21.1' + newTag: '2.21.2' # cosmos (xdd) - name: cosmos-image @@ -119,7 +124,7 @@ images: # funman taskrunner - name: funman-taskrunner-image newName: ghcr.io/darpa-askem/funman-taskrunner - newTag: '2.21.1' + newTag: '2.21.2' # Redis - name: redis-image diff --git a/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt b/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt index ac23b484..d497b123 100644 --- a/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt +++ b/kubernetes/overlays/prod/overlays/askem-staging/check-latest/images.txt @@ -15,3 +15,4 @@ software.uncharted.terarium/name=skema-mathjax ghcr.io/darpa-askem/skema-py:late software.uncharted.terarium/name=skema-rs ghcr.io/darpa-askem/skema-rs:latest software.uncharted.terarium/name=skema-unified ghcr.io/darpa-askem/skema-py:latest software.uncharted.terarium/name=terarium-docs ghcr.io/darpa-askem/terarium-docs:latest +software.uncharted.terarium/name=text-extraction-taskrunner ghcr.io/darpa-askem/text-extraction-taskrunner:latest diff --git a/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml b/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml index ed4ed0d0..919b77ca 100644 --- a/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml +++ b/kubernetes/overlays/prod/overlays/askem-staging/kustomization.yaml @@ -105,6 +105,11 @@ images: newName: ghcr.io/darpa-askem/equation-extraction-taskrunner newTag: 'latest' + # Text extraction (CPU) + - name: text-extraction-taskrunner-image + newName: ghcr.io/darpa-askem/text-extraction-taskrunner + newTag: 'latest' + # climate data - name: climate-data-image newName: ghcr.io/darpa-askem/climate-data-service