[LocalNet] Add infrastructure to run LLM inference

Tiltfile

@@ -2,6 +2,8 @@ load("ext://restart_process", "docker_build_with_restart")
 load("ext://helm_resource", "helm_resource", "helm_repo")
 load("ext://configmap", "configmap_create")
 load("ext://secret", "secret_create_generic")
+load("ext://deployment", "deployment_create")
+load('ext://execute_in_pod', 'execute_in_pod')
 
 # A list of directories where changes trigger a hot-reload of the validator
 hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg"]
@@ -40,6 +42,10 @@ localnet_config_defaults = {
  "count": 1,
  "delve": {"enabled": False}, 
  },
+ "ollama": {
+ "enabled": False,
+ "model": "qwen:0.5b",
+ },
  # By default, we use the `helm_repo` function below to point to the remote repository
  # but can update it to the locally cloned repo for testing & development
  "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"},
@@ -364,3 +370,19 @@ k8s_resource(
 )
 
 k8s_resource("anvil", labels=["data_nodes"], port_forwards=["8547"])
+
+if localnet_config["ollama"]["enabled"]:
+ print("Ollama enabled: " + str(localnet_config["ollama"]["enabled"]))
+
+ deployment_create(
+ "ollama",
+ image="ollama/ollama",
+ command=["ollama", "serve"],
+ ports="11434",
+ )
+
+ local_resource(
+ name='ollama-pull-model',
+ cmd=execute_in_pod("ollama", "ollama pull "+localnet_config["ollama"]["model"]),
+ resource_deps=['ollama'],
+ )

config.yml

@@ -132,6 +132,9 @@ genesis:
  - service:
  id: anvil
  name: ""
+ - service:
+ id: ollama
+ name: ""
  stake:
  # NB: This value should be exactly 1upokt smaller than the value in
  # `supplier1_stake_config.yaml` so that the stake command causes a state change.
@@ -148,6 +151,13 @@ genesis:
  service:
  id: anvil
  name: ""
+ - endpoints:
+ - configs: []
+ rpc_type: REST
+ url: http://relayminer1:8545
+ service:
+ id: ollama
+ name: ""
  stake:
  # NB: This value should be exactly 1upokt smaller than the value in
  # `application1_stake_config.yaml` so that the stake command causes a state change.

localnet/kubernetes/values-relayminer-1.yaml

@@ -7,3 +7,9 @@ config:
  backend_url: http://anvil:8547/
  publicly_exposed_endpoints:
  - relayminer1
+ - service_id: ollama
+ listen_url: http://0.0.0.0:8545
+ service_config:
+ backend_url: http://ollama:11434/
+ publicly_exposed_endpoints:
+ - relayminer1

localnet/kubernetes/values-relayminer-2.yaml

@@ -7,3 +7,9 @@ config:
  backend_url: http://anvil:8547/
  publicly_exposed_endpoints:
  - relayminer2
+ - service_id: ollama
+ listen_url: http://0.0.0.0:8545
+ service_config:
+ backend_url: http://ollama:11434/
+ publicly_exposed_endpoints:
+ - relayminer2

localnet/kubernetes/values-relayminer-3.yaml

@@ -7,3 +7,9 @@ config:
  backend_url: http://anvil:8547/
  publicly_exposed_endpoints:
  - relayminer3
+ - service_id: ollama
+ listen_url: http://0.0.0.0:8545
+ service_config:
+ backend_url: http://ollama:11434/
+ publicly_exposed_endpoints:
+ - relayminer3