From 3dee9c1ccbcd6549299c140b08a03454ffd66246 Mon Sep 17 00:00:00 2001 From: Dima Kniazev Date: Fri, 3 May 2024 16:24:10 -0700 Subject: [PATCH] [LocalNet] Add infrastructure to run LLM inference (#508) Adds infrastructure to run and develop against LLM on LocalNet. --- Co-authored-by: Redouane Lakrache Co-authored-by: Daniel Olshansky --- Tiltfile | 46 ++++++++++++++----- config.yml | 10 ++++ localnet/kubernetes/values-relayminer-1.yaml | 6 +++ localnet/kubernetes/values-relayminer-2.yaml | 6 +++ localnet/kubernetes/values-relayminer-3.yaml | 6 +++ .../config/supplier1_stake_config.yaml | 8 +++- .../config/supplier2_stake_config.yaml | 6 ++- .../config/supplier3_stake_config.yaml | 6 ++- x/supplier/config/supplier_configs_reader.go | 3 +- 9 files changed, 81 insertions(+), 16 deletions(-) diff --git a/Tiltfile b/Tiltfile index d4b5a18fe..5d8cca35b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,10 +2,13 @@ load("ext://restart_process", "docker_build_with_restart") load("ext://helm_resource", "helm_resource", "helm_repo") load("ext://configmap", "configmap_create") load("ext://secret", "secret_create_generic") +load("ext://deployment", "deployment_create") +load("ext://execute_in_pod", "execute_in_pod") # A list of directories where changes trigger a hot-reload of the validator hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg"] + def merge_dicts(base, updates): for k, v in updates.items(): if k in base and type(base[k]) == "dict" and type(v) == "dict": @@ -16,6 +19,7 @@ def merge_dicts(base, updates): # Replace or set the value base[k] = v + # Create a localnet config file from defaults, and if a default configuration doesn't exist, populate it with default values localnet_config_path = "localnet_config.yaml" localnet_config_defaults = { @@ -28,17 +32,19 @@ localnet_config_defaults = { "delve": {"enabled": False}, }, "observability": {"enabled": True}, - "relayminers": { - "count": 1, - "delve": {"enabled": False} - }, + "relayminers": {"count": 1, "delve": {"enabled": False}}, "gateways": { "count": 1, - "delve": {"enabled": False}, + "delve": {"enabled": False}, }, "appgateservers": { "count": 1, - "delve": {"enabled": False}, + "delve": {"enabled": False}, + }, + # TODO(#511): Add support for `REST` and enabled this. + "ollama": { + "enabled": False, + "model": "qwen:0.5b", }, # By default, we use the `helm_repo` function below to point to the remote repository # but can update it to the locally cloned repo for testing & development @@ -54,7 +60,9 @@ merge_dicts(localnet_config, localnet_config_defaults) # Then merge file contents over defaults merge_dicts(localnet_config, localnet_config_file) # Check if there are differences or if the file doesn't exist -if (localnet_config_file != localnet_config) or (not os.path.exists(localnet_config_path)): +if (localnet_config_file != localnet_config) or ( + not os.path.exists(localnet_config_path) +): print("Updating " + localnet_config_path + " with defaults") local("cat - > " + localnet_config_path, stdin=encode_yaml(localnet_config)) @@ -198,10 +206,8 @@ helm_resource( "--values=./localnet/kubernetes/values-validator.yaml", "--set=persistence.cleanupBeforeEachStart=" + str(localnet_config["validator"]["cleanupBeforeEachStart"]), - "--set=logs.level=" - + str(localnet_config["validator"]["logs"]["level"]), - "--set=logs.format=" - + str(localnet_config["validator"]["logs"]["format"]), + "--set=logs.level=" + str(localnet_config["validator"]["logs"]["level"]), + "--set=logs.format=" + str(localnet_config["validator"]["logs"]["format"]), "--set=serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), "--set=development.delve.enabled=" @@ -364,3 +370,21 @@ k8s_resource( ) k8s_resource("anvil", labels=["data_nodes"], port_forwards=["8547"]) + +if localnet_config["ollama"]["enabled"]: + print("Ollama enabled: " + str(localnet_config["ollama"]["enabled"])) + + deployment_create( + "ollama", + image="ollama/ollama", + command=["ollama", "serve"], + ports="11434", + ) + + local_resource( + name="ollama-pull-model", + cmd=execute_in_pod( + "ollama", "ollama pull " + localnet_config["ollama"]["model"] + ), + resource_deps=["ollama"], + ) diff --git a/config.yml b/config.yml index f213023fa..7a129a20f 100644 --- a/config.yml +++ b/config.yml @@ -132,6 +132,9 @@ genesis: - service: id: anvil name: "" + - service: + id: ollama + name: "" stake: # NB: This value should be exactly 1upokt smaller than the value in # `supplier1_stake_config.yaml` so that the stake command causes a state change. @@ -148,6 +151,13 @@ genesis: service: id: anvil name: "" + - endpoints: + - configs: [] + rpc_type: REST + url: http://relayminer1:8545 + service: + id: ollama + name: "" stake: # NB: This value should be exactly 1upokt smaller than the value in # `application1_stake_config.yaml` so that the stake command causes a state change. diff --git a/localnet/kubernetes/values-relayminer-1.yaml b/localnet/kubernetes/values-relayminer-1.yaml index 956f99cd3..b57b58ab2 100644 --- a/localnet/kubernetes/values-relayminer-1.yaml +++ b/localnet/kubernetes/values-relayminer-1.yaml @@ -7,3 +7,9 @@ config: backend_url: http://anvil:8547/ publicly_exposed_endpoints: - relayminer1 + - service_id: ollama + listen_url: http://0.0.0.0:8545 + service_config: + backend_url: http://ollama:11434/ + publicly_exposed_endpoints: + - relayminer1 diff --git a/localnet/kubernetes/values-relayminer-2.yaml b/localnet/kubernetes/values-relayminer-2.yaml index 78e9ea4f9..146400dcf 100644 --- a/localnet/kubernetes/values-relayminer-2.yaml +++ b/localnet/kubernetes/values-relayminer-2.yaml @@ -7,3 +7,9 @@ config: backend_url: http://anvil:8547/ publicly_exposed_endpoints: - relayminer2 + - service_id: ollama + listen_url: http://0.0.0.0:8545 + service_config: + backend_url: http://ollama:11434/ + publicly_exposed_endpoints: + - relayminer2 diff --git a/localnet/kubernetes/values-relayminer-3.yaml b/localnet/kubernetes/values-relayminer-3.yaml index 506d0ab7f..00953bca5 100644 --- a/localnet/kubernetes/values-relayminer-3.yaml +++ b/localnet/kubernetes/values-relayminer-3.yaml @@ -7,3 +7,9 @@ config: backend_url: http://anvil:8547/ publicly_exposed_endpoints: - relayminer3 + - service_id: ollama + listen_url: http://0.0.0.0:8545 + service_config: + backend_url: http://ollama:11434/ + publicly_exposed_endpoints: + - relayminer3 diff --git a/localnet/poktrolld/config/supplier1_stake_config.yaml b/localnet/poktrolld/config/supplier1_stake_config.yaml index fa1abaff1..520e72090 100644 --- a/localnet/poktrolld/config/supplier1_stake_config.yaml +++ b/localnet/poktrolld/config/supplier1_stake_config.yaml @@ -5,11 +5,15 @@ services: - service_id: svc1 endpoints: - url: http://localhost:8081 - rpc_type: json_rpc + rpc_type: JSON_RPC # The endpoint URL for the Anvil service is provided via the RelayMiner. # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - service_id: anvil endpoints: - publicly_exposed_url: http://relayminer1:8545 - rpc_type: json_rpc + rpc_type: JSON_RPC + - service_id: ollama + endpoints: + - publicly_exposed_url: http://relayminer1:8545 + rpc_type: REST diff --git a/localnet/poktrolld/config/supplier2_stake_config.yaml b/localnet/poktrolld/config/supplier2_stake_config.yaml index e03f730bc..77c36eb7e 100644 --- a/localnet/poktrolld/config/supplier2_stake_config.yaml +++ b/localnet/poktrolld/config/supplier2_stake_config.yaml @@ -6,4 +6,8 @@ services: # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - publicly_exposed_url: http://relayminer2:8545 - rpc_type: json_rpc + rpc_type: JSON_RPC + - service_id: ollama + endpoints: + - publicly_exposed_url: http://relayminer2:8545 + rpc_type: REST diff --git a/localnet/poktrolld/config/supplier3_stake_config.yaml b/localnet/poktrolld/config/supplier3_stake_config.yaml index ee0d95b25..8aca20dce 100644 --- a/localnet/poktrolld/config/supplier3_stake_config.yaml +++ b/localnet/poktrolld/config/supplier3_stake_config.yaml @@ -6,4 +6,8 @@ services: # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - publicly_exposed_url: http://relayminer3:8545 - rpc_type: json_rpc + rpc_type: JSON_RPC + - service_id: ollama + endpoints: + - publicly_exposed_url: http://relayminer3:8545 + rpc_type: REST diff --git a/x/supplier/config/supplier_configs_reader.go b/x/supplier/config/supplier_configs_reader.go index 535387397..0ed02b1f4 100644 --- a/x/supplier/config/supplier_configs_reader.go +++ b/x/supplier/config/supplier_configs_reader.go @@ -2,6 +2,7 @@ package config import ( "net/url" + "strings" sdk "github.com/cosmos/cosmos-sdk/types" "gopkg.in/yaml.v2" @@ -175,7 +176,7 @@ func parseEndpointConfigs(endpoint YAMLServiceEndpoint) ([]*sharedtypes.ConfigOp // parseEndpointRPCType parses the endpoint RPC type into a sharedtypes.RPCType func parseEndpointRPCType(endpoint YAMLServiceEndpoint) (sharedtypes.RPCType, error) { - switch endpoint.RPCType { + switch strings.ToLower(endpoint.RPCType) { case "json_rpc": return sharedtypes.RPCType_JSON_RPC, nil default: