From 177e63ea43278075c60ec3350a530106b666a4ff Mon Sep 17 00:00:00 2001 From: "Dmitry K." Date: Fri, 26 Apr 2024 17:09:16 -0700 Subject: [PATCH 1/6] add infrastructure to run llm on localnet --- Tiltfile | 22 ++++++++++++++++++++ config.yml | 10 +++++++++ localnet/kubernetes/values-relayminer-1.yaml | 6 ++++++ localnet/kubernetes/values-relayminer-2.yaml | 6 ++++++ localnet/kubernetes/values-relayminer-3.yaml | 6 ++++++ 5 files changed, 50 insertions(+) diff --git a/Tiltfile b/Tiltfile index d4b5a18fe..cb56e4d91 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,6 +2,8 @@ load("ext://restart_process", "docker_build_with_restart") load("ext://helm_resource", "helm_resource", "helm_repo") load("ext://configmap", "configmap_create") load("ext://secret", "secret_create_generic") +load("ext://deployment", "deployment_create") +load('ext://execute_in_pod', 'execute_in_pod') # A list of directories where changes trigger a hot-reload of the validator hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg"] @@ -40,6 +42,10 @@ localnet_config_defaults = { "count": 1, "delve": {"enabled": False}, }, + "ollama": { + "enabled": False, + "model": "qwen:0.5b", + }, # By default, we use the `helm_repo` function below to point to the remote repository # but can update it to the locally cloned repo for testing & development "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"}, @@ -364,3 +370,19 @@ k8s_resource( ) k8s_resource("anvil", labels=["data_nodes"], port_forwards=["8547"]) + +if localnet_config["ollama"]["enabled"]: + print("Ollama enabled: " + str(localnet_config["ollama"]["enabled"])) + + deployment_create( + "ollama", + image="ollama/ollama", + command=["ollama", "serve"], + ports="11434", + ) + + local_resource( + name='ollama-pull-model', + cmd=execute_in_pod("ollama", "ollama pull "+localnet_config["ollama"]["model"]), + resource_deps=['ollama'], + ) diff --git a/config.yml b/config.yml index f213023fa..7a129a20f 100644 --- a/config.yml +++ b/config.yml @@ -132,6 +132,9 @@ genesis: - service: id: anvil name: "" + - service: + id: ollama + name: "" stake: # NB: This value should be exactly 1upokt smaller than the value in # `supplier1_stake_config.yaml` so that the stake command causes a state change. @@ -148,6 +151,13 @@ genesis: service: id: anvil name: "" + - endpoints: + - configs: [] + rpc_type: REST + url: http://relayminer1:8545 + service: + id: ollama + name: "" stake: # NB: This value should be exactly 1upokt smaller than the value in # `application1_stake_config.yaml` so that the stake command causes a state change. diff --git a/localnet/kubernetes/values-relayminer-1.yaml b/localnet/kubernetes/values-relayminer-1.yaml index 956f99cd3..b57b58ab2 100644 --- a/localnet/kubernetes/values-relayminer-1.yaml +++ b/localnet/kubernetes/values-relayminer-1.yaml @@ -7,3 +7,9 @@ config: backend_url: http://anvil:8547/ publicly_exposed_endpoints: - relayminer1 + - service_id: ollama + listen_url: http://0.0.0.0:8545 + service_config: + backend_url: http://ollama:11434/ + publicly_exposed_endpoints: + - relayminer1 diff --git a/localnet/kubernetes/values-relayminer-2.yaml b/localnet/kubernetes/values-relayminer-2.yaml index 78e9ea4f9..146400dcf 100644 --- a/localnet/kubernetes/values-relayminer-2.yaml +++ b/localnet/kubernetes/values-relayminer-2.yaml @@ -7,3 +7,9 @@ config: backend_url: http://anvil:8547/ publicly_exposed_endpoints: - relayminer2 + - service_id: ollama + listen_url: http://0.0.0.0:8545 + service_config: + backend_url: http://ollama:11434/ + publicly_exposed_endpoints: + - relayminer2 diff --git a/localnet/kubernetes/values-relayminer-3.yaml b/localnet/kubernetes/values-relayminer-3.yaml index 506d0ab7f..00953bca5 100644 --- a/localnet/kubernetes/values-relayminer-3.yaml +++ b/localnet/kubernetes/values-relayminer-3.yaml @@ -7,3 +7,9 @@ config: backend_url: http://anvil:8547/ publicly_exposed_endpoints: - relayminer3 + - service_id: ollama + listen_url: http://0.0.0.0:8545 + service_config: + backend_url: http://ollama:11434/ + publicly_exposed_endpoints: + - relayminer3 From 4c60ad9b9ffecfc8e37d182148df30d227b2fb2d Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Mon, 29 Apr 2024 06:52:06 +0200 Subject: [PATCH 2/6] add ollama to supplier stake config --- localnet/poktrolld/config/supplier1_stake_config.yaml | 4 ++++ localnet/poktrolld/config/supplier2_stake_config.yaml | 4 ++++ localnet/poktrolld/config/supplier3_stake_config.yaml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/localnet/poktrolld/config/supplier1_stake_config.yaml b/localnet/poktrolld/config/supplier1_stake_config.yaml index fa1abaff1..281cacc7c 100644 --- a/localnet/poktrolld/config/supplier1_stake_config.yaml +++ b/localnet/poktrolld/config/supplier1_stake_config.yaml @@ -13,3 +13,7 @@ services: endpoints: - publicly_exposed_url: http://relayminer1:8545 rpc_type: json_rpc + - service_id: ollama + endpoints: + - publicly_exposed_url: http://relayminer1:8545 + rpc_type: rest diff --git a/localnet/poktrolld/config/supplier2_stake_config.yaml b/localnet/poktrolld/config/supplier2_stake_config.yaml index e03f730bc..89407d4b1 100644 --- a/localnet/poktrolld/config/supplier2_stake_config.yaml +++ b/localnet/poktrolld/config/supplier2_stake_config.yaml @@ -7,3 +7,7 @@ services: # This setup allows for flexible and dynamic service provisioning within the network. - publicly_exposed_url: http://relayminer2:8545 rpc_type: json_rpc + - service_id: ollama + endpoints: + - publicly_exposed_url: http://relayminer2:8545 + rpc_type: rest diff --git a/localnet/poktrolld/config/supplier3_stake_config.yaml b/localnet/poktrolld/config/supplier3_stake_config.yaml index ee0d95b25..8aa675efb 100644 --- a/localnet/poktrolld/config/supplier3_stake_config.yaml +++ b/localnet/poktrolld/config/supplier3_stake_config.yaml @@ -7,3 +7,7 @@ services: # This setup allows for flexible and dynamic service provisioning within the network. - publicly_exposed_url: http://relayminer3:8545 rpc_type: json_rpc + - service_id: ollama + endpoints: + - publicly_exposed_url: http://relayminer3:8545 + rpc_type: rest From 451ea9d538d613297d6ff4d8b4472cdf76204edc Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Mon, 29 Apr 2024 07:07:51 +0200 Subject: [PATCH 3/6] fix: Support uppercase service ids in configs --- localnet/poktrolld/config/supplier1_stake_config.yaml | 6 +++--- localnet/poktrolld/config/supplier2_stake_config.yaml | 4 ++-- localnet/poktrolld/config/supplier3_stake_config.yaml | 4 ++-- x/supplier/config/supplier_configs_reader.go | 3 ++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/localnet/poktrolld/config/supplier1_stake_config.yaml b/localnet/poktrolld/config/supplier1_stake_config.yaml index 281cacc7c..520e72090 100644 --- a/localnet/poktrolld/config/supplier1_stake_config.yaml +++ b/localnet/poktrolld/config/supplier1_stake_config.yaml @@ -5,15 +5,15 @@ services: - service_id: svc1 endpoints: - url: http://localhost:8081 - rpc_type: json_rpc + rpc_type: JSON_RPC # The endpoint URL for the Anvil service is provided via the RelayMiner. # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - service_id: anvil endpoints: - publicly_exposed_url: http://relayminer1:8545 - rpc_type: json_rpc + rpc_type: JSON_RPC - service_id: ollama endpoints: - publicly_exposed_url: http://relayminer1:8545 - rpc_type: rest + rpc_type: REST diff --git a/localnet/poktrolld/config/supplier2_stake_config.yaml b/localnet/poktrolld/config/supplier2_stake_config.yaml index 89407d4b1..77c36eb7e 100644 --- a/localnet/poktrolld/config/supplier2_stake_config.yaml +++ b/localnet/poktrolld/config/supplier2_stake_config.yaml @@ -6,8 +6,8 @@ services: # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - publicly_exposed_url: http://relayminer2:8545 - rpc_type: json_rpc + rpc_type: JSON_RPC - service_id: ollama endpoints: - publicly_exposed_url: http://relayminer2:8545 - rpc_type: rest + rpc_type: REST diff --git a/localnet/poktrolld/config/supplier3_stake_config.yaml b/localnet/poktrolld/config/supplier3_stake_config.yaml index 8aa675efb..8aca20dce 100644 --- a/localnet/poktrolld/config/supplier3_stake_config.yaml +++ b/localnet/poktrolld/config/supplier3_stake_config.yaml @@ -6,8 +6,8 @@ services: # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - publicly_exposed_url: http://relayminer3:8545 - rpc_type: json_rpc + rpc_type: JSON_RPC - service_id: ollama endpoints: - publicly_exposed_url: http://relayminer3:8545 - rpc_type: rest + rpc_type: REST diff --git a/x/supplier/config/supplier_configs_reader.go b/x/supplier/config/supplier_configs_reader.go index 535387397..0ed02b1f4 100644 --- a/x/supplier/config/supplier_configs_reader.go +++ b/x/supplier/config/supplier_configs_reader.go @@ -2,6 +2,7 @@ package config import ( "net/url" + "strings" sdk "github.com/cosmos/cosmos-sdk/types" "gopkg.in/yaml.v2" @@ -175,7 +176,7 @@ func parseEndpointConfigs(endpoint YAMLServiceEndpoint) ([]*sharedtypes.ConfigOp // parseEndpointRPCType parses the endpoint RPC type into a sharedtypes.RPCType func parseEndpointRPCType(endpoint YAMLServiceEndpoint) (sharedtypes.RPCType, error) { - switch endpoint.RPCType { + switch strings.ToLower(endpoint.RPCType) { case "json_rpc": return sharedtypes.RPCType_JSON_RPC, nil default: From 10cf5ef574e309c9a56219b4566d9aa847dd21d5 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 29 Apr 2024 13:07:48 -0700 Subject: [PATCH 4/6] Empty commit From d951aba097048150d578b2b17b4c938dd37f96e5 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 29 Apr 2024 13:09:20 -0700 Subject: [PATCH 5/6] Added TODO --- Tiltfile | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Tiltfile b/Tiltfile index cb56e4d91..5d8cca35b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -3,11 +3,12 @@ load("ext://helm_resource", "helm_resource", "helm_repo") load("ext://configmap", "configmap_create") load("ext://secret", "secret_create_generic") load("ext://deployment", "deployment_create") -load('ext://execute_in_pod', 'execute_in_pod') +load("ext://execute_in_pod", "execute_in_pod") # A list of directories where changes trigger a hot-reload of the validator hot_reload_dirs = ["app", "cmd", "tools", "x", "pkg"] + def merge_dicts(base, updates): for k, v in updates.items(): if k in base and type(base[k]) == "dict" and type(v) == "dict": @@ -18,6 +19,7 @@ def merge_dicts(base, updates): # Replace or set the value base[k] = v + # Create a localnet config file from defaults, and if a default configuration doesn't exist, populate it with default values localnet_config_path = "localnet_config.yaml" localnet_config_defaults = { @@ -30,18 +32,16 @@ localnet_config_defaults = { "delve": {"enabled": False}, }, "observability": {"enabled": True}, - "relayminers": { - "count": 1, - "delve": {"enabled": False} - }, + "relayminers": {"count": 1, "delve": {"enabled": False}}, "gateways": { "count": 1, - "delve": {"enabled": False}, + "delve": {"enabled": False}, }, "appgateservers": { "count": 1, - "delve": {"enabled": False}, + "delve": {"enabled": False}, }, + # TODO(#511): Add support for `REST` and enabled this. "ollama": { "enabled": False, "model": "qwen:0.5b", @@ -60,7 +60,9 @@ merge_dicts(localnet_config, localnet_config_defaults) # Then merge file contents over defaults merge_dicts(localnet_config, localnet_config_file) # Check if there are differences or if the file doesn't exist -if (localnet_config_file != localnet_config) or (not os.path.exists(localnet_config_path)): +if (localnet_config_file != localnet_config) or ( + not os.path.exists(localnet_config_path) +): print("Updating " + localnet_config_path + " with defaults") local("cat - > " + localnet_config_path, stdin=encode_yaml(localnet_config)) @@ -204,10 +206,8 @@ helm_resource( "--values=./localnet/kubernetes/values-validator.yaml", "--set=persistence.cleanupBeforeEachStart=" + str(localnet_config["validator"]["cleanupBeforeEachStart"]), - "--set=logs.level=" - + str(localnet_config["validator"]["logs"]["level"]), - "--set=logs.format=" - + str(localnet_config["validator"]["logs"]["format"]), + "--set=logs.level=" + str(localnet_config["validator"]["logs"]["level"]), + "--set=logs.format=" + str(localnet_config["validator"]["logs"]["format"]), "--set=serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), "--set=development.delve.enabled=" @@ -382,7 +382,9 @@ if localnet_config["ollama"]["enabled"]: ) local_resource( - name='ollama-pull-model', - cmd=execute_in_pod("ollama", "ollama pull "+localnet_config["ollama"]["model"]), - resource_deps=['ollama'], + name="ollama-pull-model", + cmd=execute_in_pod( + "ollama", "ollama pull " + localnet_config["ollama"]["model"] + ), + resource_deps=["ollama"], ) From ae1bcc1bf9d1e9970503f7a7cc7853f7d559632a Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 3 May 2024 14:54:46 -0700 Subject: [PATCH 6/6] Empty commit