Optimizations for running in a container based via compose/Dockerfile in repo roo

.dockerignore

@@ -0,0 +1,39 @@
+# manual ignores:
+**/.github
+**/dockerfile*
+
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/go/build-context-dockerignore/
+#
+
+**/.DS_Store
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose.y*ml
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

chipper/start.sh

@@ -25,6 +25,7 @@ if [[ ! -f ./source.sh ]]; then
     exit 0
 fi
 
+# sets env vars: (i.e. STT_SERVICE)
 source source.sh
 
 # set go tags
@@ -41,25 +42,25 @@ if [[ ${STT_SERVICE} == "leopard" ]]; then
     else
         /usr/local/go/bin/go run -tags $GOTAGS cmd/leopard/main.go
     fi
-    elif [[ ${STT_SERVICE} == "rhino" ]]; then
+elif [[ ${STT_SERVICE} == "rhino" ]]; then
     if [[ -f ./chipper ]]; then
         ./chipper
     else
         /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/rhino/main.go
     fi
-    elif [[ ${STT_SERVICE} == "houndify" ]]; then
+elif [[ ${STT_SERVICE} == "houndify" ]]; then
     if [[ -f ./chipper ]]; then
         ./chipper
     else
         /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/houndify/main.go
     fi
-    elif [[ ${STT_SERVICE} == "whisper" ]]; then
+elif [[ ${STT_SERVICE} == "whisper" ]]; then
     if [[ -f ./chipper ]]; then
         ./chipper
     else
         /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/whisper/main.go
     fi
-    elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
+elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
     if [[ -f ./chipper ]]; then
         export C_INCLUDE_PATH="../whisper.cpp"
         export LIBRARY_PATH="../whisper.cpp"
@@ -80,7 +81,7 @@ if [[ ${STT_SERVICE} == "leopard" ]]; then
             /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/whisper.cpp/main.go
         fi
     fi
-    elif [[ ${STT_SERVICE} == "vosk" ]]; then
+elif [[ ${STT_SERVICE} == "vosk" ]]; then
     if [[ -f ./chipper ]]; then
         export CGO_ENABLED=1
         export CGO_CFLAGS="-I/root/.vosk/libvosk"
@@ -92,7 +93,11 @@ if [[ ${STT_SERVICE} == "leopard" ]]; then
         export CGO_CFLAGS="-I/root/.vosk/libvosk"
         export CGO_LDFLAGS="-L /root/.vosk/libvosk -lvosk -ldl -lpthread"
         export LD_LIBRARY_PATH="/root/.vosk/libvosk:$LD_LIBRARY_PATH"
-        /usr/local/go/bin/go run -tags $GOTAGS -exec "env DYLD_LIBRARY_PATH=$HOME/.vosk/libvosk" cmd/vosk/main.go
+        if [[ ${DO_GO_BUILD} == "true" ]]; then
+            /usr/local/go/bin/go build -tags $GOTAGS -o chipper cmd/vosk/main.go
+        else
+            /usr/local/go/bin/go run -tags $GOTAGS -exec "env DYLD_LIBRARY_PATH=$HOME/.vosk/libvosk" cmd/vosk/main.go
+        fi
     fi
 else
     if [[ -f ./chipper ]]; then

compose.yaml

@@ -1,7 +1,9 @@
-version: '3.8'
 services:
   wire-pod:
     hostname: escapepod
+    build: .
+    network_mode: host # b/c mDNS escapepod.local broadcast... why use mDNS? people hosting a custom vector "server" are clearly capable of setting up a DNS record...
+    platform: linux/arm64
     image: ghcr.io/kercre123/wire-pod:main
     restart: unless-stopped
     ports:
@@ -10,10 +12,22 @@ services:
       - 80:80
       - 8084:8084
     volumes:
-      - wire-pod-data:/chipper/
+      # FYI after setup new server and join vector to it... diff shows only these changed files:
+      # $ docker container diff wire-pod-wire-pod-run-b2a5f3d4bf75
+      #   A /certs
+      #   A /certs/server_config.json
+      #   C /chipper
+      #   A /chipper/apiConfig.json    # stores config for first Submit Settings page (i.e. openai api key, weather api key, etc.)
+      #   C /chipper/jdocs
+      #   A /chipper/jdocs/botSdkInfo.json # stores vector IP, etc
+      #   A /chipper/jdocs/jdocs.json
+      #   A /vosk # vosk model extracted here.. would like to move this into image build too (very early on)
+      # TODO this is a problem to use a volume for /chipper in terms of updates, can we split out just the part that has the database we need to preserve and not cache the codebase? otherwise once created it will never update if image contents change...  admittedly the compose file here does not appear designed for the use case of updating at all (neither via a volume nor via the image...?) ... appears that update.sh uses git to pull latest code?! from this repo... so my dockerignore on .git is not gonna be so helpful then :)
+      #  /root/.anki_vector/* ??? is this the path? if so make a volume for this
+      # - wire-pod-data:/chipper/
       - wire-pod-model:/vosk/
 volumes:
-  wire-pod-data:
-    driver: local
+  # wire-pod-data:
+  #   driver: local
   wire-pod-model:
     driver: local

dockerfile

@@ -1,12 +1,45 @@
 FROM ubuntu
 
 
-COPY . .
+# *** PACKAGE INSTALLS ***
+# install packages before copying in source so we don't do this on every source change
+RUN apt-get update && \
+  apt-get install -y dos2unix avahi-daemon avahi-autoipd
+#
+# setup.sh is standalone (IIUC upon cursory inspection) at least for debian/aarch64 + vosk purposes
+# setup.sh is more or less further `apt install...` + golang install + vosk install + gen chipper/source.sh... thus run this as early as possible as it is not likely to change (as much) as the source code
+COPY setup.sh /setup.sh
+# PRN part of what setup.sh does is to install golang and other deps... why not extract those deps here for the Dockerfile and not use setup.sh inside the container build? AND why not find a golang base image to build on top of instead of installing it here?
+RUN dos2unix /setup.sh && mkdir /chipper && mkdir /vector-cloud
+RUN ["/bin/sh", "-c", "STT=vosk IMAGE_BUILD=true SETUP_STAGE=getPackages ./setup.sh"]
 
-RUN chmod +x /setup.sh && apt-get update && apt-get install -y dos2unix && dos2unix /setup.sh && apt-get install -y avahi-daemon avahi-autoipd
+# so we can install go deps (IIUC for vosk install in setup.sh/getSTT)
+COPY ./chipper/go.sum ./chipper/go.mod /chipper/
+# FYI setup.sh generates /chipper/source.sh which sets env vars only
+RUN ["/bin/sh", "-c", "STT=vosk IMAGE_BUILD=true SETUP_STAGE=getSTT ./setup.sh"]
+# *** END PACKAGE INSTALLS ***
 
-RUN ["/bin/sh", "-c", "STT=vosk ./setup.sh"]
+# *** go download deps ***
+#
+# PRN build + cache deps:  # RUN --mount=type=cache,target=/root/go/pkg/mod cd /chipper && go mod download
+#
+COPY ./vector-cloud/go.sum ./vector-cloud/go.mod /vector-cloud/
+RUN cd /chipper && go mod download
+# vector-cloud deps are normally downloaded after initial web server Submit Settings page:
+RUN cd /vector-cloud && go mod download
+# PRN IIUC vector-cloud has some vosk model download/setup, can I add it to this image build too?
+#     Downloading https://github.com/kercre123/vosk-models/raw/main/vosk-model-small-en-us-0.15.zip to /tmp/vosk-model-small-en-us-0.15.zip
+# *** END go download deps ***
 
-RUN chmod +x /chipper/start.sh && dos2unix /chipper/start.sh
+# PRN copy specific files only: # COPY chipper images vector-cloud /
+COPY . .
+RUN dos2unix /chipper/start.sh
+# TODO do we really need dos2unix? can't we use editorconfig or something else to enforce line endings? and/or force git checkout to have LF endings always? SAME with setup.sh above too
+#
+# build chipper during image build:
+#   reuse start.sh so we keep logic for changing env vars (i.e. for whipser instead of vosk)
+RUN ["/bin/sh", "-c", "DO_GO_BUILD=true ./chipper/start.sh"]
+# PRN build vector-cloud during image build too?
+# PRN use multi-stage build - with builder image (w/ cached modules) separate of runtime image (final executables only)?
 
 CMD ["/bin/sh", "-c", "./chipper/start.sh"]

setup.sh

@@ -26,13 +26,13 @@ if [[ ${UNAME} == *"Darwin"* ]]; then
         echo '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'
         exit 1
     fi
-    elif [[ -f /usr/bin/apt ]]; then
+elif [[ -f /usr/bin/apt ]]; then
     TARGET="debian"
     echo "Debian-based Linux detected."
-    elif [[ -f /usr/bin/pacman ]]; then
+elif [[ -f /usr/bin/pacman ]]; then
     TARGET="arch"
     echo "Arch Linux detected."
-    elif [[ -f /usr/bin/dnf ]]; then
+elif [[ -f /usr/bin/dnf ]]; then
     TARGET="fedora"
     echo "Fedora/openSUSE detected."
 else
@@ -47,10 +47,10 @@ fi
 if [[ "${UNAME}" == *"x86_64"* ]]; then
     ARCH="x86_64"
     echo "amd64 architecture confirmed."
-    elif [[ "${UNAME}" == *"aarch64"* ]] || [[ "${UNAME}" == *"arm64"* ]]; then
+elif [[ "${UNAME}" == *"aarch64"* ]] || [[ "${UNAME}" == *"arm64"* ]]; then
     ARCH="aarch64"
     echo "aarch64 architecture confirmed."
-    elif [[ "${UNAME}" == *"armv7l"* ]]; then
+elif [[ "${UNAME}" == *"armv7l"* ]]; then
     ARCH="armv7l"
     echo "armv7l (32-bit) WARN: The Coqui and VOSK bindings are broken for this platform at the moment, so please choose Picovoice when the script asks. wire-pod is designed for 64-bit systems."
     STT=""
@@ -91,13 +91,13 @@ function getPackages() {
     if [[ ${TARGET} == "debian" ]]; then
         apt update -y
         apt install -y wget openssl net-tools libsox-dev libopus-dev make iproute2 xz-utils libopusfile-dev pkg-config gcc curl g++ unzip avahi-daemon git libasound2-dev libsodium-dev
-        elif [[ ${TARGET} == "arch" ]]; then
+    elif [[ ${TARGET} == "arch" ]]; then
         pacman -Sy --noconfirm
         sudo pacman -S --noconfirm wget openssl net-tools sox opus make iproute2 opusfile curl unzip avahi git libsodium go pkg-config
-        elif [[ ${TARGET} == "fedora" ]]; then
+    elif [[ ${TARGET} == "fedora" ]]; then
         dnf update
         dnf install -y wget openssl net-tools sox opus make opusfile curl unzip avahi git libsodium-devel
-        elif [[ ${TARGET} == "darwin" ]]; then
+    elif [[ ${TARGET} == "darwin" ]]; then
         sudo -u $SUDO_USER brew update
         sudo -u $SUDO_USER brew install wget pkg-config opus opusfile
     fi
@@ -111,10 +111,10 @@ function getPackages() {
             if [[ ${ARCH} == "x86_64" ]]; then
                 wget -q --show-progress --no-check-certificate https://go.dev/dl/go1.19.4.linux-amd64.tar.gz
                 rm -rf /usr/local/go && tar -C /usr/local -xzf go1.19.4.linux-amd64.tar.gz
-                elif [[ ${ARCH} == "aarch64" ]]; then
+            elif [[ ${ARCH} == "aarch64" ]]; then
                 wget -q --show-progress --no-check-certificate https://go.dev/dl/go1.19.4.linux-arm64.tar.gz
                 rm -rf /usr/local/go && tar -C /usr/local -xzf go1.19.4.linux-arm64.tar.gz
-                elif [[ ${ARCH} == "armv7l" ]]; then
+            elif [[ ${ARCH} == "armv7l" ]]; then
                 wget -q --show-progress --no-check-certificate https://go.dev/dl/go1.19.4.linux-armv6l.tar.gz
                 rm -rf /usr/local/go && tar -C /usr/local -xzf go1.19.4.linux-armv6l.tar.gz
             fi
@@ -146,18 +146,18 @@ function getSTT() {
         read -p "Enter a number (3): " sttServiceNum
         if [[ ! -n ${sttServiceNum} ]]; then
             sttService="vosk"
-            elif [[ ${sttServiceNum} == "1" ]]; then
+        elif [[ ${sttServiceNum} == "1" ]]; then
             if [[ ${TARGET} == "darwin" ]]; then
                 echo "Coqui is not supported for macOS. Please select another option."
                 sttServicePrompt
             else
                 sttService="coqui"
             fi
-            elif [[ ${sttServiceNum} == "2" ]]; then
+        elif [[ ${sttServiceNum} == "2" ]]; then
             sttService="leopard"
-            elif [[ ${sttServiceNum} == "3" ]]; then
+        elif [[ ${sttServiceNum} == "3" ]]; then
             sttService="vosk"
-            elif [[ ${sttServiceNum} == "4" ]]; then
+        elif [[ ${sttServiceNum} == "4" ]]; then
             sttService="whisper"
         else
             echo
@@ -187,7 +187,7 @@ function getSTT() {
         echo "export STT_SERVICE=leopard" >> ./chipper/source.sh
         echo "export PICOVOICE_APIKEY=${picoKey}" >> ./chipper/source.sh
         echo "export PICOVOICE_APIKEY=${picoKey}" > ./chipper/pico.key
-        elif [[ ${sttService} == "vosk" ]]; then
+    elif [[ ${sttService} == "vosk" ]]; then
         echo "export STT_SERVICE=vosk" >> ./chipper/source.sh
         origDir="$(pwd)"
         if [[ ! -f ./vosk/completed ]]; then
@@ -199,11 +199,11 @@ function getSTT() {
             if [[ ${TARGET} == "darwin" ]]; then
                 VOSK_VER="0.3.42"
                 VOSK_DIR="vosk-osx-${VOSK_VER}"
-                elif [[ ${ARCH} == "x86_64" ]]; then
+            elif [[ ${ARCH} == "x86_64" ]]; then
                 VOSK_DIR="vosk-linux-x86_64-${VOSK_VER}"
-                elif [[ ${ARCH} == "aarch64" ]]; then
+            elif [[ ${ARCH} == "aarch64" ]]; then
                 VOSK_DIR="vosk-linux-aarch64-${VOSK_VER}"
-                elif [[ ${ARCH} == "armv7l" ]]; then
+            elif [[ ${ARCH} == "armv7l" ]]; then
                 VOSK_DIR="vosk-linux-armv7l-${VOSK_VER}"
             fi
             VOSK_ARCHIVE="$VOSK_DIR.zip"
@@ -222,7 +222,7 @@ function getSTT() {
             /usr/local/go/bin/go install github.com/kercre123/vosk-api/go
             cd ${origDir}
         fi
-        elif [[ ${sttService} == "whisper" ]]; then
+    elif [[ ${sttService} == "whisper" ]]; then
         echo "export STT_SERVICE=whisper.cpp" >> ./chipper/source.sh
         origDir="$(pwd)"
         echo "Getting Whisper assets"
@@ -276,11 +276,11 @@ function getSTT() {
                 fi
                 tar -xf native_client.tflite.Linux.tar.xz
                 rm -f ./native_client.tflite.Linux.tar.xz
-                elif [[ ${ARCH} == "aarch64" ]]; then
+            elif [[ ${ARCH} == "aarch64" ]]; then
                 wget -q --show-progress --no-check-certificate https://github.com/coqui-ai/STT/releases/download/v1.3.0/native_client.tflite.linux.aarch64.tar.xz
                 tar -xf native_client.tflite.linux.aarch64.tar.xz
                 rm -f ./native_client.tflite.linux.aarch64.tar.xz
-                elif [[ ${ARCH} == "armv7l" ]]; then
+            elif [[ ${ARCH} == "armv7l" ]]; then
                 wget -q --show-progress --no-check-certificate https://github.com/coqui-ai/STT/releases/download/v1.3.0/native_client.tflite.linux.armv7.tar.xz
                 tar -xf native_client.tflite.linux.armv7.tar.xz
                 rm -f ./native_client.tflite.linux.armv7.tar.xz
@@ -304,9 +304,9 @@ function getSTT() {
                 read -p "Enter a number (1): " sttModelNum
                 if [[ ! -n ${sttModelNum} ]]; then
                     sttModel="large_vocabulary"
-                    elif [[ ${sttModelNum} == "1" ]]; then
+                elif [[ ${sttModelNum} == "1" ]]; then
                     sttModel="large_vocabulary"
-                    elif [[ ${sttModelNum} == "2" ]]; then
+                elif [[ ${sttModelNum} == "2" ]]; then
                     sttModel="huge_vocabulary"
                 else
                     echo
@@ -323,7 +323,7 @@ function getSTT() {
                 wget -O model.tflite -q --show-progress --no-check-certificate https://coqui.gateway.scarf.sh/english/coqui/v1.0.0-large-vocab/model.tflite
                 echo "Getting STT scorer..."
                 wget -O model.scorer -q --show-progress --no-check-certificate https://coqui.gateway.scarf.sh/english/coqui/v1.0.0-large-vocab/large_vocabulary.scorer
-                elif [[ ${sttModel} == "huge_vocabulary" ]]; then
+            elif [[ ${sttModel} == "huge_vocabulary" ]]; then
                 echo "Getting STT model..."
                 wget -O model.tflite -q --show-progress --no-check-certificate https://coqui.gateway.scarf.sh/english/coqui/v1.0.0-huge-vocab/model.tflite
                 echo "Getting STT scorer..."
@@ -565,14 +565,14 @@ function setupSystemd() {
     if [[ ${STT_SERVICE} == "leopard" ]]; then
         echo "wire-pod.service created, building chipper with Picovoice STT service..."
         /usr/local/go/bin/go build -tags $GOTAGS cmd/leopard/main.go
-        elif [[ ${STT_SERVICE} == "vosk" ]]; then
+    elif [[ ${STT_SERVICE} == "vosk" ]]; then
         echo "wire-pod.service created, building chipper with VOSK STT service..."
         export CGO_ENABLED=1
         export CGO_CFLAGS="-I/root/.vosk/libvosk"
         export CGO_LDFLAGS="-L /root/.vosk/libvosk -lvosk -ldl -lpthread"
         export LD_LIBRARY_PATH="/root/.vosk/libvosk:$LD_LIBRARY_PATH"
         /usr/local/go/bin/go build -tags $GOTAGS cmd/vosk/main.go
-        elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
+    elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
         echo "wire-pod.service created, building chipper with Whisper.CPP STT service..."
         export CGO_ENABLED=1
         export C_INCLUDE_PATH="../whisper.cpp"
@@ -651,6 +651,22 @@ if [[ $1 == "-f" ]] && [[ $2 == "scp" ]]; then
     exit 0
 fi
 
+if [[ -n "${IMAGE_BUILD}" ]]; then
+    echo "Detected container build env..."
+    if [[ "${SETUP_STAGE}" == "getPackages" ]]; then
+        getPackages
+        exit 0
+    fi
+    # TODO add stage to install golang too and move it to first stage? or use base image with golang already installed
+    if [[ "${SETUP_STAGE}" == "getSTT" ]]; then
+        getSTT
+        echo "wire-pod is ready to run! You are ready to move to the next step and run sudo ./chipper/start.sh"
+        exit 0
+    fi
+    echo "Please provide a valid SETUP_STAGE env var, the current value is '${SETUP_STAGE}' is not recognized"
+    exit 1
+fi
+
 # echo "What would you like to do?"
 # echo "1: Full Setup (recommended) (builds chipper, gets STT stuff, generates certs, creates source.sh file, and creates server_config.json for your bot"
 # echo "2: Just build vic-cloud"