Optimizations for running in a container based via compose/Dockerfile in repo roo

.dockerignore

@@ -0,0 +1,39 @@
+# manual ignores:
+**/.github
+**/dockerfile*
+
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/go/build-context-dockerignore/
+#
+
+**/.DS_Store
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose.y*ml
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

chipper/start.sh

@@ -25,6 +25,7 @@ if [[ ! -f ./source.sh ]]; then
  exit 0
 fi
 
+# sets env vars: (i.e. STT_SERVICE)
 source source.sh
 
 # set go tags
@@ -41,25 +42,25 @@ if [[ ${STT_SERVICE} == "leopard" ]]; then
  else
  /usr/local/go/bin/go run -tags $GOTAGS cmd/leopard/main.go
  fi
- elif [[ ${STT_SERVICE} == "rhino" ]]; then
+elif [[ ${STT_SERVICE} == "rhino" ]]; then
  if [[ -f ./chipper ]]; then
  ./chipper
  else
  /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/rhino/main.go
  fi
- elif [[ ${STT_SERVICE} == "houndify" ]]; then
+elif [[ ${STT_SERVICE} == "houndify" ]]; then
  if [[ -f ./chipper ]]; then
  ./chipper
  else
  /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/houndify/main.go
  fi
- elif [[ ${STT_SERVICE} == "whisper" ]]; then
+elif [[ ${STT_SERVICE} == "whisper" ]]; then
  if [[ -f ./chipper ]]; then
  ./chipper
  else
  /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/whisper/main.go
  fi
- elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
+elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
  if [[ -f ./chipper ]]; then
  export C_INCLUDE_PATH="../whisper.cpp"
  export LIBRARY_PATH="../whisper.cpp"
@@ -80,7 +81,7 @@ if [[ ${STT_SERVICE} == "leopard" ]]; then
  /usr/local/go/bin/go run -tags $GOTAGS cmd/experimental/whisper.cpp/main.go
  fi
  fi
- elif [[ ${STT_SERVICE} == "vosk" ]]; then
+elif [[ ${STT_SERVICE} == "vosk" ]]; then
  if [[ -f ./chipper ]]; then
  export CGO_ENABLED=1
  export CGO_CFLAGS="-I/root/.vosk/libvosk"
@@ -92,7 +93,11 @@ if [[ ${STT_SERVICE} == "leopard" ]]; then
  export CGO_CFLAGS="-I/root/.vosk/libvosk"
  export CGO_LDFLAGS="-L /root/.vosk/libvosk -lvosk -ldl -lpthread"
  export LD_LIBRARY_PATH="/root/.vosk/libvosk:$LD_LIBRARY_PATH"
- /usr/local/go/bin/go run -tags $GOTAGS -exec "env DYLD_LIBRARY_PATH=$HOME/.vosk/libvosk" cmd/vosk/main.go
+ if [[ ${DO_GO_BUILD} == "true" ]]; then
+ /usr/local/go/bin/go build -tags $GOTAGS -o chipper cmd/vosk/main.go
+ else
+ /usr/local/go/bin/go run -tags $GOTAGS -exec "env DYLD_LIBRARY_PATH=$HOME/.vosk/libvosk" cmd/vosk/main.go
+ fi
  fi
 else
  if [[ -f ./chipper ]]; then

compose.yaml

@@ -1,7 +1,9 @@
-version: '3.8'
 services:
  wire-pod:
  hostname: escapepod
+ build: .
+ network_mode: host # b/c mDNS escapepod.local broadcast... why use mDNS? people hosting a custom vector "server" are clearly capable of setting up a DNS record...
+ platform: linux/arm64
  image: ghcr.io/kercre123/wire-pod:main
  restart: unless-stopped
  ports:
@@ -10,10 +12,22 @@ services:
  - 80:80
  - 8084:8084
  volumes:
- - wire-pod-data:/chipper/
+ # FYI after setup new server and join vector to it... diff shows only these changed files:
+ # $ docker container diff wire-pod-wire-pod-run-b2a5f3d4bf75
+ # A /certs
+ # A /certs/server_config.json
+ # C /chipper
+ # A /chipper/apiConfig.json # stores config for first Submit Settings page (i.e. openai api key, weather api key, etc.)
+ # C /chipper/jdocs
+ # A /chipper/jdocs/botSdkInfo.json # stores vector IP, etc
+ # A /chipper/jdocs/jdocs.json
+ # A /vosk # vosk model extracted here.. would like to move this into image build too (very early on)
+ # TODO this is a problem to use a volume for /chipper in terms of updates, can we split out just the part that has the database we need to preserve and not cache the codebase? otherwise once created it will never update if image contents change... admittedly the compose file here does not appear designed for the use case of updating at all (neither via a volume nor via the image...?) ... appears that update.sh uses git to pull latest code?! from this repo... so my dockerignore on .git is not gonna be so helpful then :)
+ # /root/.anki_vector/* ??? is this the path? if so make a volume for this
+ # - wire-pod-data:/chipper/
  - wire-pod-model:/vosk/
 volumes:
- wire-pod-data:
- driver: local
+ # wire-pod-data:
+ #  driver: local
  wire-pod-model:
  driver: local

dockerfile

@@ -1,12 +1,45 @@
 FROM ubuntu
 
 
-COPY . .
+# *** PACKAGE INSTALLS ***
+# install packages before copying in source so we don't do this on every source change
+RUN apt-get update && \
+ apt-get install -y dos2unix avahi-daemon avahi-autoipd
+#
+# setup.sh is standalone (IIUC upon cursory inspection) at least for debian/aarch64 + vosk purposes
+# setup.sh is more or less further `apt install...` + golang install + vosk install + gen chipper/source.sh... thus run this as early as possible as it is not likely to change (as much) as the source code
+COPY setup.sh /setup.sh
+# PRN part of what setup.sh does is to install golang and other deps... why not extract those deps here for the Dockerfile and not use setup.sh inside the container build? AND why not find a golang base image to build on top of instead of installing it here?
+RUN dos2unix /setup.sh && mkdir /chipper && mkdir /vector-cloud
+RUN ["/bin/sh", "-c", "STT=vosk IMAGE_BUILD=true SETUP_STAGE=getPackages ./setup.sh"]
 
-RUN chmod +x /setup.sh && apt-get update && apt-get install -y dos2unix && dos2unix /setup.sh && apt-get install -y avahi-daemon avahi-autoipd
+# so we can install go deps (IIUC for vosk install in setup.sh/getSTT)
+COPY ./chipper/go.sum ./chipper/go.mod /chipper/
+# FYI setup.sh generates /chipper/source.sh which sets env vars only
+RUN ["/bin/sh", "-c", "STT=vosk IMAGE_BUILD=true SETUP_STAGE=getSTT ./setup.sh"]
+# *** END PACKAGE INSTALLS ***
 
-RUN ["/bin/sh", "-c", "STT=vosk ./setup.sh"]
+# *** go download deps ***
+#
+# PRN build + cache deps: # RUN --mount=type=cache,target=/root/go/pkg/mod cd /chipper && go mod download
+#
+COPY ./vector-cloud/go.sum ./vector-cloud/go.mod /vector-cloud/
+RUN cd /chipper && go mod download
+# vector-cloud deps are normally downloaded after initial web server Submit Settings page:
+RUN cd /vector-cloud && go mod download
+# PRN IIUC vector-cloud has some vosk model download/setup, can I add it to this image build too?
+# Downloading https://github.com/kercre123/vosk-models/raw/main/vosk-model-small-en-us-0.15.zip to /tmp/vosk-model-small-en-us-0.15.zip
+# *** END go download deps ***
 
-RUN chmod +x /chipper/start.sh && dos2unix /chipper/start.sh
+# PRN copy specific files only: # COPY chipper images vector-cloud /
+COPY . .
+RUN dos2unix /chipper/start.sh
+# TODO do we really need dos2unix? can't we use editorconfig or something else to enforce line endings? and/or force git checkout to have LF endings always? SAME with setup.sh above too
+#
+# build chipper during image build:
+# reuse start.sh so we keep logic for changing env vars (i.e. for whipser instead of vosk)
+RUN ["/bin/sh", "-c", "DO_GO_BUILD=true ./chipper/start.sh"]
+# PRN build vector-cloud during image build too?
+# PRN use multi-stage build - with builder image (w/ cached modules) separate of runtime image (final executables only)?
 
 CMD ["/bin/sh", "-c", "./chipper/start.sh"]

setup.sh

@@ -26,13 +26,13 @@ if [[ ${UNAME} == *"Darwin"* ]]; then
  echo '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'
  exit 1
  fi
- elif [[ -f /usr/bin/apt ]]; then
+elif [[ -f /usr/bin/apt ]]; then
  TARGET="debian"
  echo "Debian-based Linux detected."
- elif [[ -f /usr/bin/pacman ]]; then
+elif [[ -f /usr/bin/pacman ]]; then
  TARGET="arch"
  echo "Arch Linux detected."
- elif [[ -f /usr/bin/dnf ]]; then
+elif [[ -f /usr/bin/dnf ]]; then
  TARGET="fedora"
  echo "Fedora/openSUSE detected."
 else
@@ -47,10 +47,10 @@ fi
 if [[ "${UNAME}" == *"x86_64"* ]]; then
  ARCH="x86_64"
  echo "amd64 architecture confirmed."
- elif [[ "${UNAME}" == *"aarch64"* ]] || [[ "${UNAME}" == *"arm64"* ]]; then
+elif [[ "${UNAME}" == *"aarch64"* ]] || [[ "${UNAME}" == *"arm64"* ]]; then
  ARCH="aarch64"
  echo "aarch64 architecture confirmed."
- elif [[ "${UNAME}" == *"armv7l"* ]]; then
+elif [[ "${UNAME}" == *"armv7l"* ]]; then
  ARCH="armv7l"
  echo "armv7l (32-bit) WARN: The Coqui and VOSK bindings are broken for this platform at the moment, so please choose Picovoice when the script asks. wire-pod is designed for 64-bit systems."
  STT=""
@@ -91,13 +91,13 @@ function getPackages() {
  if [[ ${TARGET} == "debian" ]]; then
  apt update -y
  apt install -y wget openssl net-tools libsox-dev libopus-dev make iproute2 xz-utils libopusfile-dev pkg-config gcc curl g++ unzip avahi-daemon git libasound2-dev libsodium-dev
-  elif [[ ${TARGET} == "arch" ]]; then
+ elif [[ ${TARGET} == "arch" ]]; then
  pacman -Sy --noconfirm
  sudo pacman -S --noconfirm wget openssl net-tools sox opus make iproute2 opusfile curl unzip avahi git libsodium go pkg-config
-  elif [[ ${TARGET} == "fedora" ]]; then
+ elif [[ ${TARGET} == "fedora" ]]; then
  dnf update
  dnf install -y wget openssl net-tools sox opus make opusfile curl unzip avahi git libsodium-devel
-  elif [[ ${TARGET} == "darwin" ]]; then
+ elif [[ ${TARGET} == "darwin" ]]; then
  sudo -u $SUDO_USER brew update
  sudo -u $SUDO_USER brew install wget pkg-config opus opusfile
  fi
@@ -111,10 +111,10 @@ function getPackages() {
  if [[ ${ARCH} == "x86_64" ]]; then
  wget -q --show-progress --no-check-certificate https://go.dev/dl/go1.19.4.linux-amd64.tar.gz
  rm -rf /usr/local/go && tar -C /usr/local -xzf go1.19.4.linux-amd64.tar.gz
-  elif [[ ${ARCH} == "aarch64" ]]; then
+ elif [[ ${ARCH} == "aarch64" ]]; then
  wget -q --show-progress --no-check-certificate https://go.dev/dl/go1.19.4.linux-arm64.tar.gz
  rm -rf /usr/local/go && tar -C /usr/local -xzf go1.19.4.linux-arm64.tar.gz
-  elif [[ ${ARCH} == "armv7l" ]]; then
+ elif [[ ${ARCH} == "armv7l" ]]; then
  wget -q --show-progress --no-check-certificate https://go.dev/dl/go1.19.4.linux-armv6l.tar.gz
  rm -rf /usr/local/go && tar -C /usr/local -xzf go1.19.4.linux-armv6l.tar.gz
  fi
@@ -146,18 +146,18 @@ function getSTT() {
  read -p "Enter a number (3): " sttServiceNum
  if [[ ! -n ${sttServiceNum} ]]; then
  sttService="vosk"
-  elif [[ ${sttServiceNum} == "1" ]]; then
+ elif [[ ${sttServiceNum} == "1" ]]; then
  if [[ ${TARGET} == "darwin" ]]; then
  echo "Coqui is not supported for macOS. Please select another option."
  sttServicePrompt
  else
  sttService="coqui"
  fi
-  elif [[ ${sttServiceNum} == "2" ]]; then
+ elif [[ ${sttServiceNum} == "2" ]]; then
  sttService="leopard"
-  elif [[ ${sttServiceNum} == "3" ]]; then
+ elif [[ ${sttServiceNum} == "3" ]]; then
  sttService="vosk"
-  elif [[ ${sttServiceNum} == "4" ]]; then
+ elif [[ ${sttServiceNum} == "4" ]]; then
  sttService="whisper"
  else
  echo
@@ -187,7 +187,7 @@ function getSTT() {
  echo "export STT_SERVICE=leopard" >> ./chipper/source.sh
  echo "export PICOVOICE_APIKEY=${picoKey}" >> ./chipper/source.sh
  echo "export PICOVOICE_APIKEY=${picoKey}" > ./chipper/pico.key
-  elif [[ ${sttService} == "vosk" ]]; then
+ elif [[ ${sttService} == "vosk" ]]; then
  echo "export STT_SERVICE=vosk" >> ./chipper/source.sh
  origDir="$(pwd)"
  if [[ ! -f ./vosk/completed ]]; then
@@ -199,11 +199,11 @@ function getSTT() {
  if [[ ${TARGET} == "darwin" ]]; then
  VOSK_VER="0.3.42"
  VOSK_DIR="vosk-osx-${VOSK_VER}"
-  elif [[ ${ARCH} == "x86_64" ]]; then
+ elif [[ ${ARCH} == "x86_64" ]]; then
  VOSK_DIR="vosk-linux-x86_64-${VOSK_VER}"
-  elif [[ ${ARCH} == "aarch64" ]]; then
+ elif [[ ${ARCH} == "aarch64" ]]; then
  VOSK_DIR="vosk-linux-aarch64-${VOSK_VER}"
-  elif [[ ${ARCH} == "armv7l" ]]; then
+ elif [[ ${ARCH} == "armv7l" ]]; then
  VOSK_DIR="vosk-linux-armv7l-${VOSK_VER}"
  fi
  VOSK_ARCHIVE="$VOSK_DIR.zip"
@@ -222,7 +222,7 @@ function getSTT() {
  /usr/local/go/bin/go install github.com/kercre123/vosk-api/go
  cd ${origDir}
  fi
-  elif [[ ${sttService} == "whisper" ]]; then
+ elif [[ ${sttService} == "whisper" ]]; then
  echo "export STT_SERVICE=whisper.cpp" >> ./chipper/source.sh
  origDir="$(pwd)"
  echo "Getting Whisper assets"
@@ -276,11 +276,11 @@ function getSTT() {
  fi
  tar -xf native_client.tflite.Linux.tar.xz
  rm -f ./native_client.tflite.Linux.tar.xz
-  elif [[ ${ARCH} == "aarch64" ]]; then
+ elif [[ ${ARCH} == "aarch64" ]]; then
  wget -q --show-progress --no-check-certificate https://github.com/coqui-ai/STT/releases/download/v1.3.0/native_client.tflite.linux.aarch64.tar.xz
  tar -xf native_client.tflite.linux.aarch64.tar.xz
  rm -f ./native_client.tflite.linux.aarch64.tar.xz
-  elif [[ ${ARCH} == "armv7l" ]]; then
+ elif [[ ${ARCH} == "armv7l" ]]; then
  wget -q --show-progress --no-check-certificate https://github.com/coqui-ai/STT/releases/download/v1.3.0/native_client.tflite.linux.armv7.tar.xz
  tar -xf native_client.tflite.linux.armv7.tar.xz
  rm -f ./native_client.tflite.linux.armv7.tar.xz
@@ -304,9 +304,9 @@ function getSTT() {
  read -p "Enter a number (1): " sttModelNum
  if [[ ! -n ${sttModelNum} ]]; then
  sttModel="large_vocabulary"
-  elif [[ ${sttModelNum} == "1" ]]; then
+ elif [[ ${sttModelNum} == "1" ]]; then
  sttModel="large_vocabulary"
-  elif [[ ${sttModelNum} == "2" ]]; then
+ elif [[ ${sttModelNum} == "2" ]]; then
  sttModel="huge_vocabulary"
  else
  echo
@@ -323,7 +323,7 @@ function getSTT() {
  wget -O model.tflite -q --show-progress --no-check-certificate https://coqui.gateway.scarf.sh/english/coqui/v1.0.0-large-vocab/model.tflite
  echo "Getting STT scorer..."
  wget -O model.scorer -q --show-progress --no-check-certificate https://coqui.gateway.scarf.sh/english/coqui/v1.0.0-large-vocab/large_vocabulary.scorer
-  elif [[ ${sttModel} == "huge_vocabulary" ]]; then
+ elif [[ ${sttModel} == "huge_vocabulary" ]]; then
  echo "Getting STT model..."
  wget -O model.tflite -q --show-progress --no-check-certificate https://coqui.gateway.scarf.sh/english/coqui/v1.0.0-huge-vocab/model.tflite
  echo "Getting STT scorer..."
@@ -565,14 +565,14 @@ function setupSystemd() {
  if [[ ${STT_SERVICE} == "leopard" ]]; then
  echo "wire-pod.service created, building chipper with Picovoice STT service..."
  /usr/local/go/bin/go build -tags $GOTAGS cmd/leopard/main.go
-  elif [[ ${STT_SERVICE} == "vosk" ]]; then
+ elif [[ ${STT_SERVICE} == "vosk" ]]; then
  echo "wire-pod.service created, building chipper with VOSK STT service..."
  export CGO_ENABLED=1
  export CGO_CFLAGS="-I/root/.vosk/libvosk"
  export CGO_LDFLAGS="-L /root/.vosk/libvosk -lvosk -ldl -lpthread"
  export LD_LIBRARY_PATH="/root/.vosk/libvosk:$LD_LIBRARY_PATH"
  /usr/local/go/bin/go build -tags $GOTAGS cmd/vosk/main.go
-  elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
+ elif [[ ${STT_SERVICE} == "whisper.cpp" ]]; then
  echo "wire-pod.service created, building chipper with Whisper.CPP STT service..."
  export CGO_ENABLED=1
  export C_INCLUDE_PATH="../whisper.cpp"
@@ -651,6 +651,22 @@ if [[ $1 == "-f" ]] && [[ $2 == "scp" ]]; then
  exit 0
 fi
 
+if [[ -n "${IMAGE_BUILD}" ]]; then
+ echo "Detected container build env..."
+ if [[ "${SETUP_STAGE}" == "getPackages" ]]; then
+ getPackages
+ exit 0
+ fi
+ # TODO add stage to install golang too and move it to first stage? or use base image with golang already installed
+ if [[ "${SETUP_STAGE}" == "getSTT" ]]; then
+ getSTT
+ echo "wire-pod is ready to run! You are ready to move to the next step and run sudo ./chipper/start.sh"
+ exit 0
+ fi
+ echo "Please provide a valid SETUP_STAGE env var, the current value is '${SETUP_STAGE}' is not recognized"
+ exit 1
+fi
+
 # echo "What would you like to do?"
 # echo "1: Full Setup (recommended) (builds chipper, gets STT stuff, generates certs, creates source.sh file, and creates server_config.json for your bot"
 # echo "2: Just build vic-cloud"