From 597d9d77403ce4ef743f1e6e1b615ca9b84f2836 Mon Sep 17 00:00:00 2001 From: Kevin Birk Date: Thu, 13 Jun 2024 14:37:37 -0400 Subject: [PATCH] Run xxx-taskrunner develop containerrs with hotswapping (#3845) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../scripts/docker-compose-taskrunner.yml | 127 +++++++++++------- packages/funman/.gitignore | 40 ++++++ packages/funman/Dockerfile | 61 ++++++--- packages/funman/dev.sh | 11 ++ packages/gollm/.gitignore | 40 ++++++ packages/gollm/Dockerfile | 60 ++++++--- packages/gollm/dev.sh | 11 ++ packages/mira/.gitignore | 40 ++++++ packages/mira/Dockerfile | 58 +++++--- packages/mira/dev.sh | 11 ++ packages/taskrunner/.gitignore | 1 + packages/taskrunner/Dockerfile | 8 ++ packages/taskrunner/README.md | 32 +---- 13 files changed, 363 insertions(+), 137 deletions(-) create mode 100644 packages/funman/.gitignore create mode 100755 packages/funman/dev.sh create mode 100644 packages/gollm/.gitignore create mode 100755 packages/gollm/dev.sh create mode 100644 packages/mira/.gitignore create mode 100755 packages/mira/dev.sh create mode 100644 packages/taskrunner/Dockerfile diff --git a/containers/scripts/docker-compose-taskrunner.yml b/containers/scripts/docker-compose-taskrunner.yml index 178ca44c24..8f46f27866 100644 --- a/containers/scripts/docker-compose-taskrunner.yml +++ b/containers/scripts/docker-compose-taskrunner.yml @@ -1,50 +1,85 @@ --- services: - gollm-taskrunner: - container_name: gollm-taskrunner - image: ghcr.io/darpa-askem/gollm-taskrunner:latest - networks: - - terarium - environment: - TERARIUM_MQ-ADDRESSES: "amqp://rabbitmq:5672" - TERARIUM_MQ-PASSWORD: "terarium123" - TERARIUM_MQ-USERNAME: "terarium" - TERARIUM_TASKRUNNER_REQUEST-TYPE: "gollm" - OPENAI_API_KEY: "${secret_gollm_openai_key}" - depends_on: - rabbitmq: - condition: service_healthy - extra_hosts: - - "${local_host_name}:host-gateway" + taskrunner-dev-builder: + build: + dockerfile: ../../packages/taskrunner/Dockerfile + target: taskrunner_dev_builder + container_name: taskrunner-dev-builder + volumes: + - ../../packages/taskrunner:/taskrunner - mira-taskrunner: - container_name: mira-taskrunner - image: ghcr.io/darpa-askem/mira-taskrunner:latest - networks: - - terarium - environment: - TERARIUM_MQ-ADDRESSES: "amqp://rabbitmq:5672" - TERARIUM_MQ-PASSWORD: "terarium123" - TERARIUM_MQ-USERNAME: "terarium" - TERARIUM_TASKRUNNER_REQUEST-TYPE: "mira" - depends_on: - rabbitmq: - condition: service_healthy - extra_hosts: - - "${local_host_name}:host-gateway" + gollm-taskrunner: + build: + dockerfile: ./packages/gollm/Dockerfile + target: gollm_taskrunner_builder + container_name: gollm-taskrunner + image: ghcr.io/darpa-askem/gollm-taskrunner:latest + networks: + - terarium + environment: + TERARIUM_MQ_ADDRESSES: "amqp://rabbitmq:5672" + TERARIUM_MQ_PASSWORD: "terarium123" + TERARIUM_MQ_USERNAME: "terarium" + TERARIUM_TASKRUNNER_REQUEST_TYPE: "gollm" + OPENAI_API_KEY: "${secret_gollm_openai_key}" + depends_on: + taskrunner-dev-builder: + condition: service_started + rabbitmq: + condition: service_healthy + extra_hosts: + - "${local_host_name}:host-gateway" + volumes: + - ../../packages/gollm:/gollm_task + - ../../packages/taskrunner:/taskrunner + command: /gollm_task/dev.sh - funman-taskrunner: - container_name: funman-taskrunner - image: ghcr.io/darpa-askem/funman-taskrunner:latest - networks: - - terarium - environment: - TERARIUM_MQ-ADDRESSES: "amqp://rabbitmq:5672" - TERARIUM_MQ-PASSWORD: "terarium123" - TERARIUM_MQ-USERNAME: "terarium" - TERARIUM_TASKRUNNER_REQUEST-TYPE: "funman" - depends_on: - rabbitmq: - condition: service_healthy - extra_hosts: - - "${local_host_name}:host-gateway" + mira-taskrunner: + build: + dockerfile: ../../packages/mira/Dockerfile + target: mira_taskrunner_builder + container_name: mira-taskrunner + image: ghcr.io/darpa-askem/mira-taskrunner:latest + networks: + - terarium + environment: + TERARIUM_MQ_ADDRESSES: "amqp://rabbitmq:5672" + TERARIUM_MQ_PASSWORD: "terarium123" + TERARIUM_MQ_USERNAME: "terarium" + TERARIUM_TASKRUNNER_REQUEST_TYPE: "mira" + depends_on: + taskrunner-dev-builder: + condition: service_started + rabbitmq: + condition: service_healthy + extra_hosts: + - "${local_host_name}:host-gateway" + volumes: + - ../../packages/mira:/mira_task + - ../../packages/taskrunner:/taskrunner + command: /mira_task/dev.sh + + funman-taskrunner: + build: + dockerfile: ../../packages/funman/Dockerfile + target: funman_taskrunner_builder + container_name: funman-taskrunner + image: ghcr.io/darpa-askem/funman-taskrunner:latest + networks: + - terarium + environment: + TERARIUM_MQ_ADDRESSES: "amqp://rabbitmq:5672" + TERARIUM_MQ_PASSWORD: "terarium123" + TERARIUM_MQ_USERNAME: "terarium" + TERARIUM_TASKRUNNER_REQUEST_TYPE: "funman" + depends_on: + taskrunner-dev-builder: + condition: service_started + rabbitmq: + condition: service_healthy + extra_hosts: + - "${local_host_name}:host-gateway" + volumes: + - ../../packages/funman:/funman_task + - ../../packages/taskrunner:/taskrunner + command: /funman_task/dev.sh diff --git a/packages/funman/.gitignore b/packages/funman/.gitignore new file mode 100644 index 0000000000..8a0dd2e3c8 --- /dev/null +++ b/packages/funman/.gitignore @@ -0,0 +1,40 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ +funman_task.egg-info + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ +mira.egg-info +__pycache__ diff --git a/packages/funman/Dockerfile b/packages/funman/Dockerfile index 7e96cd98d8..ed4414b957 100644 --- a/packages/funman/Dockerfile +++ b/packages/funman/Dockerfile @@ -1,5 +1,37 @@ # Build the Spring Boot application -FROM eclipse-temurin:17.0.11_9-jdk-focal AS taskrunner_builder +FROM eclipse-temurin:17.0.11_9-jdk-focal AS funman_taskrunner_builder + +###### DEV ONLY ###### +#VVVVVVVVVVVVVVVVVVVVV + +# These deps are installed only for use during local development + +# Install Python +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.10 \ + wget \ + git \ + python3-pip && \ + rm -rf /var/lib/apt/lists/* + +# Install funman-api +COPY ./packages/funman/funman-version.txt /funmanVersion.txt +RUN COMMIT_SHA="$(cat /funmanVersion.txt)" && \ + echo "Using FUNMAN commit $COMMIT_SHA" && \ + wget --progress=dot:giga -O funman.tar.gz "https://github.com/DARPA-ASKEM/funman-api/archive/${COMMIT_SHA}.tar.gz" && \ + tar -zxvf funman.tar.gz && \ + rm funman.tar.gz && \ + mv -v funman-* funman-api +WORKDIR /funman-api +RUN pip install --no-cache-dir . && \ + pip install --no-cache-dir auxiliary_packages/funman_dreal && \ + pip install --no-cache-dir auxiliary_packages/funman_demo + +#^^^^^^^^^^^^^^^^^^^^ +###################### WORKDIR /taskrunner @@ -14,37 +46,26 @@ FROM ghcr.io/darpa-askem/funman-base:latest WORKDIR / -# Install OpenJDK JRE and git +# Install OpenJDK JRE and wget RUN apt-get update && \ - apt-get install -y --no-install-recommends openjdk-17-jre-headless && \ - apt-get install -y --no-install-recommends git && \ + apt-get install -y --no-install-recommends \ + openjdk-17-jre-headless && \ rm -rf /var/lib/apt/lists/* # Copy the Spring Boot fat JAR from the builder image -COPY --from=taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar - -# Copy the echo script for testing -COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py +COPY --from=funman_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar # Install funman-api -RUN echo "Cache bust so we can always pull latest Funman" $(date -u) -COPY ./packages/funman/funman-version.txt /funmanVersion.txt - -RUN git clone https://github.com/DARPA-ASKEM/funman-api.git - +COPY --from=funman_taskrunner_builder /funman-api /funman-api WORKDIR /funman-api -RUN COMMIT_SHA=$(cat /funmanVersion.txt) && \ - echo "Using FUNMAN commit $COMMIT_SHA" && \ - git reset --hard $COMMIT_SHA RUN pip install --no-cache-dir . && \ - pip install --no-cache-dir auxiliary_packages/funman_dreal && \ - pip install --no-cache-dir auxiliary_packages/funman_demo + pip install --no-cache-dir auxiliary_packages/funman_dreal && \ + pip install --no-cache-dir auxiliary_packages/funman_demo # Install taskrunner -WORKDIR / +COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py COPY ./packages/taskrunner/setup.py /taskrunner/setup.py COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py - WORKDIR /taskrunner RUN pip install --no-cache-dir -e . diff --git a/packages/funman/dev.sh b/packages/funman/dev.sh new file mode 100755 index 0000000000..f4fc9edc96 --- /dev/null +++ b/packages/funman/dev.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# ensure the volume mounted python code is using editable mode +echo "Installing python tasks" +cd /funman_task +pip install -e . + +# run it +echo "Running taskrunner" +cd /taskrunner +./gradlew --project-cache-dir /tmp/.gradle/$$ bootRun diff --git a/packages/gollm/.gitignore b/packages/gollm/.gitignore new file mode 100644 index 0000000000..7c2e11f087 --- /dev/null +++ b/packages/gollm/.gitignore @@ -0,0 +1,40 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ +gollm_task.egg-info + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ +mira.egg-info +__pycache__ diff --git a/packages/gollm/Dockerfile b/packages/gollm/Dockerfile index b527331ab4..5ca851fe46 100644 --- a/packages/gollm/Dockerfile +++ b/packages/gollm/Dockerfile @@ -1,5 +1,34 @@ # Build the Spring Boot application -FROM eclipse-temurin:17.0.11_9-jdk-focal AS taskrunner_builder +FROM eclipse-temurin:17.0.11_9-jdk-focal AS gollm_taskrunner_builder + +###### DEV ONLY ###### +#VVVVVVVVVVVVVVVVVVVVV + +# These deps are installed only for use during local development + +# Install Python +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.10 \ + wget \ + python3-pip && \ + rm -rf /var/lib/apt/lists/* + +# Install gollm +COPY ./packages/gollm/gollm-version.txt /gollmVersion.txt +RUN COMMIT_SHA="$(cat /gollmVersion.txt)" && \ + echo "Using GoLLM commit $COMMIT_SHA" && \ + wget --progress=dot:giga -O gollm.tar.gz "https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz" && \ + tar -zxvf gollm.tar.gz && \ + rm gollm.tar.gz && \ + mv GoLLM-* GoLLM +WORKDIR /GoLLM +RUN pip install --no-cache-dir . + +#^^^^^^^^^^^^^^^^^^^^ +###################### WORKDIR /taskrunner @@ -7,6 +36,8 @@ COPY ./packages/taskrunner . RUN ./gradlew bootJar +WORKDIR / + # ------------------------------------------------------------------------------ # Set up the Python image with JRE @@ -16,35 +47,22 @@ WORKDIR / # Install OpenJDK JRE and wget RUN apt-get update && \ - apt-get install -y --no-install-recommends openjdk-17-jre-headless && \ - apt-get install -y --no-install-recommends wget && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends \ + openjdk-17-jre-headless && \ + rm -rf /var/lib/apt/lists/* # Copy the Spring Boot fat JAR from the builder image -COPY --from=taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar - -# Copy the echo script for testing -COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py +COPY --from=gollm_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar # Install GoLLM -RUN echo "Cache bust so we can always pull latest GoLLM" $(date -u) -COPY ./packages/gollm/gollm-version.txt /gollmVersion.txt -RUN COMMIT_SHA=$(cat /gollmVersion.txt) && \ - echo "Using GoLLM commit $COMMIT_SHA" && \ - wget -O gollm.tar.gz https://github.com/DARPA-ASKEM/GoLLM/archive/${COMMIT_SHA}.tar.gz - -RUN tar -zxvf gollm.tar.gz && \ - rm gollm.tar.gz -RUN mv GoLLM-* GoLLM - +COPY --from=gollm_taskrunner_builder /GoLLM /GoLLM WORKDIR /GoLLM -RUN pip install --no-cache-dir -e . +RUN pip install --no-cache-dir . # Install taskrunner -WORKDIR / +COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py COPY ./packages/taskrunner/setup.py /taskrunner/setup.py COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py - WORKDIR /taskrunner RUN pip install --no-cache-dir -e . diff --git a/packages/gollm/dev.sh b/packages/gollm/dev.sh new file mode 100755 index 0000000000..0f7f49a49f --- /dev/null +++ b/packages/gollm/dev.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# ensure the volume mounted python code is using editable mode +echo "Installing python tasks" +cd /gollm_task +pip install -e . + +# run it +echo "Running taskrunner" +cd /taskrunner +./gradlew --project-cache-dir /tmp/.gradle/$$ bootRun diff --git a/packages/mira/.gitignore b/packages/mira/.gitignore new file mode 100644 index 0000000000..17044b79c3 --- /dev/null +++ b/packages/mira/.gitignore @@ -0,0 +1,40 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ +mira_task.egg-info + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ +mira.egg-info +__pycache__ diff --git a/packages/mira/Dockerfile b/packages/mira/Dockerfile index db70ed6c30..a095999120 100644 --- a/packages/mira/Dockerfile +++ b/packages/mira/Dockerfile @@ -1,5 +1,34 @@ # Build the Spring Boot application -FROM eclipse-temurin:17.0.11_9-jdk-focal AS taskrunner_builder +FROM eclipse-temurin:17.0.11_9-jdk-focal AS mira_taskrunner_builder + +###### DEV ONLY ###### +#VVVVVVVVVVVVVVVVVVVVV + +# These deps are installed only for use during local development + +# Install Python +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.10 \ + wget \ + python3-pip && \ + rm -rf /var/lib/apt/lists/* + +# Install Mira +COPY ./packages/mira/mira-version.txt /miraVersion.txt +RUN COMMIT_SHA="$(cat /miraVersion.txt)" && \ + echo "Using MIRA commit $COMMIT_SHA" && \ + wget --progress=dot:giga -O mira.tar.gz "https://github.com/gyorilab/mira/archive/${COMMIT_SHA}.tar.gz" && \ + tar -zxvf mira.tar.gz && \ + rm mira.tar.gz && \ + mv -v mira-* mira +WORKDIR /mira +RUN pip install --no-cache-dir .[sbml,biomodels] + +#^^^^^^^^^^^^^^^^^^^^ +###################### WORKDIR /taskrunner @@ -16,35 +45,22 @@ WORKDIR / # Install OpenJDK JRE and wget RUN apt-get update && \ - apt-get install -y --no-install-recommends openjdk-17-jre-headless && \ - apt-get install -y --no-install-recommends wget && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends \ + openjdk-17-jre-headless && \ + rm -rf /var/lib/apt/lists/* # Copy the Spring Boot fat JAR from the builder image -COPY --from=taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar - -# Copy the echo script for testing -COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py +COPY --from=mira_taskrunner_builder /taskrunner/build/libs/*.jar /taskrunner.jar # Install Mira -RUN echo "Cache bust so we can always pull latest MIRA" $(date -u) -COPY ./packages/mira/mira-version.txt /miraVersion.txt -RUN COMMIT_SHA=$(cat /miraVersion.txt) && \ - echo "Using MIRA commit $COMMIT_SHA" && \ - wget -O mira.tar.gz https://github.com/gyorilab/mira/archive/${COMMIT_SHA}.tar.gz - -RUN tar -zxvf mira.tar.gz && \ - rm mira.tar.gz -RUN mv -v mira-* mira - +COPY --from=mira_taskrunner_builder /mira /mira WORKDIR /mira -RUN pip install --no-cache-dir -e .[sbml,biomodels] +RUN pip install --no-cache-dir .[sbml,biomodels] # Install taskrunner -WORKDIR / +COPY ./packages/taskrunner/src/test/resources/echo.py /echo.py COPY ./packages/taskrunner/setup.py /taskrunner/setup.py COPY ./packages/taskrunner/taskrunner.py /taskrunner/taskrunner.py - WORKDIR /taskrunner RUN pip install --no-cache-dir -e . diff --git a/packages/mira/dev.sh b/packages/mira/dev.sh new file mode 100755 index 0000000000..5e9d2166e4 --- /dev/null +++ b/packages/mira/dev.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# ensure the volume mounted python code is using editable mode +echo "Installing python tasks" +cd /mira_task +pip install -e . + +# run it +echo "Running taskrunner" +cd /taskrunner +./gradlew --project-cache-dir /tmp/.gradle/$$ bootRun diff --git a/packages/taskrunner/.gitignore b/packages/taskrunner/.gitignore index c399e0665b..440861bb6c 100644 --- a/packages/taskrunner/.gitignore +++ b/packages/taskrunner/.gitignore @@ -4,6 +4,7 @@ build/ !gradle/wrapper/gradle-wrapper.jar !**/src/main/**/build/ !**/src/test/**/build/ +taskrunner.egg-info ### STS ### .apt_generated diff --git a/packages/taskrunner/Dockerfile b/packages/taskrunner/Dockerfile new file mode 100644 index 0000000000..3c572273db --- /dev/null +++ b/packages/taskrunner/Dockerfile @@ -0,0 +1,8 @@ +# Build the Spring Boot application +FROM eclipse-temurin:17.0.11_9-jdk-focal AS taskrunner_dev_builder + +COPY ./packages/taskrunner . + +WORKDIR /taskrunner + +CMD [ "./gradlew", "build", "--continuous", "--quiet", "-x", "test"] diff --git a/packages/taskrunner/README.md b/packages/taskrunner/README.md index a846a4f9b2..e3752a0066 100644 --- a/packages/taskrunner/README.md +++ b/packages/taskrunner/README.md @@ -27,37 +27,11 @@ Use the following as a reference for creating a taskrunner image: # Local Development: -In the `docker-compose-taskrunner.yml` volume mount your local task code: +Everything is setup in the `docker-compose-taskrunner.yml` for local development. The docker-compose entries use the *builder* image and volume mounts the taskrunner directories into the running container. Both the java `taskrunner` package and the python repos will automatically pickup and local changes and hotswap accordingly. -GoLLM task volume mounting: -``` -volumes: - - ./packages/gollm:/gollm_task -``` - -Mira task volume mounting: -``` -volumes: - - ./packages/mira:/mira_task -``` - -Funman task volume mounting: -``` -volumes: - - ./packages/funman:/funman_task -``` +This also runs an auxiliary container called `taskrunner-dev-builer` which re-builds the java code on changes. -Then exec into the container: - -``` -docker exec -it {IMAGE_TAG} /bin/bash -``` - -On code changes, re-install the tasks: - -``` -pip install -e -``` +**NOTE: When starting these local images, it takes about a minute or so to build and start the java taskrunner.** # Building docker images: