diff --git a/.github/actions/cleanup-runner/action.yaml b/.github/actions/cleanup-runner/action.yaml new file mode 100644 index 00000000..6276de58 --- /dev/null +++ b/.github/actions/cleanup-runner/action.yaml @@ -0,0 +1,33 @@ +name: 'Cleanup Runner' +description: 'Cleans up GitHub-hosted runner to free up disk space for large builds' +runs: + using: "composite" + steps: + - name: Cleanup hosted runner + shell: bash + run: | + # Remove unnecessary packages + sudo apt purge -yqq dotnet-* mono-* llvm-* libllvm* powershell* openjdk-* \ + temurin-* mongodb-* firefox mysql-* \ + hhvm google-chrome-stable \ + libgl1-mesa-dri microsoft-edge-stable azure-cli || true + + # Clean apt cache + sudo apt clean + sudo apt autoremove -y + + # Remove large directories + sudo rm -rf /usr/share/dotnet /usr/local/lib/android + sudo rm -rf /usr/local/share/chromium /usr/local/share/chrome + sudo rm -rf /usr/local/.ghcup /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache/* /usr/local/lib/node_modules + + # Clean npm and yarn caches + npm cache clean --force || true + yarn cache clean || true + + # Remove Docker images and build cache + docker system prune -af + + # Show available space after cleanup + df -h diff --git a/.github/workflows/ai-runner-live-pipelines-docker.yaml b/.github/workflows/ai-runner-live-pipelines-docker.yaml new file mode 100644 index 00000000..6b03031b --- /dev/null +++ b/.github/workflows/ai-runner-live-pipelines-docker.yaml @@ -0,0 +1,241 @@ +name: Build ai-runner live pipeline Docker images + +on: + pull_request: + paths: + - "runner/docker/Dockerfile.live-*" + - "runner/app/**" + - "runner/images/**" + push: + branches: + - main + tags: + - '*' + paths: + - "runner/docker/Dockerfile.live-*" + - "runner/app/**" + - "runner/images/**" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-common-base: + name: Build common live base image + permissions: + pull-requests: read + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 50 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + runner/docker/Dockerfile.live-base + + - name: Check if build needed + id: check_build + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || + ( + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + steps.changed-files.outputs.any_changed == 'true' + ) + run: echo "should_build=true" >> $GITHUB_OUTPUT + + - name: Cleanup hosted runner + uses: ./.github/actions/cleanup-runner + if: steps.check_build.outputs.should_build == 'true' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.CI_DOCKERHUB_USERNAME }} + password: ${{ secrets.CI_DOCKERHUB_TOKEN }} + + - name: Build and push live-base image + uses: docker/build-push-action@v5 + if: steps.check_build.outputs.should_build == 'true' + with: + context: "{{defaultContext}}:runner" + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: livepeer/ai-runner:live-base + file: docker/Dockerfile.live-base + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max + + build-pipeline-images: + name: Build pipeline images + needs: build-common-base + runs-on: ubuntu-20.04 + permissions: + pull-requests: read + strategy: + matrix: + pipeline: [streamdiffusion, comfyui, liveportrait] + fail-fast: false + steps: + - name: Check out code + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 50 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files_yaml: | + base_dockerfile: + - runner/docker/Dockerfile.live-base-${{ matrix.pipeline }} + liveportrait: + - runner/images/** + - runner/requirements-liveportrait.txt + + - name: Cleanup hosted runner + uses: ./.github/actions/cleanup-runner + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.CI_DOCKERHUB_USERNAME }} + password: ${{ secrets.CI_DOCKERHUB_TOKEN }} + + - name: Build and push pipeline base image + uses: docker/build-push-action@v5 + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || + ( + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + ( + steps.changed-files.outputs.base_dockerfile_any_changed == 'true' || + ( + matrix.pipeline == 'liveportrait' && + steps.changed-files.outputs.liveportrait_any_changed == 'true' + ) + ) + ) + with: + context: "{{defaultContext}}:runner" + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: livepeer/ai-runner:live-base-${{ matrix.pipeline }} + file: docker/Dockerfile.live-base-${{ matrix.pipeline }} + build-args: | + PIPELINE=${{ matrix.pipeline }} + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max + + - name: Extract metadata for app image + id: meta + uses: docker/metadata-action@v5 + with: + images: livepeer/ai-runner + tags: | + type=raw,value=live-app-${{ matrix.pipeline }} + type=sha,prefix=live-app-${{ matrix.pipeline }}- + type=ref,event=pr,prefix=live-app-${{ matrix.pipeline }}- + type=ref,event=tag,prefix=live-app-${{ matrix.pipeline }}- + type=raw,value=latest,enable={{is_default_branch}},prefix=live-app-${{ matrix.pipeline }}- + + - name: Build and push pipeline app image + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:runner" + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + file: docker/Dockerfile.live-app__PIPELINE__ + build-args: | + PIPELINE=${{ matrix.pipeline }} + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max + + build-noop: + name: Build pipeline image (noop) + needs: build-common-base + runs-on: ubuntu-20.04 + permissions: + pull-requests: read + steps: + - name: Check out code + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 50 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + runner/docker/Dockerfile.live-app-noop + runner/app/** + + - name: Check if build needed + id: check_build + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || + ( + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + steps.changed-files.outputs.any_changed == 'true' + ) + run: echo "should_build=true" >> $GITHUB_OUTPUT + + - name: Cleanup hosted runner + uses: ./.github/actions/cleanup-runner + if: steps.check_build.outputs.should_build == 'true' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.CI_DOCKERHUB_USERNAME }} + password: ${{ secrets.CI_DOCKERHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: livepeer/ai-runner + tags: | + type=raw,value=live-app-noop + type=sha,prefix=live-app-noop- + type=ref,event=pr,prefix=live-app-noop- + type=ref,event=tag,prefix=live-app-noop- + type=raw,value=latest,enable={{is_default_branch}},prefix=live-app-noop- + + - name: Build and push noop image + uses: docker/build-push-action@v5 + if: steps.check_build.outputs.should_build == 'true' + with: + context: "{{defaultContext}}:runner" + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + file: docker/Dockerfile.live-app-noop + cache-from: type=registry,ref=livepeerci/build:cache + cache-to: type=registry,ref=livepeerci/build:cache,mode=max diff --git a/.github/workflows/ai-runner-pipelines-docker.yaml b/.github/workflows/ai-runner-pipelines-docker.yaml index cd25a150..fce99da3 100644 --- a/.github/workflows/ai-runner-pipelines-docker.yaml +++ b/.github/workflows/ai-runner-pipelines-docker.yaml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - dockerfile: + dockerfile: - docker/Dockerfile.segment_anything_2 - docker/Dockerfile.text_to_speech steps: @@ -42,13 +42,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: Cleanup hosted runner - run: | - sudo apt purge -yqq dotnet-* mono-* llvm-* libllvm* powershell* openjdk-* \ - temurin-* mongodb-* firefox mysql-* \ - hhvm google-chrome-stable \ - libgl1-mesa-dri microsoft-edge-stable azure-cli || true - sudo apt autoremove -y - sudo rm -rf /usr/share/dotnet /usr/local/lib/android + uses: ./.github/actions/cleanup-runner - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/runner/app/live/infer.py b/runner/app/live/infer.py index 546d3ff0..00837ea4 100644 --- a/runner/app/live/infer.py +++ b/runner/app/live/infer.py @@ -100,7 +100,7 @@ async def start_control_subscriber(handler: PipelineStreamer, control_url: str): "--publish-url", type=str, required=True, help="URL to publish output frames (trickle). For zeromq this is the output socket address" ) parser.add_argument( - "--control-url", type=str, help="URL to subscribe for Control API JSON messages" + "--control-url", type=str, help="URL to subscribe for Control API JSON messages to update inference params" ) parser.add_argument( "--input-timeout", diff --git a/runner/docker/Dockerfile.live-base-liveportrait b/runner/docker/Dockerfile.live-base-liveportrait index 42cf6ba8..05e3e0b0 100644 --- a/runner/docker/Dockerfile.live-base-liveportrait +++ b/runner/docker/Dockerfile.live-base-liveportrait @@ -2,7 +2,7 @@ ARG BASE_IMAGE=livepeer/ai-runner:live-base FROM ${BASE_IMAGE} # Download and install the NVIDIA TensorRT repository local deb -RUN wget https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/secure/8.6.1/local_repos/nv-tensorrt-local-repo-ubuntu2204-8.6.1-cuda-12.0_1.0-1_amd64.deb && \ +RUN wget --progress=dot:mega https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/secure/8.6.1/local_repos/nv-tensorrt-local-repo-ubuntu2204-8.6.1-cuda-12.0_1.0-1_amd64.deb && \ dpkg -i nv-tensorrt-local-repo-ubuntu2204-8.6.1-cuda-12.0_1.0-1_amd64.deb && \ cp /var/nv-tensorrt-local-repo-ubuntu2204-8.6.1-cuda-12.0/*-keyring.gpg /usr/share/keyrings/ && \ rm nv-tensorrt-local-repo-ubuntu2204-8.6.1-cuda-12.0_1.0-1_amd64.deb @@ -58,7 +58,7 @@ COPY images/flame-serious.jpg \ COPY requirements-liveportrait.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt -# TODO: Setup dependencies for animal models +# TODO: Setup dependencies for animal models (needs some custom deps detected on runtime which are not on pypi) WORKDIR /app diff --git a/runner/run-lv2v.sh b/runner/run-lv2v.sh index b22b147d..db6b9dc6 100755 --- a/runner/run-lv2v.sh +++ b/runner/run-lv2v.sh @@ -13,8 +13,12 @@ PORT=${4:-9000} # Build images, this will be quick if everything is cached docker build -t livepeer/ai-runner:live-base -f docker/Dockerfile.live-base . -docker build -t livepeer/ai-runner:live-base-${PIPELINE} -f docker/Dockerfile.live-base-${PIPELINE} . -docker build -t livepeer/ai-runner:live-app-${PIPELINE} -f docker/Dockerfile.live-app__PIPELINE__ --build-arg PIPELINE=${PIPELINE} . +if [ "${PIPELINE}" = "noop" ]; then + docker build -t livepeer/ai-runner:live-app-noop -f docker/Dockerfile.live-app-noop . +else + docker build -t livepeer/ai-runner:live-base-${PIPELINE} -f docker/Dockerfile.live-base-${PIPELINE} . + docker build -t livepeer/ai-runner:live-app-${PIPELINE} -f docker/Dockerfile.live-app__PIPELINE__ --build-arg PIPELINE=${PIPELINE} . +fi CONTAINER_NAME=live-video-to-video-${PIPELINE} docker run -it --rm --name ${CONTAINER_NAME} \