Lipsync pipeline using real3dportrait with parler-tts

cmd/lipsync/Dockerfile.debug

@@ -0,0 +1,18 @@
+# A Dockerfile for attaching a debugger to the app using debugpy. For more information, see
+# https://code.visualstudio.com/docs/python/debugging#_command-line-debugging.
+FROM livepeer/ai-runner:lipsync
+
+RUN pip install debugpy
+
+EXPOSE 5678
+# Set the entrypoint to activate the base venv, as well as start the debugger and the application with Uvicorn.
+# This version attaches the debugger and starts the app in one command.
+ENTRYPOINT ["bash", "-c", "\
+    source /opt/venv/bin/activate && \
+    python -m debugpy --listen 0.0.0.0:5678 \
+    # Uncomment the line below to wait for the debugger to attach \
+    # --wait-for-client && \
+    exec uvicorn app.main:app \
+    --log-config app/cfg/uvicorn_logging_config.json \
+    --host 0.0.0.0 \
+    --port 8000"]

cmd/lipsync/Dockerfile.lipsync

@@ -0,0 +1,112 @@
+FROM pschroedl/ai-runner:base
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Add the DeadSnakes PPA to get Python 3.10
+RUN apt-get update && apt-get install -y software-properties-common \
+    && add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update
+
+# Install necessary build tools and Python
+RUN apt-get update && apt-get install -y \
+    ffmpeg libsm6 libxext6 python3-dev ninja-build \
+    gawk bison gcc make wget tar git bzip2 python3-pip python3.10 python3.10-dev python3.10-venv\
+    build-essential cmake libgl1-mesa-dev libglib2.0-0 \
+    libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
+    curl llvm libncurses5-dev libncursesw5-dev \
+    xz-utils tk-dev libffi-dev liblzma-dev python3-openssl \
+    portaudio19-dev \
+    libasound2-dev \
+    libjack-jackd2-dev \
+    libopenblas-dev \
+    liblapack-dev \
+    build-essential \
+    autoconf \
+    automake \
+    libtool \
+    pkg-config \
+    nasm \
+    libpng-dev \
+    libjpeg-dev \
+    libwebp-dev \
+    libbrotli-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set Python 3.10 as the version for lipsync venv
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 \
+    && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 \
+    && python --version
+
+# create and activate our venv
+ENV VIRTUAL_ENV=/opt/lipsync_venv
+RUN python3.10 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Upgrade pip (24.2?) and install openmim
+# RUN python3.9 -m pip install --upgrade pip
+# RUN apt-get remove -y python3-yaml
+RUN pip install -U openmim==0.3.9
+
+# Install PyTorch with CUDA 12.1 support
+RUN pip3 install torch==2.3.1 --extra-index-url https://download.pytorch.org/whl/cu121
+
+RUN pip3 install setuptools 
+# Install mmcv using openmim
+RUN pip3 install mmcv==2.2.0 -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.4/index.html
+
+# Download and install GLIBC 2.35
+RUN wget http://mirrors.edge.kernel.org/ubuntu/pool/main/g/glibc/libc6_2.35-0ubuntu3_amd64.deb && \
+    dpkg -i libc6_2.35-0ubuntu3_amd64.deb || apt-get install -f -y
+
+# Clone the repository
+ARG REPO_URL=https://github.com/pschroedl/Real3DPortrait.git
+ARG REPO_DIR=/models--yerfor--Real3DPortrait
+RUN git clone -b upgrade_scipy $REPO_URL $REPO_DIR
+
+# Set the working directory
+WORKDIR $REPO_DIR
+# Install remaining Python dependencies
+RUN pip3 install setuptools==60.2.0
+RUN pip3 install numpy==1.23.0
+RUN pip3 install cython && \
+    pip3 install -r docs/prepare_env/requirements.txt -v && \
+    pip3 install -r docs/prepare_env/requirements.txt -v --use-deprecated=legacy-resolver
+
+# Install PyTorch3D with CUDA 12.1 support
+RUN pip3 install iopath fvcore
+RUN pip3 install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py310_cu121_pyt231/download.html
+
+# Install CUDA toolkit from the NVIDIA repository
+RUN apt-get update && apt-get install -y wget software-properties-common \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin \
+    && mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub -O /tmp/cuda-repo.gpg \
+    && apt-key add /tmp/cuda-repo.gpg \
+    && add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" \
+    && apt-get update \
+    && apt-get install -y cuda-toolkit-12-1
+
+WORKDIR /app/
+
+# reactivate origininal venv for overrides TODO: Fix me
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+COPY ./runner/requirements.txt /app
+# RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir \
+    transformers==4.43.0 \
+    peft \
+    deepcache \
+    soundfile \
+    opencv-python \
+    pillow \
+    g2p-en \
+    torchshow \ 
+    flash_attn \
+    git+https://github.com/huggingface/parler-tts.git
+
+WORKDIR /app/
+
+ENV PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/sbin:$PATH"
+COPY ./runner/app/ /app/app
+# Ensure the virtual environment is activated before starting Uvicorn
+ENTRYPOINT ["bash", "-c", "source /opt/venv/bin/activate && exec uvicorn app.main:app --log-config app/cfg/uvicorn_logging_config.json --host 0.0.0.0 --port 8000"]

cmd/lipsync/README.md

@@ -0,0 +1,75 @@
+# Setup for lipsync pipeline
+
+Real3DPortrait https://github.com/yerfor/Real3DPortrait is utilized for the lipsync pipeline
+
+Conda environment is built and configured on the host at /models/models--yerfor--Real3DPortrait/anaconda3 to mimic the paths that will be encountered by the containerized app.
+
+### Setup
+
+On the host machine, run :
+
+
+``` sudo ./setup_real3dportrait_env.sh ~```
+
+Models are expected to be stored at ~/.lpData/models/ on the host machine, and the final step of this script copies the new conda environment to ~/.lpData/models/models/models--yerfor--Real3DPortrait/anaconda3.
+
+### Usage
+
+```
+git clone https://github.com/livepeer/ai-worker.git
+cd ai-worker/
+docker build -f ./cmd/lipsync/Dockerfile.lipsync -t livepeer/ai-runner:lipsync .
+docker run --name lipsync-public -e PIPELINE=lipsync -e MODEL_ID=real3dportrait -e HUGGINGFACE_TOKEN={your token or env var} --gpus 0 -p 8005:8000 -v ~/.lpData/models:/models livepeer/ai-runner:lipsync
+```
+
+### Implementation details
+
+The lipsync pipeline in ai-worker/runner invokes a helper script ( cmd/lipsync/run_real3dportrait.sh ) to activate the conda environment inside a shell running in the container, Then, the real3dportrait python inference script is invoked with the parameters provided to the helper script.
+
+### Debugging the lipsync pipeline
+
+Launch.json for Visual Studio Code: 
+```
+{
+    "version": "0.2.0",
+    "configurations": [
+
+        {
+            "name": "Python: Uvicorn",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "uvicorn",
+            "args": [
+                "app.main:app",
+                "--log-config", "app/cfg/uvicorn_logging_config.json",
+                "--host", "0.0.0.0",
+                "--port", "8000"
+            ],
+            "env": {
+                "PIPELINE": "lipsync",
+                "MODEL_ID": "real3dportrait",
+                "HUGGINGFACE_TOKEN": "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"
+            },
+            "python": "python",
+            "console": "integratedTerminal"
+        }
+    ]
+}
+
+```
+
+Changes to reflect use of the lipsync debug dockerfile need to be made to .devcontainer:
+
+```
+diff --git a/runner/.devcontainer/devcontainer.json b/runner/.devcontainer/devcontainer.json
+index 3253288..f5c94d6 100644
+--- a/runner/.devcontainer/devcontainer.json
++++ b/runner/.devcontainer/devcontainer.json
+@@ -5,7 +5,11 @@
+
+        // Image to use for the dev container. More info: https://containers.dev/guide/dockerfile.
+        "build": {
+-               "dockerfile": "../Dockerfile"
++               "context": "../..",
++               "dockerfile": "../../cmd/lipsync/Dockerfile.debug",
+```

cmd/lipsync/check_models_downloaded.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+ROOT_DIR=$(eval echo $1)
+# Navigate to the repository directory
+cd ${ROOT_DIR}/models/models--yerfor--Real3DPortrait
+# Check if all required files are in place
+required_files=(
+    "deep_3drecon/BFM/01_MorphableModel.mat"
+    "deep_3drecon/BFM/BFM_exp_idx.mat"
+    "deep_3drecon/BFM/BFM_front_idx.mat"
+    "deep_3drecon/BFM/BFM_model_front.mat"
+    "deep_3drecon/BFM/Exp_Pca.bin"
+    "deep_3drecon/BFM/facemodel_info.mat"
+    "deep_3drecon/BFM/std_exp.txt"
+    "deep_3drecon/reconstructor_opt.pkl"
+    "checkpoints/240210_real3dportrait_orig/audio2secc_vae/config.yaml"
+    "checkpoints/240210_real3dportrait_orig/audio2secc_vae/model_ckpt_steps_400000.ckpt"
+    "checkpoints/240210_real3dportrait_orig/secc2plane_torso_orig/config.yaml"
+    "checkpoints/240210_real3dportrait_orig/secc2plane_torso_orig/model_ckpt_steps_100000.ckpt"
+    "checkpoints/pretrained_ckpts/mit_b0.pth"
+)
+
+all_files_exist=true
+
+for file in "${required_files[@]}"; do
+    if [ ! -f "$file" ]; then
+        echo "File not found: $file"
+        all_files_exist=false
+    fi
+done
+
+if $all_files_exist; then
+    echo "All files are in the correct places."
+else
+    echo "Some files are missing."
+fi
+
+cd ..
+mv $CONDA_DIR $REPO_DIR/anaconda3

cmd/lipsync/setup_real3dportrait_env.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+# Check if the root directory is passed as an argument
+if [ -z "$1" ]; then
+  echo "Please provide the root directory as an argument."
+  exit 1
+fi
+
+ROOT_DIR=$(eval echo $1)
+
+# Navigate to the repository directory
+cd ${ROOT_DIR}/models/
+mkdir models--yerfor--Real3DPortrait
+cd models--yerfor--Real3DPortrait
+
+# Required for downloading real3dportrait checkpoints
+pip install gdown
+
+# Create the required directories
+mkdir -p deep_3drecon/BFM
+mkdir -p checkpoints/240126_real3dportrait_orig/audio2secc_vae
+mkdir -p checkpoints/240126_real3dportrait_orig/secc2plane_torso_orig
+mkdir -p checkpoints/pretrained_ckpts
+
+echo "Directory structure created."
+
+# Download 3DMM BFM Model
+echo "Downloading 3DMM BFM Model..."
+cd deep_3drecon/BFM
+gdown https://drive.google.com/uc?id=1SPM3IHsyNAaVMwqZZGV6QVaV7I2Hly0v
+gdown https://drive.google.com/uc?id=1MSldX9UChKEb3AXLVTPzZQcsbGD4VmGF
+gdown https://drive.google.com/uc?id=180ciTvm16peWrcpl4DOekT9eUQ-lJfMU
+gdown https://drive.google.com/uc?id=1KX9MyGueFB3M-X0Ss152x_johyTXHTfU
+gdown https://drive.google.com/uc?id=19-NyZn_I0_mkF-F5GPyFMwQJ_-WecZIL
+gdown https://drive.google.com/uc?id=11ouQ7Wr2I-JKStp2Fd1afedmWeuifhof
+gdown https://drive.google.com/uc?id=18ICIvQoKX-7feYWP61RbpppzDuYTptCq
+gdown https://drive.google.com/uc?id=1VktuY46m0v_n_d4nvOupauJkK4LF6mHE
+cd ../..
+
+# Download Pre-trained Real3D-Portrait models
+echo "Downloading Pre-trained Real3D-Portrait models..."
+cd checkpoints
+gdown https://drive.google.com/uc?id=1gz8A6xestHp__GbZT5qozb43YaybRJhZ
+gdown https://drive.google.com/uc?id=1gSUIw2AkkKnlLJnNfS2FCqtaVw9tw3QF
+unzip 240210_real3dportrait_orig.zip
+unzip pretrained_ckpts.zip
+ls
+cd ..
+# Navigate to the repository directory
+cd ${ROOT_DIR}/models/models--yerfor--Real3DPortrait
+# Check if all required files are in place
+required_files=(
+    "deep_3drecon/BFM/01_MorphableModel.mat"
+    "deep_3drecon/BFM/BFM_exp_idx.mat"
+    "deep_3drecon/BFM/BFM_front_idx.mat"
+    "deep_3drecon/BFM/BFM_model_front.mat"
+    "deep_3drecon/BFM/Exp_Pca.bin"
+    "deep_3drecon/BFM/facemodel_info.mat"
+    "deep_3drecon/BFM/std_exp.txt"
+    "deep_3drecon/reconstructor_opt.pkl"
+    "checkpoints/240210_real3dportrait_orig/audio2secc_vae/config.yaml"
+    "checkpoints/240210_real3dportrait_orig/audio2secc_vae/model_ckpt_steps_400000.ckpt"
+    "checkpoints/240210_real3dportrait_orig/secc2plane_torso_orig/config.yaml"
+    "checkpoints/240210_real3dportrait_orig/secc2plane_torso_orig/model_ckpt_steps_100000.ckpt"
+    "checkpoints/pretrained_ckpts/mit_b0.pth"
+)
+
+all_files_exist=true
+
+for file in "${required_files[@]}"; do
+    if [ ! -f "$file" ]; then
+        echo "File not found: $file"
+        all_files_exist=false
+    fi
+done
+
+if $all_files_exist; then
+    echo "All files are in the correct places."
+else
+    echo "Some files are missing."
+fi
+
+cd ..
+mv $CONDA_DIR $REPO_DIR/anaconda3