Use poetry for dependency management (#411)

* Use poetry for dependency management

* Fix test

* Prevent bind mount from overwriting assets

* Add just target for regenerating poetry lock file

Dockerfile

@@ -1,61 +1,51 @@
-FROM python:3.12.1-slim-bullseye as base
+# node layer to build static assets
+FROM node:16 AS nodejs
+WORKDIR /usr/src/app/
+COPY package.json yarn.lock ./
+RUN yarn install
+COPY OpenOversight/app/static/ OpenOversight/app/static/
+RUN yarn build
 
-WORKDIR /usr/src/app
 
+FROM python:3.12.1-slim-bullseye as base
+ARG IS_PROD
+ENV DEBIAN_FRONTEND noninteractive
+ENV PIP_NO_CACHE_DIR=1
+ENV POETRY_NO_INTERACTION=1
+ENV POETRY_VIRTUALENVS_CREATE=0
 ENV PYTHONDONTWRITEBYTECODE=1
-ENV DEBIAN-FRONTEND noninteractive
 ENV PYTHONUNBUFFERED=1
-ENV PIP_NO_CACHE_DIR=1
-ENV NODE_MAJOR=16
+WORKDIR /usr/src/app
 
-RUN apt-get update && \
- apt-get install -y ca-certificates curl gnupg && \
- mkdir -p /etc/apt/keyrings && \
- curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
- echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+# Install packages depending on ENV arg from docker-compose
+ARG BASE_PACKAGES="gcc libpq-dev libjpeg62-turbo-dev libsqlite3-0 zlib1g-dev"
+ARG DEV_PACKAGES="firefox-esr xvfb"
+RUN if [ "$IS_PROD" = "true" ]; then \
+ PACKAGES_TO_INSTALL="$BASE_PACKAGES"; \
+ else \
+ PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL $DEV_PACKAGES"; \
+ fi && \
  apt-get update && \
- apt-get install -y \
- gcc \
- libpq-dev \
- python3-dev \
- nodejs \
- libjpeg62-turbo-dev \
- libsqlite3-0 \
- zlib1g-dev && \
+ apt-get install -y -qq --no-install-recommends $PACKAGES_TO_INSTALL && \
  apt-get clean && \
  rm -rf /var/lib/apt/lists/*
 
-RUN npm install -g yarn && \
- mkdir /var/www ./node_modules /.cache /.yarn /.mozilla && \
- touch /usr/src/app/yarn-error.log
-COPY yarn.lock /usr/src/app/
-RUN chmod -R 777 /usr/src/app/ /.cache /.yarn
-
-# Add prod requirements to base image
-COPY requirements.txt /usr/src/app/
-RUN pip3 install -r requirements.txt
-
-COPY package.json /usr/src/app/
-RUN yarn
-COPY create_db.py /usr/src/app/
-
-WORKDIR /usr/src/app/
+# Add runtime dependencies to base image
+RUN pip3 install poetry~=1.8.0
+COPY pyproject.toml poetry.lock ./
+RUN if [ "$IS_PROD" = "true" ]; then \
+ poetry install --only main --no-root; \
+ else \
+ poetry install --no-root; \
+ fi
+
+# Setup application
+COPY create_db.py .
 COPY OpenOversight OpenOversight
+COPY --from=nodejs /usr/src/app/OpenOversight/app/static/dist/ OpenOversight/app/static/dist/
 
-# Development Target
-FROM base as development
-
-RUN apt-get update && \
- apt-get install -y firefox-esr xvfb && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/*
-
-# Install additional development requirements
-COPY requirements-dev.txt /usr/src/app/
-RUN pip3 install -r /usr/src/app/requirements-dev.txt
-
-CMD ["OpenOversight/scripts/entrypoint.sh"]
-
-# Production Target
-FROM base as production
-CMD ["OpenOversight/scripts/entrypoint.sh"]
+CMD if [ "$IS_PROD" = "true" ]; then \
+ gunicorn -w 4 -b 0.0.0.0:3000 OpenOversight.app:app; \
+ else \
+ flask run --host=0.0.0.0 --port=3000; \
+ fi

README.md

@@ -22,6 +22,10 @@ To run the project locally:
  4. `Commit changes?`: `y`
 3. Run `just up` and visit http://localhost:3000!
 
+To re-generate the poetry lock file (if you need to update dependencies in pyproject.toml):
+1. Run `just lock`
+2. Include the modified `poetry.lock` file in your commit
+
 
 ## Deployment
 

docker-compose.dev.yml

@@ -42,7 +42,6 @@ services:
  web:
  build:
  context: .
- target: development
  depends_on:
  - postgres
  - minio
@@ -51,9 +50,14 @@ services:
  FLASK_DEBUG: 1
  SQLALCHEMY_WARN_20: 1
  volumes:
+ - ./poetry.lock:/usr/src/app/poetry.lock
+ - ./pyproject.toml:/usr/src/app/pyproject.toml
  - ./OpenOversight:/usr/src/app/OpenOversight
+ - oo-assets:/usr/src/app/OpenOversight/app/static/dist
  ports:
  - "3000:3000"
 
 volumes:
  minio:
+ # Prevent bind mount from overwriting assets: https://stackoverflow.com/q/57065750
+ oo-assets:

docker-compose.prod.yml

@@ -4,7 +4,8 @@ services:
  web:
  build:
  context: .
- target: production
+ args:
+ IS_PROD: "true"
  environment:
  ENV: production
  volumes:

docker-compose.yml

@@ -29,7 +29,6 @@ services:
  secrets:
  - source: service-account-key
  target: /usr/src/app/service_account_key.json
- command: OpenOversight/scripts/entrypoint.sh
 
 volumes:
  postgres:

justfile

@@ -85,6 +85,10 @@ import +args:
 lint:
  pre-commit run --all-files
 
+# Generate poetry lockfile
+lock:
+ just run --no-deps web poetry lock
+
 # Run Flask-Migrate tasks in the web container
 db +migrateargs:
  just run --no-deps web flask db {{ migrateargs }}