codeforboston#382 import loop

README.md

@@ -32,7 +32,7 @@ This method uses Docker to run the complete application stack.
 > **Note**
 > When running locally, you may need to update one of the ports in the `.env` file if it conflicts with another application on your machine.
 
-3. Build and run the project with `docker-compose build && docker-compose up -d && docker-compose logs -f`
+3. Build and run the project with `docker compose build && docker compose up -d && docker compose logs -f`
 
 ## Installation (Frontend Only)
 
@@ -57,15 +57,15 @@ You'll need to replace `police-data-trust-api-1` with the name of the container
 docker container ls
 CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
 c0cf******** police-data-trust-api "/bin/sh -c '/wait &…" About a minute ago Up About a minute 0.0.0.0:5001->5001/tcp police-data-trust-api-1
-5e6f******** postgres:16.1 "docker-entrypoint.s…" 3 days ago Up About a minute 0.0.0.0:5432->5432/tcp police-data-trust-db-1
+5e6f******** postgres:16  "docker-entrypoint.s…" 3 days ago Up About a minute 0.0.0.0:5432->5432/tcp police-data-trust-db-1
 dacd******** police-data-trust-web "docker-entrypoint.s…" 3 days ago Up About a minute 0.0.0.0:3000->3000/tcp police-data-trust-web-1
 ```
 
 ### Backend Tests
 
 The current backend tests can be found in the GitHub Actions workflow file [python-tests.yml](https://github.com/codeforboston/police-data-trust/blob/0488d03c2ecc01ba774cf512b1ed2f476441948b/.github/workflows/python-tests.yml)
 
-To run the tests locally, first start the application with docker-compose. Then open up a command line interface to the running container:
+To run the tests locally, first start the application with docker compose. Then open up a command line interface to the running container:
 
 ```
 docker exec -it "police-data-trust-api-1" /bin/bash
@@ -82,7 +82,7 @@ python -m pytest
 
 The current frontend tests can be found in the GitHub Actions workflow file [frontend-checks.yml](https://github.com/codeforboston/police-data-trust/blob/0488d03c2ecc01ba774cf512b1ed2f476441948b/.github/workflows/frontend-checks.yml)
 
-To run the tests locally, first start the application with docker-compose. Then open up a command line interface to the running container:
+To run the tests locally, first start the application with dockerccompose. Then open up a command line interface to the running container:
 
 ```
 docker exec -it "police-data-trust-web-1" /bin/bash

backend/Dockerfile.cloud

@@ -15,7 +15,7 @@ RUN arch=$(arch) && \
  file=pandas-2.2.2-cp312-cp312-manylinux_2_17_${arch}.manylinux2014_${arch}.whl && \
  url="https://pypi.debian.net/pandas/${file}" && \
  wget ${url} && \
- sed -i "s/pandas==1.5.3/${file}/" prod.txt
+ sed -i "s/pandas==2.2.2/${file}/" prod.txt
 RUN pip install --no-cache-dir -r prod.txt
 
 COPY . .

backend/api.py

@@ -33,6 +33,9 @@ def create_app(config: Optional[str] = None):
  # def _():
  # db.create_all()
 
+ # start background processor for SQS imports
+
+
  return app
 
 

backend/import/loop.py

@@ -0,0 +1,50 @@
+from io import BytesIO
+from logging import getLogger
+from time import sleep
+
+import boto3
+import ujson
+
+class Importer:
+ def __init__(self, queue_name: str, region: str = "us-east-1"):
+ self.queue_name = queue_name
+ self.session = boto3.Session(region_name=region)
+ self.sqs_client = self.session.client("sqs")
+ self.s3_client = self.session.client("s3")
+ self.sqs_queue_url = self.sqs_client.get_queue_url(QueueName=self.queue_name)
+ self.logger = getLogger(self.__class__.__name__)
+
+ def run(self):
+ while True:
+ resp = self.sqs_client.receive_message(
+ QueueUrl=self.sqs_queue_url,
+ MaxNumberOfMessages=1, # retrieve one message at a time - we could up this and parallelize but no point until way more files.
+ VisibilityTimeout=600, # 10 minutes to process message before it becomes visible for another consumer.
+ )
+ # if no messages found, wait 5m for next poll
+ if len(resp["Messages"]) == 0:
+ sleep(600)
+ continue
+
+ for message in resp["Messages"]:
+ sqs_body = ujson.loads(message["Body"])
+ for record in sqs_body["Records"]: # this comes through as a list, but we expect one object
+ bucket_name = record["s3"]["bucket"]["name"]
+ key = record["s3"]["object"]["key"]
+ with BytesIO() as fileobj:
+ self.s3_client.download_fileobj(bucket_name, key, fileobj)
+ fileobj.seek(0)
+ content = fileobj.read()
+
+ # TODO: we now have an in-memory copy of the s3 file content. This is where we would run the import.
+ # we want a standardized importer class; we would call something like below:
+ # loader = Loader(content).load()
+
+ self.logger.info(f"Imported s3://{bucket_name}/{key}")
+
+class Loader:
+ def __init__(self, content: bytes):
+ self.content = content
+
+ def load(self):
+ raise Exception("unimplemented; extend this class to write a load migration.")

backend/scraper/data_scrapers/README.md

@@ -14,10 +14,10 @@ You can also run the scraper in Docker:
 
 ```bash
 # From the base of the repository
-docker-compose build api
-docker-compose run -u $(id -u) api flask scrape
+docker compose build api
+docker compose run -u $(id -u) api flask scrape
 # Stop the database service
-docker-compose down
+docker compose down
 ```
 
 You may see several warnings about mixed types. The script could also take several minutes.

backend/scraper/notebooks/cpdp.ipynb

@@ -29,7 +29,7 @@
  "\n",
  "```bash\n",
  "# Stop services and remove volumes, rebuild images, start the database, create tables, run seeds, and follow logs\n",
- "docker-compose down -v && docker-compose up --build -d db api && docker-compose logs -f\n",
+ "docker compose down -v && docker compose up --build -d db api && docker compose logs -f\n",
  "```\n",
  "\n",
  "Then open the notebook with either [VSCode](https://code.visualstudio.com/) or `jupyter notebook`.\n",

backend/scraper/notebooks/mpv.ipynb

@@ -23,7 +23,7 @@
  "\n",
  "```bash\n",
  "# Stop services and remove volumes, rebuild images, start the database, create tables, run seeds, and follow logs\n",
- "docker-compose down -v && docker-compose up --build -d db api && docker-compose logs -f\n",
+ "docker compose down -v && docker compose up --build -d db api && docker compose logs -f\n",
  "```\n",
  "\n",
  "Then open the notebook with either [VSCode](https://code.visualstudio.com/) or `jupyter notebook`.\n",

docker-compose.notebook.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
  api:
  command: bash -c '/wait && flask psql create && flask psql init && jupyter notebook --allow-root --ip=0.0.0.0 --port=8889'

docker-compose.yml

@@ -1,7 +1,6 @@
-version: "3"
 services:
  db:
- image: postgres:16.2 #AWS RDS latest version
+ image: postgres:16 #AWS RDS latest version
  env_file:
  - ".env"
  volumes:

requirements/Dockerfile

@@ -2,7 +2,7 @@
 # requirements, so this image starts with the same image as the database
 # containers and installs the same version of python as the api containers
 
-FROM postgres:16.2 as base
+FROM postgres:16 as base
 
 RUN apt-get update && apt-get install -y \
  make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
@@ -15,9 +15,9 @@ SHELL ["bash", "-lc"]
 RUN curl https://pyenv.run | bash && \
  echo 'export PATH="$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH"' >> ~/.bashrc
 
-ENV PYTHON_VERSION=3.12.3
+ENV PYTHON_VERSION=3.12.4
 RUN pyenv install ${PYTHON_VERSION} && pyenv global ${PYTHON_VERSION}
-RUN pip install pip-tools
+RUN pip install -U pip-tools
 
 COPY . requirements/
 

requirements/README.md

@@ -20,7 +20,7 @@ python -m pip install -r requirements/dev_unix.txt
 
 ```bash
 cd requirements
-docker-compose up --build --force-recreate
+docker compose up --build --force-recreate
 ```
 
 If you run the application natively, first install the pip-compile tool:

requirements/_core.in

@@ -1,5 +1,6 @@
 bcrypt==3.2.2
 black
+boto3
 celery
 flake8
 flask
@@ -35,4 +36,5 @@ numpy
 spectree
 jupyter
 mixpanel
-ua-parser
+ua-parser
+ujson

requirements/dev_unix.txt

@@ -44,6 +44,12 @@ bleach==6.1.0
  # via nbconvert
 blinker==1.7.0
  # via flask-mail
+boto3==1.34.133
+ # via -r requirements/_core.in
+botocore==1.34.133
+ # via
+ # boto3
+ # s3transfer
 build==1.2.1
  # via pip-tools
 celery==5.3.6
@@ -186,6 +192,10 @@ jinja2==3.1.3
  # jupyterlab
  # jupyterlab-server
  # nbconvert
+jmespath==1.0.1
+ # via
+ # boto3
+ # botocore
 json5==0.9.25
  # via jupyterlab-server
 jsonpointer==2.4
@@ -405,6 +415,7 @@ pytest-postgresql==5.1.0
 python-dateutil==2.9.0
  # via
  # arrow
+ # botocore
  # celery
  # jupyter-client
  # pandas
@@ -451,6 +462,8 @@ rpds-py==0.18.0
  # via
  # jsonschema
  # referencing
+s3transfer==0.10.2
+ # via boto3
 send2trash==1.8.2
  # via jupyter-server
 six==1.16.0
@@ -528,10 +541,13 @@ tzdata==2024.1
  # pandas
 ua-parser==0.18.0
  # via -r requirements/_core.in
+ujson==5.10.0
+ # via -r requirements/_core.in
 uri-template==1.3.0
  # via jsonschema
 urllib3==1.26.18
  # via
+ # botocore
  # mixpanel
  # requests
 vine==5.1.0

requirements/dev_windows.txt

@@ -44,6 +44,12 @@ bleach==6.1.0
  # via nbconvert
 blinker==1.7.0
  # via flask-mail
+boto3==1.34.133
+ # via -r requirements/_core.in
+botocore==1.34.133
+ # via
+ # boto3
+ # s3transfer
 build==1.2.1
  # via pip-tools
 celery==5.3.6
@@ -186,6 +192,10 @@ jinja2==3.1.3
  # jupyterlab
  # jupyterlab-server
  # nbconvert
+jmespath==1.0.1
+ # via
+ # boto3
+ # botocore
 json5==0.9.25
  # via jupyterlab-server
 jsonpointer==2.4
@@ -405,6 +415,7 @@ pytest-postgresql==5.1.0
 python-dateutil==2.9.0
  # via
  # arrow
+ # botocore
  # celery
  # jupyter-client
  # pandas
@@ -451,6 +462,8 @@ rpds-py==0.18.0
  # via
  # jsonschema
  # referencing
+s3transfer==0.10.2
+ # via boto3
 send2trash==1.8.2
  # via jupyter-server
 six==1.16.0
@@ -528,10 +541,13 @@ tzdata==2024.1
  # pandas
 ua-parser==0.18.0
  # via -r requirements/_core.in
+ujson==5.10.0
+ # via -r requirements/_core.in
 uri-template==1.3.0
  # via jsonschema
 urllib3==1.26.18
  # via
+ # botocore
  # mixpanel
  # requests
 vine==5.1.0

requirements/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
  pip-compile:
  build: