Merge branch 'redis-hashed-requests-and-tasks' of https://github.com/…

…Igor8mr/turbinia into redis-hashed-requests-and-tasks

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -11,15 +11,15 @@ body:
         Please fill in as much of the following form as you're able.
   - type: textarea
     attributes:
-      label: What would this feature improve or what problem would it solve?
+      label: What is the feature you are proposing?
+      description: Describe the feature. If you already have something in mind... PRs are welcome!
     validations:
       required: true
   - type: textarea
     attributes:
-      label: What is the feature you are proposing?
-      description: Describe the feature. If you already have something in mind... PRs are welcome!
+      label: What would this feature improve or what problem would it solve?
     validations:
       required: true
   - type: textarea
     attributes:
-      label: What alternatives have you considered?
+      label: What alternatives have you considered?

.github/workflows/actions.yml

@@ -18,22 +18,17 @@ jobs:
       - name: Pull latest docker image for cache
         run: |
           docker pull ${{ matrix.docker_base_image }}
-          docker pull us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker-dev:latest
-      - name: Build Turbinia worker docker image
-        run: docker build --build-arg PPA_TRACK=${{ matrix.gift_ppa_track }} --cache-from=${{ matrix.docker_base_image }},us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker-dev:latest -t turbinia-worker-dev -f docker/worker/Dockerfile .
-      - name: Run test (turbinia-worker) container
+          docker pull us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-unit-tests:latest
+      - name: Build Turbinia unit test docker image
+        run: docker build --build-arg PPA_TRACK=${{ matrix.gift_ppa_track }} --cache-from=${{ matrix.docker_base_image }},us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-unit-tests:latest -t turbinia-unit-tests -f docker/tests/Dockerfile .
+      - name: Run test (turbinia-unit-tests) container
         run: |
-          docker run --name turbinia-worker --entrypoint "/bin/bash" -it -d -t turbinia-worker-dev:latest
+          docker run --name turbinia-unit-tests --entrypoint "/bin/bash" -it -d -t turbinia-unit-tests:latest
       - name: Configure python3
         run: |
-          docker exec -u root -t turbinia-worker bash -c "update-alternatives --install /usr/bin/python python /usr/bin/python3 1"
-          docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -V"
-      - name: Install and configure test dependencies
-        run: |
-          docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -m pip install --quiet --upgrade pip"
-          docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -m pip install --quiet mock nose coverage yapf"
-          docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -m pip install --quiet tox"
+          docker exec -u root -t turbinia-unit-tests bash -c "update-alternatives --install /usr/bin/python python /usr/bin/python3 1"
+          docker exec -u root -t turbinia-unit-tests bash -c "/usr/bin/python -V"
       - name: Run Tests
         run: |
-          docker exec -u root -t turbinia-worker bash -c "cd /tmp && ./run_tests.py"
-          docker exec -u root -t turbinia-worker bash -c "cd /tmp && tox --sitepackages ${TOXENV}"
+          docker exec -u root -t turbinia-unit-tests bash -c "poetry run ./run_tests.py"
+          docker exec -u root -t turbinia-unit-tests bash -c "tox --sitepackages ${TOXENV}"

.github/workflows/e2e.yml

@@ -40,4 +40,4 @@ jobs:
       - name: Run E2E test
         run: |
           chmod +x ./turbinia/e2e/e2e-local.sh
-          ./turbinia/e2e/e2e-local.sh
+          ./turbinia/e2e/e2e-local.sh

.github/workflows/publish-pypi.yml

@@ -0,0 +1,107 @@
+name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Install pypa/build
+      run: python3 -m  pip install poetry wheel --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m poetry build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/20')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/turbinia
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      Sign the Python 🐍 distribution 📦 with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/[email protected]
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Upload artifact signatures to GitHub Release
+      # Te release must already be created or this will fail.
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        '${{ github.ref_name }}' dist/**
+        --repo '${{ github.repository }}'
+
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/turbinia
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/

.gitignore

@@ -41,4 +41,4 @@ venv/
 .venv/
 
 # Ignore OpenAPI code generator temporary files
-.openapi-generator/
+.openapi-generator/

AUTHORS

@@ -5,8 +5,10 @@ Turbinia is developed by (in alphabetical order):
 
 * Google Inc. (*@google.com)
 * Aaron Peterson ([email protected])
-* Wajih Yassine ([email protected])
 * Johan Berggren ([email protected])
+* Juan Leaniz ([email protected])
+* Ramses de Beer ([email protected])
+* Wajih Yassine ([email protected])
 
 To reach the authors, please use the Turbinia development mailing
 list <[email protected]>.

README.md

@@ -108,7 +108,7 @@ Commands:
   submit    Submit new requests to the Turbinia API server.
 ```
 
-Check out the `turbinia-client` documentation [page]((https://turbinia.readthedocs.io/en/latest/user/turbinia-client.html#turbinia-api-cli-tool-turbinia-client)) for a detailed user guide.
+Check out the `turbinia-client` documentation [page](https://turbinia.readthedocs.io/en/latest/user/turbinia-client.html#turbinia-api-cli-tool-turbinia-client) for a detailed user guide.
 
 You can also interact with Turbinia directly from Python by using the API library. We provide some examples [here](https://github.com/google/turbinia/tree/master/turbinia/api/client)
 

docker/api_server/Dockerfile

@@ -7,38 +7,45 @@ COPY web/. .
 RUN npm run build
 
 # Build Turbinia API Server, copy from build, and setup rest of requirements
-FROM ubuntu:22.04
+FROM ubuntu:22.04 as build-stage2
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PIP_NO_CACHE_DIR=1 
 
-ENV DEBIAN_FRONTEND=noninteractive 
 COPY --from=build-stage /tmp/web/dist /web/dist
 RUN apt-get update && apt-get -y upgrade
-RUN apt-get -y install python3-pip git wget supervisor curl
-
+RUN apt-get -y install git python3-pip
 RUN pip3 install pip --upgrade
-RUN pip3 install urllib3 cryptography requests --upgrade
-
-ADD requirements.txt /tmp/
-RUN cd /tmp/ && pip3 install -r requirements.txt
-
-ADD . /tmp/
-
-# unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone
-RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi
-
-RUN cd /tmp/ && python3 setup.py install
-
-RUN useradd -r -s /bin/nologin -u 999 turbinia
+RUN pip3 install poetry
 
+RUN useradd -r -s /sbin/nologin -u 999 turbinia
 RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \
     && mkdir -p /var/log/turbinia/ && chown -R turbinia:turbinia /mnt/turbinia/ \
     && mkdir -p /etc/turbinia/ \
     && chown -R turbinia:turbinia /var/lib/turbinia/ \
     && chown -R turbinia:turbinia /etc/turbinia/ \
-    && chown -R turbinia:turbinia /var/log/turbinia/
+    && chown -R turbinia:turbinia /var/log/turbinia/ \
+    && mkdir -p /home/turbinia && chown -R turbinia:turbinia /home/turbinia
 
-COPY docker/api_server/start.sh /home/turbinia/start.sh
-RUN chmod +rwx /home/turbinia/start.sh
+# Drop privileges and set the working directory
 USER turbinia
+WORKDIR /home/turbinia
+
+# Copy requirements and install dependencies to cache them in docker layer
+COPY --chown=turbinia:turbinia ./pyproject.toml ./poetry.toml ./poetry.lock /home/turbinia/
+RUN poetry install --no-interaction --no-ansi --no-root
+
+ENV PATH="/home/turbinia/.venv/bin:$PATH" \
+    VIRTUAL_ENV=/home/turbinia/.venv
+
+# Copy the source directory to the container
+COPY --chown=turbinia:turbinia . /home/turbinia/
+COPY --chown=turbinia:turbinia docker/api_server/start.sh /home/turbinia/start.sh
+RUN chmod +rwx /home/turbinia/start.sh
+
+# Install Turbinia package -- will skip dependencies if installed
+RUN poetry install --no-interaction --no-ansi
+
 CMD ["/home/turbinia/start.sh"]
 # Expose Prometheus and API endpoints.
 EXPOSE 9200/tcp

docker/api_server/Dockerfile.dockerignore

@@ -0,0 +1,17 @@
+nodemodules
+dist
+.git
+.venv
+.vscode
+test_data
+.github
+.devcontainer
+docs
+config
+build
+*.pyc
+*.pyo
+*.pyd
+__pycache__
+.cache
+turbinia.egg-info

docker/api_server/start.sh

@@ -7,7 +7,12 @@ then
 fi
 
 # Start Turbinia API server
-/usr/local/bin/turbiniactl api_server
+if [ ! -z ${TURBINIA_LOG_FILE+x} ]
+then
+    poetry run turbiniactl $TURBINIA_EXTRA_ARGS -L $TURBINIA_LOG_FILE api_server
+else
+    poetry run turbiniactl $TURBINIA_EXTRA_ARGS api_server
+fi
 
 # Don't exit
 while sleep 1000; do :; done

docker/local/docker-compose.yml

@@ -57,17 +57,18 @@ services:
             - LC_ALL=C.UTF-8
             - LANG=C.UTF-8
             - TURBINIA_EXTRA_ARGS=${TURBINIA_EXTRA_ARGS}
-    # Uncomment below in case you want to run a second worker on the same host.
-    #  turbinia-worker2:
-    #     #image: "turbinia-worker-dev" # Use this for local development and comment out below line
-    #     image: "us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker:latest" # Latest stable
-    #     container_name: turbinia-worker2
-    #     privileged: true
 
-    #     volumes:
-    #      - $PWD/evidence:/evidence
-    #      - $PWD/conf/turbinia.conf:/etc/turbinia/turbinia.conf
+      # Uncomment below in case you want to run a second worker on the same host.
+      # turbinia-worker2:
+      #   image: "turbinia-worker-dev" # Use this for local development and comment out below line
+      #   image: "us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker:latest" # Latest stable
+      #   container_name: turbinia-worker2
+      #   privileged: true
 
-    #     environment:
-    #      - LC_ALL=C.UTF-8
-    #      - LANG=C.UTF-8
+      #   volumes:
+      #    - $PWD/evidence:/evidence
+      #    - $PWD/conf/turbinia.conf:/etc/turbinia/turbinia.conf
+
+      #   environment:
+      #    - LC_ALL=C.UTF-8
+      #    - LANG=C.UTF-8