From 112d862e92403394bd3be0a0305693f4103cfa72 Mon Sep 17 00:00:00 2001 From: jyotipm29 Date: Wed, 30 Oct 2024 12:36:13 +0100 Subject: [PATCH 1/4] bump to 24.1 --- .github/workflows/compose.yml | 115 ++++++++---------- .github/workflows/lint.yml | 10 +- .github/workflows/pull-request.yml | 88 ++++++-------- compose/README.md | 54 ++++---- .../galaxy-cluster-base/Dockerfile | 18 +-- .../galaxy-container-base/Dockerfile | 33 ++--- compose/base_config.yml | 79 ++++++++---- compose/docker-compose.htcondor.yml | 3 +- compose/docker-compose.k8s.yml | 1 - compose/docker-compose.pulsar.mq.yml | 1 - compose/docker-compose.pulsar.yml | 5 +- compose/docker-compose.singularity.yml | 1 - compose/docker-compose.slurm.yml | 11 +- compose/docker-compose.yml | 16 ++- compose/galaxy-configurator/Dockerfile | 4 +- compose/galaxy-configurator/customize.py | 2 +- compose/galaxy-configurator/run.sh | 6 +- .../galaxy/container_resolvers_conf.xml.j2 | 66 ---------- .../galaxy/container_resolvers_conf.yml.j2 | 83 +++++++++++++ .../galaxy/dependency_resolvers_conf.xml.j2 | 18 +++ .../templates/galaxy/galaxy.yml.j2 | 19 ++- .../templates/galaxy/job_conf.xml.j2 | 12 +- .../templates/nginx/nginx.conf.j2 | 24 ++-- .../pulsar/dependency_resolvers_conf.xml.j2 | 56 --------- compose/galaxy-htcondor/Dockerfile | 30 ++--- compose/galaxy-htcondor/start.sh | 1 + compose/galaxy-kind/Dockerfile | 10 +- compose/galaxy-nginx/Dockerfile | 2 +- compose/galaxy-server/Dockerfile | 65 +++++----- .../galaxy-server/files/create_galaxy_user.py | 3 +- compose/galaxy-server/files/start.sh | 11 +- .../galaxy-slurm-node-discovery/Dockerfile | 2 +- compose/galaxy-slurm-node-discovery/run.sh | 6 +- compose/galaxy-slurm/Dockerfile | 28 ++--- compose/galaxy-slurm/start.sh | 5 +- compose/pulsar/Dockerfile | 6 +- .../tests/docker-compose.test.bioblend.yml | 6 +- .../tests/docker-compose.test.selenium.yml | 5 +- .../tests/docker-compose.test.workflows.yml | 5 +- compose/tests/docker-compose.test.yml | 5 +- compose/tests/galaxy-bioblend-test/Dockerfile | 9 +- compose/tests/galaxy-bioblend-test/run.sh | 11 +- compose/tests/galaxy-selenium-test/Dockerfile | 18 +-- compose/tests/galaxy-selenium-test/run.sh | 2 +- compose/tests/galaxy-workflow-test/Dockerfile | 13 +- compose/tests/galaxy-workflow-test/run.sh | 4 +- 46 files changed, 496 insertions(+), 476 deletions(-) delete mode 100644 compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.xml.j2 create mode 100644 compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.yml.j2 create mode 100644 compose/galaxy-configurator/templates/galaxy/dependency_resolvers_conf.xml.j2 delete mode 100644 compose/galaxy-configurator/templates/pulsar/dependency_resolvers_conf.xml.j2 diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index c076434d3..c45feefbb 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -2,25 +2,25 @@ name: build-and-test on: [push] jobs: build_container_base: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set image tag id: image_tag run: | if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then - echo "::set-output name=image_tag::latest"; + echo "image_tag=latest" >> $GITHUB_OUTPUT; else - echo "::set-output name=image_tag::${GITHUB_REF#refs/heads/}" + echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT; fi - name: Docker Login run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }} - name: Set up Docker Buildx id: buildx - uses: crazy-max/ghaction-docker-buildx@v1 + uses: docker/setup-buildx-action@v3 with: - buildx-version: v0.3.1 + version: v0.17.1 - name: Run Buildx env: image_name: galaxy-container-base @@ -41,22 +41,22 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set image tag id: image_tag run: | if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then - echo "::set-output name=image_tag::latest"; + echo "image_tag=latest" >> $GITHUB_OUTPUT; else - echo "::set-output name=image_tag::${GITHUB_REF#refs/heads/}" + echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT; fi - name: Docker Login run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }} - name: Set up Docker Buildx id: buildx - uses: crazy-max/ghaction-docker-buildx@v1 + uses: docker/setup-buildx-action@v3 with: - buildx-version: v0.3.1 + version: v0.17.1 - name: Run Buildx env: image_name: galaxy-cluster-base @@ -95,22 +95,22 @@ jobs: fail-fast: false steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set image tag id: image_tag run: | if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then - echo "::set-output name=image_tag::latest"; + echo "image_tag=latest" >> $GITHUB_OUTPUT; else - echo "::set-output name=image_tag::${GITHUB_REF#refs/heads/}" + echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT; fi - name: Docker Login run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }} - name: Set up Docker Buildx id: buildx - uses: crazy-max/ghaction-docker-buildx@v1 + uses: docker/setup-buildx-action@v3 with: - buildx-version: v0.3.1 + version: v0.17.1 - name: Run Buildx run: | for i in {1..4}; do @@ -133,69 +133,43 @@ jobs: infrastructure: - name: galaxy-base files: -f docker-compose.yml - exclude_test: - - workflow_example1 - name: galaxy-proxy-prefix files: -f docker-compose.yml - env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix + env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix EXTRA_SKIP_TESTS_BIOBLEND="not test_import_export_workflow_dict and not test_import_export_workflow_from_local_path" exclude_test: - - workflow_example1 + - selenium - name: galaxy-htcondor files: -f docker-compose.yml -f docker-compose.htcondor.yml - exclude_test: - - workflow_example1 - name: galaxy-slurm files: -f docker-compose.yml -f docker-compose.slurm.yml env: SLURM_NODE_COUNT=3 options: --scale slurm_node=3 - exclude_test: - - workflow_example1 - name: galaxy-pulsar files: -f docker-compose.yml -f docker-compose.pulsar.yml exclude_test: - - workflow_example1 - - workflow_mapping_by_sequencing - # - name: galaxy-pulsar-mq - # files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml - # exclude_test: - # - workflow_example1 - # - workflow_mapping_by_sequencing + - workflow_quality_control + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job" + - name: galaxy-pulsar-mq + files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml + exclude_test: + - workflow_quality_control + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job" - name: galaxy-k8s files: -f docker-compose.yml -f docker-compose.k8s.yml - exclude_test: - - bioblend - - workflow_example1 - - workflow_ard - - workflow_mapping_by_sequencing - - selenium - name: galaxy-singularity files: -f docker-compose.yml -f docker-compose.singularity.yml + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" + - name: galaxy-pulsar-mq-singularity + files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job and not test_get_container_resolvers and not test_show_container_resolver" exclude_test: - - bioblend - - workflow_ard - - workflow_mapping_by_sequencing - - selenium - # - name: galaxy-pulsar-mq-singularity - # files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml - # exclude_test: - # - bioblend - # - workflow_ard - # - workflow_mapping_by_sequencing - # - selenium + - workflow_quality_control - name: galaxy-slurm-singularity files: -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml - exclude_test: - - bioblend - - workflow_ard - - workflow_mapping_by_sequencing - - selenium + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" - name: galaxy-htcondor-singularity files: -f docker-compose.yml -f docker-compose.htcondor.yml -f docker-compose.singularity.yml - exclude_test: - - bioblend - - workflow_ard - - workflow_mapping_by_sequencing - - selenium + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" test: - name: bioblend files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml @@ -208,10 +182,10 @@ jobs: workflow: sklearn/ard/ard.ga timeout: 60 second_run: "true" - - name: workflow_mapping_by_sequencing + - name: workflow_quality_control files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml exit-from: galaxy-workflow-test - workflow: training/variant-analysis/mapping-by-sequencing/mapping_by_sequencing.ga + workflow: training/sequence-analysis/quality-control/quality_control.ga timeout: 60 - name: workflow_example1 files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml @@ -229,9 +203,9 @@ jobs: - name: Check if test should be run id: run_check if: contains(matrix.infrastructure.exclude_test, matrix.test.name) != true - run: echo ::set-output name=run::true + run: echo "run=true" >> $GITHUB_OUTPUT - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set image tag in env run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - name: Master branch - Set image to to 'latest' @@ -240,6 +214,9 @@ jobs: - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV + - name: Install Docker Compose + run: | + sudo apt-get update -qq && sudo apt-get install docker-compose -y - name: Run tests for the first time if: steps.run_check.outputs.run run: | @@ -270,11 +247,15 @@ jobs: shell: bash working-directory: ./compose continue-on-error: false + - name: Fix file names before saving artifacts + if: failure() + run: | + sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \; - name: Allow upload-artifact read access if: failure() run: sudo chmod -R +r ./compose/export/galaxy/database - name: Save artifacts for debugging a failed test - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: failure() with: name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_first-run @@ -286,7 +267,7 @@ jobs: sudo rm -rf export/galaxy/database working-directory: ./compose - name: Run tests a second time - if: matrix.test.second_run == 'true' && steps.run_check.run + if: matrix.test.second_run == 'true' && steps.run_check.outputs.run run: | export DOCKER_REGISTRY=${{ secrets.docker_registry }} export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} @@ -312,11 +293,15 @@ jobs: shell: bash working-directory: ./compose continue-on-error: false + - name: Fix file names before saving artifacts + if: failure() && matrix.test.second_run == 'true' + run: | + sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \; - name: Allow upload-artifact read access if: failure() && matrix.test.second_run == 'true' run: sudo chmod -R +r ./compose/export/galaxy/database - name: Save artifacts for debugging a failed test - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: failure() && matrix.test.second_run == 'true' with: name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_second-run diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 36692201a..e33397ff8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,18 +5,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 - - name: Cleanup to only use compose - run: rm -R docs galaxy test + uses: actions/checkout@v4 + # - name: Cleanup to only use compose + # run: rm -R docs galaxy test - name: Run shellcheck with reviewdog - uses: reviewdog/action-shellcheck@v1.1.3 + uses: reviewdog/action-shellcheck@v1.27.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-check level: warning pattern: "*.sh" - name: Run hadolint with reviewdog - uses: reviewdog/action-hadolint@v1.16.0 + uses: reviewdog/action-hadolint@v1.46.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-check diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 12baa530c..2505f0a7d 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -2,76 +2,49 @@ name: pr-test on: pull_request jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: infrastructure: - name: galaxy-base files: -f docker-compose.yml - exclude_test: - - workflow_example1 - name: galaxy-proxy-prefix files: -f docker-compose.yml - env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix + env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix EXTRA_SKIP_TESTS_BIOBLEND="not test_import_export_workflow_dict and not test_import_export_workflow_from_local_path" exclude_test: - - workflow_example1 + - selenium - name: galaxy-htcondor files: -f docker-compose.yml -f docker-compose.htcondor.yml - exclude_test: - - workflow_example1 - name: galaxy-slurm files: -f docker-compose.yml -f docker-compose.slurm.yml env: SLURM_NODE_COUNT=3 options: --scale slurm_node=3 - exclude_test: - - workflow_example1 - name: galaxy-pulsar files: -f docker-compose.yml -f docker-compose.pulsar.yml + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job" + exclude_test: + - workflow_quality_control + - name: galaxy-pulsar-mq + files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job" exclude_test: - - workflow_example1 - - workflow_mapping_by_sequencing - - workflow_ard - # - name: galaxy-pulsar-mq - # files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml - # exclude_test: - # - workflow_example1 - # - workflow_mapping_by_sequencing + - workflow_quality_control - name: galaxy-k8s files: -f docker-compose.yml -f docker-compose.k8s.yml - exclude_test: - - bioblend - - workflow_ard - - workflow_mapping_by_sequencing - - workflow_example1 - - selenium - name: galaxy-singularity files: -f docker-compose.yml -f docker-compose.singularity.yml + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" + - name: galaxy-pulsar-mq-singularity + files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job and not test_get_container_resolvers and not test_show_container_resolver" exclude_test: - - bioblend - - workflow_ard - - workflow_mapping_by_sequencing - - selenium - # - name: galaxy-pulsar-mq-singularity - # files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml - # exclude_test: - # - bioblend - # - workflow_ard - # - workflow_mapping_by_sequencing - # - selenium + - workflow_quality_control - name: galaxy-slurm-singularity files: -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml - exclude_test: - - bioblend - - workflow_ard - - workflow_mapping_by_sequencing - - selenium + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" - name: galaxy-htcondor-singularity files: -f docker-compose.yml -f docker-compose.htcondor.yml -f docker-compose.singularity.yml - exclude_test: - - bioblend - - workflow_ard - - workflow_mapping_by_sequencing - - selenium + env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" test: - name: bioblend files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml @@ -84,10 +57,10 @@ jobs: workflow: sklearn/ard/ard.ga timeout: 60 second_run: "true" - - name: workflow_mapping_by_sequencing + - name: workflow_quality_control files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml exit-from: galaxy-workflow-test - workflow: training/variant-analysis/mapping-by-sequencing/mapping_by_sequencing.ga + workflow: training/sequence-analysis/quality-control/quality_control.ga timeout: 60 - name: workflow_example1 files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml @@ -105,9 +78,9 @@ jobs: - name: Check if test should be run id: run_check if: contains(matrix.infrastructure.exclude_test, matrix.test.name) != true - run: echo ::set-output name=run::true + run: echo "run=true" >> $GITHUB_OUTPUT - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV @@ -129,6 +102,9 @@ jobs: --build-arg IMAGE_TAG=ci-testing \ $image_name working-directory: ./compose/base-images + - name: Install Docker Compose + run: | + sudo apt-get update -qq && sudo apt-get install docker-compose -y - name: Run tests for the first time if: steps.run_check.outputs.run run: | @@ -160,11 +136,15 @@ jobs: shell: bash working-directory: ./compose continue-on-error: false + - name: Fix file names before saving artifacts + if: failure() + run: | + sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \; - name: Allow upload-artifact read access if: failure() run: sudo chmod -R +r ./compose/export/galaxy/database - name: Save artifacts for debugging a failed test - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: failure() with: name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_first-run @@ -176,7 +156,7 @@ jobs: sudo rm -rf export/galaxy/database working-directory: ./compose - name: Run tests a second time - if: matrix.test.second_run == 'true' && steps.run_check.run + if: matrix.test.second_run == 'true' && steps.run_check.outputs.run run: | export IMAGE_TAG=ci-testing export COMPOSE_DOCKER_CLI_BUILD=1 @@ -202,11 +182,15 @@ jobs: shell: bash working-directory: ./compose continue-on-error: false + - name: Fix file names before saving artifacts + if: failure() && matrix.test.second_run == 'true' + run: | + sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \; - name: Allow upload-artifact read access - if: failure() + if: failure() && matrix.test.second_run == 'true' run: sudo chmod -R +r ./compose/export/galaxy/database - name: Save artifacts for debugging a failed test - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: failure() && matrix.test.second_run == 'true' with: name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_second-run diff --git a/compose/README.md b/compose/README.md index d73d39a5d..5527cded2 100644 --- a/compose/README.md +++ b/compose/README.md @@ -1,7 +1,7 @@ # Galaxy Docker Compose This setup is built on the idea to use a basic docker-compose file and extend it for additional use cases. Therefore the `docker-compose.yml` is the base of the -whole setup. By concatinating additional files, you can extend it to use, for +whole setup. By concatenating additional files, you can extend it to use, for example, HTCondor (see [Usage](#usage)). All working data (database, virtual environment, etc.) is exported in the @@ -107,7 +107,7 @@ template folder (remember to add the `.j2` extension!) and mention it in the While Kind is starting up the cluster, it blocks Galaxy from starting itself. This is needed as Galaxy will parse the KUBECONFIG (that is created after k8s has started) -only once on startup. So don't be surprised if Galaxy is quite for some time :) +only once on startup. So don't be surprised if Galaxy is quiet for some time :) Note that the cluster is being rebuilt on every start (to be more precise, a `kind delete cluster` is called on shut down), so manual changes will @@ -151,9 +151,9 @@ All options are discussed under [configuration reference](#configuration-referen ### Use specific Galaxy version or Docker images The `IMAGE_TAG` environment variable allows to use specific versions of the -setup. Say, you want to stay with Galaxy v20.09 for now: +setup. Say, you want to stay with Galaxy v24.1 for now: -> export IMAGE_TAG=20.09 +> export IMAGE_TAG=24.1 > docker-compose up Without setting this variable, you will always get updated to the newest @@ -205,13 +205,13 @@ base-images are tagged differently or are stored in a different repository. ### Extend the Galaxy-Configurator It is possible to extend the usage of the configurator, both in extending the -Jinj2 templates, but also in adding additional files. +Jinja2 templates, but also in adding additional files. All environment variables of the `galaxy-configurator` are accessible within the templates. Additionally, the configurator parses specific `*_CONFIG_*` variables and makes them accessible as a dict (for example `galaxy` or -`galaxy_uwsgi`). It may be helpful to understand the current use cases +`gravity`). It may be helpful to understand the current use cases within the templates and how the `customize.py` file (actually just an extension of the [J2cli](https://github.com/kolypto/j2cli) parses env variables. @@ -238,7 +238,7 @@ overwrites a single environment variable and sets a new one. The neat thing of this approach is that if you don't need to run HTCondor, the base setup will work just fine without -much additional balast. However, adding HTCondor isn't a hassle +much additional ballast. However, adding HTCondor isn't a hassle either. **Singularity** @@ -249,7 +249,7 @@ can avoid that. In normal cases, Galaxy should run jobs in the shell directly, changing that to Singularity requires some different settings. The file is a good example in how you can quickly overwrite settings and be able to reuse it for different -occasions (remember that by concatinating this file behind +occasions (remember that by concatenating this file behind HTCondor, Slurm, or Pulsar enables Singularity the same way). Another example would be to create a custom `docker-compose.debug.yml` file that could be used to enable some debug flags or @@ -289,7 +289,7 @@ Have a look inside the `tests` folder. There you find the containers that run the tests and their docker-compose files. The containers are essentially just a wrapper around the test tools to simplify using them. Running a tests is the same as extending -any other part of the setup: Just concatinate the test file at the end. +any other part of the setup: Just concatenate the test file at the end. To run, for example, some Planemo Worklow tests against a Galaxy installation that is connected to a HTCondor cluster using Singularity, just enter: `docker-compose -f docker-compose.yml -f docker-compose.htcondor.yml @@ -300,7 +300,7 @@ This returns the exit code of the test container (should be 0 if successful), which you could use for further automation. The tests are run using GitHub Actions on every commit. So feel free to inspect -the `.github/workflows/compose-v2.yml` file for more test cases and get inspired +the `.github/workflows/compose.yml` file for more test cases and get inspired by them :) ### Planemo workflow tests @@ -313,7 +313,7 @@ by setting the `WORKFLOWS` env variable to a comma separated list of paths to so ### Selenium tests The Selenium tests simulate a real user that is accessing Galaxy through the -browser to performe some actions. For that it uses a headless Chrome to runs the +browser to perform some actions. For that it uses a headless Chrome to runs the tests from the [Galaxy repo](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy_test/selenium). The GitHub Actions currently just run a few of those. To select more tests, set the env variable `TESTS` to a comma separated list (like `TESTS=navigates_galaxy.py,login.py`). @@ -330,7 +330,7 @@ which tests we have excluded (at least for now). Tool specific configuration can be applied via `base_config.yml` or the following environment variables: * `GALAXY_CONFIG_` -* `GALAXY_UWSGI_CONFIG_` +* `GRAVITY_CONFIG_` * `NGINX_CONFIG_` * `PULSAR_CONFIG_` * `HTCONDOR_MASTER_CONFIG_` @@ -354,7 +354,7 @@ The following are settings specific to this docker-compose setup: | Variable | Description | |---------------------------|--------------------------------------------------------------------------------------------------------------------| | `NGINX_OVERWRITE_CONFIG` | Also see `GALAXY_OVERWRITE_CONFIG`. | -| `NGINX_UWSGI_READ_TIMEOUT` | Determines how long Nginx will wait (in seconds) for Galaxy to respond to a request until it times out. Defaults to 180 seconds. | +| `NGINX_PROXY_READ_TIMEOUT` | Determines how long Nginx will wait (in seconds) for Galaxy to respond to a request until it times out. Defaults to 180 seconds. | ### Pulsar | Variable | Description | @@ -378,7 +378,7 @@ The following are settings specific to this docker-compose setup: | `GALAXY_K8S_DOCKER_REPO_DEFAULT` | The Docker Repo/Registry to use if the resolver could not resolve the proper image for a job. Defaults to `docker.io`. | | `GALAXY_K8S_DOCKER_OWNER_DEFAULT` | The Owner/Username to use if the resolver could not resolve the proper image for a job. Is not set by default. | | `GALAXY_K8S_DOCKER_IMAGE_DEFAULT` | The Image to use if the resolver could not resolve the proper image for a job. Defaults to `ubuntu`. | -| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `20.04`. | +| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `22.04`. | ### HTCondor | Variable | Description | @@ -394,18 +394,20 @@ The following are settings specific to this docker-compose setup: | `SLURM_NODE_MEMORY` | Amount of memory per node. Defaults to 1024. | | `SLURM_NODE_HOSTNAME` | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. | -### Github Workflow Tests (Branch 20.09) -| Setup | bioblend | workflow ard | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium | -|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------| -| Galaxy Base | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | -| Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | -| HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | -| Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | -| Pulsar | :heavy_check_mark: | :x: | :x: | :x: | :heavy_check_mark: | -| k8s | :x: | :x: | :x: | :x: | :x: | -| Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | -| Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | -| HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | +### Github Workflow Tests (Branch 24.1) +| Setup | bioblend | workflow ard | workflow quality_control | workflow wf3-shed-tools (example1) | selenium | +|------------------------|--------------------|--------------------|--------------------------|------------------------------------|--------------------| +| Galaxy Base | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | +| HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Pulsar-REST | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | +| Pulsar-MQ | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | +| k8s | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Singularity | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Pulsar-MQ + Singularity| :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | +| Slurm + Singularity | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| HTCondor + Singularity | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Implemented: :heavy_check_mark: diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile index ff7c91cce..1319623df 100644 --- a/compose/base-images/galaxy-cluster-base/Dockerfile +++ b/compose/base-images/galaxy-cluster-base/Dockerfile @@ -5,23 +5,25 @@ ARG IMAGE_TAG=latest FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG # Base dependencies -RUN apt update && apt install --no-install-recommends gnupg2 -y \ +RUN apt update && apt install --no-install-recommends gnupg2 curl -y \ && /usr/bin/common_cleanup.sh # Install HTCondor ENV DEBIAN_FRONTEND noninteractive -RUN apt update && apt install --no-install-recommends htcondor -y \ - && rm /etc/condor/condor_config.local \ +RUN curl -fsSL https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-current-Key | apt-key add - \ + && echo "deb https://research.cs.wisc.edu/htcondor/repo/ubuntu/current jammy main" >> /etc/apt/sources.list \ + && apt update && apt install --no-install-recommends htcondor -y \ + && rm -f /etc/condor/condor_config.local \ && /usr/bin/common_cleanup.sh # Install Slurm client -ENV MUNGER_USER=munge \ +ENV MUNGE_USER=munge \ MUNGE_UID=1200 \ MUNGE_GID=1200 -RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ - && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ - && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ - && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ +RUN groupadd -r $MUNGE_USER -g $MUNGE_GID \ + && useradd -u $MUNGE_UID -r -g $MUNGE_USER $MUNGE_USER \ + && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu jammy main" >> /etc/apt/sources.list \ + && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu jammy main" >> /etc/apt/sources.list \ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \ && apt update \ && apt install --no-install-recommends python3-distutils slurm-client slurmd slurmctld slurm-drmaa1 -y \ diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile index 6cf33a516..6cf59cfb0 100644 --- a/compose/base-images/galaxy-container-base/Dockerfile +++ b/compose/base-images/galaxy-container-base/Dockerfile @@ -1,32 +1,33 @@ -FROM buildpack-deps:20.04 as build_singularity +FROM buildpack-deps:22.04 as build_apptainer COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh -# Install Go (only needed for building singularity) -ENV GO_VERSION=1.13 -RUN apt update && apt install --no-install-recommends cryptsetup-bin uuid-dev -y \ +# Install Go (only needed for building apptainer) +ENV GO_VERSION=1.22.7 +RUN apt update && apt install --no-install-recommends cryptsetup-bin uuid-dev libseccomp-dev libfuse-dev libfuse3-dev -y \ && wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz \ && tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz \ && rm go${GO_VERSION}.linux-amd64.tar.gz \ && /usr/bin/common_cleanup.sh ENV PATH=/usr/local/go/bin:${PATH} -ENV SINGULARITY_VERSION=3.5.3 -RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-${SINGULARITY_VERSION}.tar.gz \ - && tar -xzf singularity-${SINGULARITY_VERSION}.tar.gz \ - && cd singularity \ - && ./mconfig \ +ENV APPTAINER_VERSION=1.3.4 +RUN wget https://github.com/apptainer/apptainer/releases/download/v${APPTAINER_VERSION}/apptainer-${APPTAINER_VERSION}.tar.gz \ + && mkdir -p apptainer \ + && tar -xzf apptainer-${APPTAINER_VERSION}.tar.gz --strip-components=1 -C apptainer \ + && cd apptainer \ + && ./mconfig --with-suid \ && make -C builddir \ && /usr/bin/common_cleanup.sh # --- Final image --- -FROM ubuntu:20.04 as final +FROM ubuntu:22.04 as final COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Base dependencies -RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \ +RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools tzdata -y \ && /usr/bin/common_cleanup.sh # Install Docker @@ -34,11 +35,11 @@ RUN apt update \ && apt install --no-install-recommends docker.io -y \ && /usr/bin/common_cleanup.sh -# Install Singularity -COPY --from=build_singularity /singularity /singularity +# Install Apptainer +COPY --from=build_apptainer /apptainer /apptainer RUN apt update && apt install --no-install-recommends make -y \ - && make -C /singularity/builddir install \ + && make -C /apptainer/builddir install \ && apt remove make -y \ - && rm -rf /singularity \ - && sed -e '/bind path = \/etc\/localtime/s/^/#/g' -i /usr/local/etc/singularity/singularity.conf \ + && rm -rf /apptainer \ + && sed -e '/bind path = \/etc\/localtime/s/^/#/g' -i /usr/local/etc/apptainer/apptainer.conf \ && /usr/bin/common_cleanup.sh diff --git a/compose/base_config.yml b/compose/base_config.yml index 46f37dc7c..324220a10 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -1,35 +1,45 @@ -galaxy_uwsgi: - socket: :5555 - buffer-size: 16384 - processes: 1 - threads: 4 - offload-threads: 2 - static-map: /static=static - static-map: /favicon.ico=static/favicon.ico - master: false - virtualenv: .venv - pythonpath: lib - thunder-lock: true - die-on-term: true - hook-master-start: unix_signal:2 gracefully_kill_them_all - hook-master-start: unix_signal:15 gracefully_kill_them_all - enable-threads: true +gravity: + process_manager: supervisor + galaxy_root: /galaxy + virtualenv: /galaxy/.venv + gunicorn: + enable: True + bind: 0.0.0.0:5555 + workers: 2 + celery: + enable: true + enable_beat: true + concurrency: 2 + handlers: + handler: + processes: 2 + pools: + - job-handlers + - workflow-schedulers galaxy: tool_dependency_dir: /tool_deps tool_data_table_config_path: /cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml + enable_celery_tasks: true + celery_conf: + result_backend: redis://redis:6379/0 pulsar: conda_auto_init: True conda_auto_install: True tool_dependency_dir: dependencies - dependency_resolvers_config_file: config/dependency_resolvers_conf.xml + dependency_resolution: + resolvers: + - type: conda + auto_init: true + auto_install: true + - type: conda + versionless: true # Probably needs more polishing, but at least it works.. slurm: - ControlMachine: "slurmctld" + SlurmctldHost: "slurmctld" AuthType: "auth/munge" - CacheGroups: "0" CryptoType: "crypto/munge" MpiDefault: "none" ProctrackType: "proctrack/pgid" @@ -49,13 +59,11 @@ slurm: SlurmctldTimeout: "120" SlurmdTimeout: "300" Waittime: "0" - FastSchedule: "1" SchedulerType: "sched/backfill" - SchedulerPort: "7321" SelectType: "select/cons_res" SelectTypeParameters: "CR_Core_Memory" AccountingStorageType: "accounting_storage/none" - AccountingStoreJobComment: "YES" + AccountingStoreFlags: "job_comment" ClusterName: "Cluster" JobCompType: "jobcomp/none" JobAcctGatherFrequency: "30" @@ -70,11 +78,20 @@ htcondor_galaxy: ALLOW_READ: "*" ALLOW_WRITE: "*" ALLOW_CLIENT: "*" + ALLOW_DAEMON: "*" ALLOW_NEGOTIATOR: "*" DAEMON_LIST: "MASTER, SCHEDD" UID_DOMAIN: "galaxy" DISCARD_SESSION_KEYRING_ON_STARTUP: "False" TRUST_UID_DOMAIN: "true" + SEC_PASSWORD_FILE: "/var/lib/condor/pool_password" + SEC_DAEMON_AUTHENTICATION: "REQUIRED" + SEC_DAEMON_INTEGRITY: "REQUIRED" + SEC_DAEMON_AUTHENTICATION_METHODS: "PASSWORD" + SEC_NEGOTIATOR_AUTHENTICATION: "REQUIRED" + SEC_NEGOTIATOR_INTEGRITY: "REQUIRED" + SEC_NEGOTIATOR_AUTHENTICATION_METHODS: "PASSWORD" + SEC_CLIENT_AUTHENTICATION_METHODS: "FS, PASSWORD" htcondor_master: BASE_CGROUP: "" @@ -93,9 +110,18 @@ htcondor_master: ALLOW_READ_COLLECTOR: "*" ALLOW_READ_STARTD: "*" ALLOW_CLIENT: "*" + ALLOW_DAEMON: "*" DOCKER_IMAGE_CACHE_SIZE: "20" UID_DOMAIN: "galaxy" TRUST_UID_DOMAIN: "TRUE" + SEC_PASSWORD_FILE: "/var/lib/condor/pool_password" + SEC_DAEMON_AUTHENTICATION: "REQUIRED" + SEC_DAEMON_INTEGRITY: "REQUIRED" + SEC_DAEMON_AUTHENTICATION_METHODS: "PASSWORD" + SEC_NEGOTIATOR_AUTHENTICATION: "REQUIRED" + SEC_NEGOTIATOR_INTEGRITY: "REQUIRED" + SEC_NEGOTIATOR_AUTHENTICATION_METHODS: "PASSWORD" + SEC_CLIENT_AUTHENTICATION_METHODS: "FS, PASSWORD" htcondor_executor: CONDOR_HOST: "htcondor-master" @@ -110,6 +136,7 @@ htcondor_executor: ALLOW_READ: "*" ALLOW_WRITE: "*" ALLOW_CLIENT: "*" + ALLOW_DAEMON: "*" ALLOW_NEGOTIATOR_SCHEDD: "*" ALLOW_WRITE_COLLECTOR: "*" ALLOW_WRITE_STARTD: "*" @@ -117,3 +144,11 @@ htcondor_executor: ALLOW_READ_STARTD: "*" UID_DOMAIN: "galaxy" SCHED_NAME: "htcondor-master" + SEC_PASSWORD_FILE: "/var/lib/condor/pool_password" + SEC_DAEMON_AUTHENTICATION: "REQUIRED" + SEC_DAEMON_INTEGRITY: "REQUIRED" + SEC_DAEMON_AUTHENTICATION_METHODS: "PASSWORD" + SEC_NEGOTIATOR_AUTHENTICATION: "REQUIRED" + SEC_NEGOTIATOR_INTEGRITY: "REQUIRED" + SEC_NEGOTIATOR_AUTHENTICATION_METHODS: "PASSWORD" + SEC_CLIENT_AUTHENTICATION_METHODS: "FS, PASSWORD" diff --git a/compose/docker-compose.htcondor.yml b/compose/docker-compose.htcondor.yml index 35084397f..2281a4ced 100644 --- a/compose/docker-compose.htcondor.yml +++ b/compose/docker-compose.htcondor.yml @@ -1,6 +1,5 @@ # Extend Galaxy to run jobs using HTCondor. # Example: `docker-compose -f docker-compose.yml -f docker-compose.htcondor.yml up` -version: "3.7" services: galaxy-configurator: environment: @@ -14,6 +13,7 @@ services: hostname: htcondor-master environment: - HTCONDOR_TYPE=master + - HTCONDOR_POOL_PASSWORD=123456789changeme volumes: - ${EXPORT_DIR:-./export}/htcondor:/config networks: @@ -25,6 +25,7 @@ services: environment: - HTCONDOR_TYPE=executor - CONDOR_HOST=htcondor-master + - HTCONDOR_POOL_PASSWORD=123456789changeme volumes: - ${EXPORT_DIR:-./export}/htcondor:/config - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database diff --git a/compose/docker-compose.k8s.yml b/compose/docker-compose.k8s.yml index dd8536953..305f54494 100644 --- a/compose/docker-compose.k8s.yml +++ b/compose/docker-compose.k8s.yml @@ -2,7 +2,6 @@ # This will set up Kubernetes using kind (https://kind.sigs.k8s.io). # Note that this extension is not compatible with others like Pulsar, HTCondor, Singularity, etc. # Example: `docker-compose -f docker-compose.yml -f docker-compose.k8s.yml up` -version: "3.7" services: galaxy-configurator: environment: diff --git a/compose/docker-compose.pulsar.mq.yml b/compose/docker-compose.pulsar.mq.yml index 0a02ffc23..1311eb27e 100644 --- a/compose/docker-compose.pulsar.mq.yml +++ b/compose/docker-compose.pulsar.mq.yml @@ -2,7 +2,6 @@ # for communicating with Galaxy. # Requirements: `docker-compose.pulsar.yml` # Example: `docker-compose -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml up` -version: "3.7" services: galaxy-configurator: environment: diff --git a/compose/docker-compose.pulsar.yml b/compose/docker-compose.pulsar.yml index 12f1d4f14..e56d58df1 100644 --- a/compose/docker-compose.pulsar.yml +++ b/compose/docker-compose.pulsar.yml @@ -3,10 +3,9 @@ # Galaxy will send all the needed files for Pulsar, and Pulsar # will handle the rest locally on its side. # This docker-compose file enables for Galaxy and Pulsar to -# communicate over HTTP. To enable the MQ, concatinate the +# communicate over HTTP. To enable the MQ, concatenate the # docker-compose.pulsar.mq.yml after this one. # Example: `docker-compose -f docker-compose.yml -f docker-compose.pulsar.yml up` -version: "3.7" services: galaxy-configurator: environment: @@ -26,5 +25,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/pulsar/config:/pulsar/config - ${EXPORT_DIR:-./export}/pulsar/dependencies:/pulsar/dependencies + - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database + - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data networks: - galaxy diff --git a/compose/docker-compose.singularity.yml b/compose/docker-compose.singularity.yml index 8f556a34e..a81bb7735 100644 --- a/compose/docker-compose.singularity.yml +++ b/compose/docker-compose.singularity.yml @@ -5,7 +5,6 @@ # Examples: # * `docker-compose -f docker-compose.yml -f docker-compose.singularity.yml up` # * `docker-compose -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml up` -version: "3.7" services: galaxy-configurator: environment: diff --git a/compose/docker-compose.slurm.yml b/compose/docker-compose.slurm.yml index 4cf6e29cb..6be800595 100644 --- a/compose/docker-compose.slurm.yml +++ b/compose/docker-compose.slurm.yml @@ -1,6 +1,5 @@ # Extend Galaxy to run jobs using Slurm. # Example: `docker-compose -f docker-compose.yml -f docker-compose.slurm.yml up` -version: "3.7" services: galaxy-configurator: environment: @@ -9,18 +8,18 @@ services: - SLURM_NODE_COUNT=${SLURM_NODE_COUNT:-1} - SLURM_NODE_HOSTNAME=compose_slurm_node volumes: - - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm-llnl + - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm galaxy-server: volumes: - ${EXPORT_DIR:-./export}/munge:/etc/munge - - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm-llnl + - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm slurmctld: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-slurm:${IMAGE_TAG:-latest} build: galaxy-slurm command: ["slurmctld"] hostname: slurmctld volumes: - - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm-llnl + - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm - ${EXPORT_DIR:-./export}/munge:/etc/munge networks: - galaxy @@ -28,7 +27,7 @@ services: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-slurm-node-discovery:${IMAGE_TAG:-latest} build: galaxy-slurm-node-discovery volumes: - - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm-llnl + - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm - /var/run/docker.sock:/var/run/docker.sock slurm_node: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-slurm:${IMAGE_TAG:-latest} @@ -44,7 +43,7 @@ services: - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv - ${EXPORT_DIR:-./export}/tool_deps:/tool_deps - - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm-llnl + - ${EXPORT_DIR:-./export}/slurm_config:/etc/slurm - ${EXPORT_DIR:-./export}/munge:/etc/munge - /var/run/docker.sock:/var/run/docker.sock networks: diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index b1246757a..17f2a10a6 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.7" services: galaxy-server: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-server:${IMAGE_TAG:-latest} @@ -8,10 +7,9 @@ services: - GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org - GALAXY_DEFAULT_ADMIN_PASSWORD=password - GALAXY_DEFAULT_ADMIN_KEY=fakekey + - HTCONDOR_POOL_PASSWORD=123456789changeme hostname: galaxy-server privileged: True - ports: - - 5555:5555 volumes: # This is the directory where all your files from Galaxy will be stored # on your host system @@ -20,6 +18,7 @@ services: depends_on: - postgres - rabbitmq + - redis networks: - galaxy # The galaxy-configurator is responsible for the whole configuration of @@ -48,7 +47,7 @@ services: - ./galaxy-configurator/templates:/templates # The database for Galaxy postgres: - image: postgres:12 + image: postgres:15 hostname: postgres environment: - POSTGRES_PASSWORD=chaopagoosaequuashie @@ -87,5 +86,14 @@ services: - ${EXPORT_DIR:-./export}/rabbitmq:/var/lib/rabbitmq:delegated networks: - galaxy + # Backend for Celery + redis: + image: redis:alpine + container_name: galaxy-redis + hostname: redis + volumes: + - ${EXPORT_DIR:-./export}/redis:/data:delegated + networks: + - galaxy networks: galaxy: diff --git a/compose/galaxy-configurator/Dockerfile b/compose/galaxy-configurator/Dockerfile index 8efa5611c..5cf3eee5e 100644 --- a/compose/galaxy-configurator/Dockerfile +++ b/compose/galaxy-configurator/Dockerfile @@ -1,6 +1,6 @@ -FROM alpine:3.11 +FROM alpine:3.17 -RUN apk add --no-cache bash python3 \ +RUN apk add --no-cache bash python3 py3-pip \ && pip3 install j2cli[yaml] jinja2-ansible-filters COPY ./templates /templates diff --git a/compose/galaxy-configurator/customize.py b/compose/galaxy-configurator/customize.py index fd1f33e25..77f5c297d 100644 --- a/compose/galaxy-configurator/customize.py +++ b/compose/galaxy-configurator/customize.py @@ -25,7 +25,7 @@ def alter_context(context): translations = { "GALAXY_CONFIG_": "galaxy", - "GALAXY_UWSGI_CONFIG_": "galaxy_uwsgi", + "GRAVITY_CONFIG_": "gravity", "GALAXY_JOB_METRICS_": "galaxy_job_metrics", "NGINX_CONFIG_": "nginx", "SLURM_CONFIG_": "slurm", diff --git a/compose/galaxy-configurator/run.sh b/compose/galaxy-configurator/run.sh index fd37d78a5..e2e72d2f7 100755 --- a/compose/galaxy-configurator/run.sh +++ b/compose/galaxy-configurator/run.sh @@ -3,7 +3,7 @@ # Set default config dirs export GALAXY_CONF_DIR=${GALAXY_CONF_DIR:-/galaxy/config} \ NGINX_CONF_DIR=${NGINX_CONF_DIR:-/etc/nginx/} \ - SLURM_CONF_DIR=${SLURM_CONF_DIR:-/etc/slurm-llnl} \ + SLURM_CONF_DIR=${SLURM_CONF_DIR:-/etc/slurm} \ HTCONDOR_CONF_DIR=${HTCONDOR_CONF_DIR:-/htcondor} \ PULSAR_CONF_DIR=${PULSAR_CONF_DIR:-/pulsar/config} \ KIND_CONF_DIR=${KIND_CONF_DIR:-/kind} @@ -70,7 +70,7 @@ fi if [ "$PULSAR_OVERWRITE_CONFIG" != "true" ]; then echo "PULSAR_OVERWRITE_CONFIG is not true. Skipping configuration of Pulsar" else - pulsar_configs=( "server.ini" "app.yml" "dependency_resolvers_conf.xml" ) + pulsar_configs=( "server.ini" "app.yml" ) for conf in "${pulsar_configs[@]}"; do echo "Configuring $conf" @@ -138,7 +138,7 @@ if [ ! -f /base_config.yml ]; then touch /base_config.yml fi -galaxy_configs=( "job_conf.xml" "galaxy.yml" "job_metrics.xml" "container_resolvers_conf.xml" "GALAXY_PROXY_PREFIX.txt" ) +galaxy_configs=( "job_conf.xml" "galaxy.yml" "job_metrics.xml" "container_resolvers_conf.yml" "dependency_resolvers_conf.xml" "GALAXY_PROXY_PREFIX.txt" ) for conf in "${galaxy_configs[@]}"; do echo "Configuring $conf" diff --git a/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.xml.j2 deleted file mode 100644 index 8e94d6962..000000000 --- a/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.xml.j2 +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.yml.j2 b/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.yml.j2 new file mode 100644 index 000000000..b1a39418b --- /dev/null +++ b/compose/galaxy-configurator/templates/galaxy/container_resolvers_conf.yml.j2 @@ -0,0 +1,83 @@ +# Resolvers that are potentially used by default are uncommented (comments describe under +# which premises they are in the defaults). + +# Note that commented yaml does not have a space after the # +# while additional explanations do. + +# Explicit container resolvers +# ============================ + +# get a container description (URI) for an explicit singularity container requirement +- type: explicit_singularity + +# get a cached container description (path) for singularity +# pulls the container into a cache directory if not yet there +- type: cached_explicit_singularity + # set the cache directory for storing images + #cache_directory: database/container_cache/singularity/explicit + +# Mulled container resolvers +# ========================== + +# The following uncommented container resolvers are in the defaults +# if ``enable_mulled_containers`` is set in ``galaxy.yml`` (which is the default). + +# get a container description for a cached mulled singularity container +# checks if the image file exists in `cache_directory` +- type: cached_mulled_singularity + # + #cache_directory: database/container_cache/singularity/mulled + # + # the method for caching directory listings (not the method for image caching) + # can be uncached or dir_mtime (the latter only determines the directory listing + # if the modification time of the directory changed) + #cache_directory_cacher_type: uncached + +# Resolves container images from quay.io/NAMESPACE/MULLED_HASH where the +# mulled hash describes which packages and versions should be in the container +# +# These resolvers are generally listed after the cached_* resolvers, so that images +# are not pulled if they are already cached. +# +# When pulling the image file will be stored in the configured cache dir. +# If auto_install is True the result will point to the cached image file +# and to quay.io/NAMESPACE/MULLED_HASH otherwise. +- type: mulled_singularity + auto_install: False + #namespace: biocontainers + # In addition to the arguments of `mulled` there are cache_directory + # and cache_directory_cacher_type. See the description at `cached_explicit_singularity` + # and note the minor difference in the default for `cache_directory` + #cache_directory: database/container_cache/singularity/mulled + #cache_directory_cacher_type: uncached + +# Building container resolvers +# ---------------------------- +# +# The following uncommented container resolvers are included in the default +# if ``docker`` is available + +- type: build_mulled_singularity + auto_install: False + #hash_func: v2 + #cache_directory: database/container_cache/singularity/mulled + #cache_directory_cacher_type: uncached + +# Other explicit container resolvers +# ---------------------------------- + +#-type: fallback_singularity + #identifier: A_VALID_CONTAINER_IDENTIFIER +#-type: fallback_no_requirements_singularity + #identifier: A_VALID_CONTAINER_IDENTIFIER +#-type: requires_galaxy_environment_singularity + #identifier: A_VALID_CONTAINER_IDENTIFIER + +# The mapping container resolver allows to specify a list of mappings from tools +# (tool_id) to containers (type and identifier). + +#-type: mapping + #mappings: + #- container_type: singularity + #tool_id: A_TOOL_ID + #identifier: A_VALID_CONTAINER_IDENTIFIER diff --git a/compose/galaxy-configurator/templates/galaxy/dependency_resolvers_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/dependency_resolvers_conf.xml.j2 new file mode 100644 index 000000000..45988b0bb --- /dev/null +++ b/compose/galaxy-configurator/templates/galaxy/dependency_resolvers_conf.xml.j2 @@ -0,0 +1,18 @@ + + {% if GALAXY_DEPENDENCY_RESOLUTION != 'singularity' %} + + + + + + + + + + + {% endif %} + diff --git a/compose/galaxy-configurator/templates/galaxy/galaxy.yml.j2 b/compose/galaxy-configurator/templates/galaxy/galaxy.yml.j2 index a1ca72778..f221c5445 100644 --- a/compose/galaxy-configurator/templates/galaxy/galaxy.yml.j2 +++ b/compose/galaxy-configurator/templates/galaxy/galaxy.yml.j2 @@ -1,19 +1,14 @@ -uwsgi: -{{ galaxy_uwsgi | to_nice_yaml(indent=2) | indent(2, first=True) }} - - {% if GALAXY_PROXY_PREFIX %} - mount: /{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }}=galaxy.webapps.galaxy.buildapp:uwsgi_app() - manage-script-name: true - {% else %} - module: galaxy.webapps.galaxy.buildapp:uwsgi_app() - manage-script-name: false - {% endif %} - +gravity: +{{ gravity | to_nice_yaml(indent=2) | indent(2, first=True) }} galaxy: {{ galaxy | to_nice_yaml(indent=2) | indent(2, first=True) }} + {% if GALAXY_PROXY_PREFIX %} + galaxy_url_prefix: /{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }} + {% endif %} + {% if GALAXY_DEPENDENCY_RESOLUTION == 'singularity' %} enable_mulled_containers: true - containers_resolvers_config_file: container_resolvers_conf.xml + containers_resolvers_config_file: container_resolvers_conf.yml {% endif %} diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 index 9b1ca49ed..a68cd09f0 100644 --- a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 +++ b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 @@ -26,6 +26,7 @@ {% endif -%} + @@ -34,16 +35,19 @@ {% if GALAXY_DEPENDENCY_RESOLUTION == 'singularity' -%} /home/galaxy + C + /tmp/singularity + /tmp true {% if GALAXY_JOB_RUNNER == 'local' -%} - {{ EXPORT_DIR | regex_replace("^.", "") }}/$galaxy_root:$galaxy_root:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$tool_directory:$tool_directory:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$job_directory:$job_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$working_directory:$working_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$default_file_path:$default_file_path:rw + {{ EXPORT_DIR | regex_replace("^.", "") }}/$galaxy_root:$galaxy_root:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$galaxy_root/database/tmp:$galaxy_root/database/tmp:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$tool_directory:$tool_directory:ro,{{ EXPORT_DIR | regex_replace("^.", "") }}/$job_directory:$job_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$working_directory:$working_directory:rw,{{ EXPORT_DIR | regex_replace("^.", "") }}/$default_file_path:$default_file_path:rw {% endif -%} {% elif GALAXY_DEPENDENCY_RESOLUTION == 'docker' -%} true false {% if GALAXY_JOB_RUNNER == 'local' -%} - {{ HOST_EXPORT_DIR }}/$galaxy_root:$galaxy_root:ro,{{ HOST_EXPORT_DIR }}/$tool_directory:$tool_directory:ro,{{ HOST_EXPORT_DIR }}/$job_directory:$job_directory:rw,{{ HOST_EXPORT_DIR }}/$working_directory:$working_directory:rw,{{ HOST_EXPORT_DIR }}/$default_file_path:$default_file_path:rw + {{ HOST_EXPORT_DIR }}/$galaxy_root:$galaxy_root:ro,{{ HOST_EXPORT_DIR }}/$galaxy_root/database/tmp:$galaxy_root/database/tmp:rw,{{ HOST_EXPORT_DIR }}/$tool_directory:$tool_directory:ro,{{ HOST_EXPORT_DIR }}/$job_directory:$job_directory:rw,{{ HOST_EXPORT_DIR }}/$working_directory:$working_directory:rw,{{ HOST_EXPORT_DIR }}/$default_file_path:$default_file_path:rw {% endif -%} {% elif not GALAXY_JOB_RUNNER.startswith('pulsar') and GALAXY_JOB_RUNNER != 'k8s' -%} @@ -59,8 +63,8 @@ {% if GALAXY_JOB_RUNNER == 'k8s' -%} {{ GALAXY_K8S_DOCKER_REPO_DEFAULT | default('docker.io') }} {% if GALAXY_K8S_DOCKER_OWNER_DEFAULT -%}{{ GALAXY_K8S_DOCKER_OWNER_DEFAULT }}{% endif -%} - {{ GALAXY_K8S_DOCKER_IMAGE_DEFAULT | default('ubuntu') }} - {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('20.04') }} + {{ GALAXY_K8S_DOCKER_IMAGE_DEFAULT | default('python') }} + {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('3.10.15') }} true {% endif -%} diff --git a/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 b/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 index fc348c448..256a6d927 100644 --- a/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 +++ b/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 @@ -14,7 +14,10 @@ http { gzip_buffers 16 8k; # allow up to 3 minutes for Galaxy to respond to slow requests before timing out - uwsgi_read_timeout {{ NGINX_UWSGI_READ_TIMEOUT | default(180, true) }}; + proxy_read_timeout {{ NGINX_PROXY_READ_TIMEOUT | default(180, true) }}; + + proxy_buffers 8 16k; + proxy_buffer_size 16k; # maximum file upload size client_max_body_size 10g; @@ -28,11 +31,13 @@ http { set $galaxy_static /export/galaxy/static; set $galaxy_root /export/galaxy; - # proxy all requests not matching other locations to uWSGI + # proxy all requests not matching other locations to gunicorn location /{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }} { - uwsgi_pass galaxy-server:5555; - include uwsgi_params; - uwsgi_param UWSGI_SCHEME /{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }}; + proxy_pass http://galaxy-server:5555; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; } # serve framework static content @@ -49,10 +54,11 @@ http { expires 24h; } - # serve visualization and interactive environment plugin static content - location ~ ^{{ GALAXY_PROXY_PREFIX | regex_replace("/$", "") }}/plugins/(?.+?)/(?.+?)/static/(?.*?)$ { - alias $galaxy_root/config/plugins/$plug_type/$vis_name/static/$static_file; - expires 24; + # serve visualization plugin static content + location ~ ^{{ GALAXY_PROXY_PREFIX | regex_replace("/$", "") }}/plugins/(?[^/]+?)/((?[^/_]*)_?)?(?[^/]*?)/static/(?.*?)$ { + alias $galaxy_root/config/plugins/$plug_type/; + try_files $vis_d/${vis_d}_${vis_name}/static/$static_file + $vis_d/static/$static_file =404; } rewrite ^/{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }}$ /{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }}/ last; diff --git a/compose/galaxy-configurator/templates/pulsar/dependency_resolvers_conf.xml.j2 b/compose/galaxy-configurator/templates/pulsar/dependency_resolvers_conf.xml.j2 deleted file mode 100644 index 87f9657cf..000000000 --- a/compose/galaxy-configurator/templates/pulsar/dependency_resolvers_conf.xml.j2 +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index ac49a826b..41a3ea389 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -2,22 +2,22 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:20.04 as galaxy_dependencies +FROM buildpack-deps:22.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.09 +ARG GALAXY_RELEASE=release_24.1 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy -ENV GALAXY_ROOT=/galaxy -ENV GALAXY_LIBRARY=$GALAXY_ROOT/lib +ENV GALAXY_ROOT_DIR=/galaxy +ENV GALAXY_LIBRARY=$GALAXY_ROOT_DIR/lib # Download Galaxy source, but only keep necessary dependencies -RUN mkdir "${GALAXY_ROOT}" \ - && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ - && cd $GALAXY_ROOT \ +RUN mkdir "${GALAXY_ROOT_DIR}" \ + && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT_DIR \ + && cd $GALAXY_ROOT_DIR \ && ls . | grep -v "lib" | xargs rm -rf \ - && cd $GALAXY_ROOT/lib \ + && cd $GALAXY_ROOT_DIR/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ - && cd $GALAXY_ROOT/lib/galaxy \ + && cd $GALAXY_ROOT_DIR/lib/galaxy \ && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf @@ -30,7 +30,7 @@ ENV GALAXY_USER=galaxy \ GALAXY_UID=1450 \ GALAXY_GID=1450 \ GALAXY_HOME=/home/galaxy \ - GALAXY_ROOT=/galaxy + GALAXY_ROOT_DIR=/galaxy RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \ @@ -49,9 +49,11 @@ ENV CONDOR_CPUS=1 \ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup \ && echo 'Acquire::http::Timeout "20";' > /etc/apt/apt.conf.d/98AcquireTimeout \ && echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/99AcquireRetries \ - && apt-get update -qq && apt-get install -y --no-install-recommends locales \ + && apt-get update -qq && apt-get install -y --no-install-recommends locales gnupg2 curl \ && locale-gen en_US.UTF-8 && dpkg-reconfigure locales \ - && apt-get install -y --no-install-recommends \ + && curl -fsSL https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-current-Key | apt-key add - \ + && echo "deb https://research.cs.wisc.edu/htcondor/repo/ubuntu/current jammy main" >> /etc/apt/sources.list \ + && apt-get update -qq && apt-get install -y --no-install-recommends \ supervisor \ htcondor \ wget \ @@ -62,11 +64,11 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup \ ADD supervisord.conf /etc/supervisord.conf # Copy Galaxy dependencies -COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $GALAXY_ROOT +COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT_DIR $GALAXY_ROOT_DIR COPY start.sh /usr/bin/start.sh RUN apt update && apt install python3 -y -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ENTRYPOINT /usr/bin/start.sh diff --git a/compose/galaxy-htcondor/start.sh b/compose/galaxy-htcondor/start.sh index dc1236b16..973330b1a 100755 --- a/compose/galaxy-htcondor/start.sh +++ b/compose/galaxy-htcondor/start.sh @@ -7,5 +7,6 @@ until [ ! -f /config/configurator.lock ] && echo Lock released; do done; cp -f "/config/$HTCONDOR_TYPE.conf" /etc/condor/condor_config.local +condor_store_cred -p "$HTCONDOR_POOL_PASSWORD" -f /var/lib/condor/pool_password /usr/bin/supervisord diff --git a/compose/galaxy-kind/Dockerfile b/compose/galaxy-kind/Dockerfile index 514c2ea62..970550295 100644 --- a/compose/galaxy-kind/Dockerfile +++ b/compose/galaxy-kind/Dockerfile @@ -1,15 +1,15 @@ -FROM alpine:3.11 +FROM alpine:3.17 -ARG KIND_RELEASE=v0.8.1 -ARG KUBECTL_RELEASE=v1.18.3 +ARG KIND_RELEASE=v0.24.0 +ARG KUBECTL_RELEASE=v1.31.1 RUN apk add --no-cache docker RUN apk add --no-cache --virtual build-deps wget \ && apk add --no-cache bash \ - && wget -O /usr/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_RELEASE}/kind-linux-amd64 \ + && wget -O /usr/bin/kind https://kind.sigs.k8s.io/dl/${KIND_RELEASE}/kind-linux-amd64 \ && chmod +x /usr/bin/kind \ - && wget -O /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_RELEASE}/bin/linux/amd64/kubectl \ + && wget -O /usr/bin/kubectl https://dl.k8s.io/release/${KUBECTL_RELEASE}/bin/linux/amd64/kubectl \ && chmod +x /usr/bin/kubectl \ && apk del build-deps diff --git a/compose/galaxy-nginx/Dockerfile b/compose/galaxy-nginx/Dockerfile index 051684687..7ff6631fb 100644 --- a/compose/galaxy-nginx/Dockerfile +++ b/compose/galaxy-nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.17-alpine +FROM nginx:1.27-alpine COPY start.sh /usr/bin/start.sh diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index d789c2250..3b8b8f5a5 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -2,20 +2,20 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:20.04 as build_base +FROM buildpack-deps:22.04 as build_base ENV EXPORT_DIR=/export \ - GALAXY_ROOT=/galaxy \ + GALAXY_ROOT_DIR=/galaxy \ HTCONDOR_ROOT=/opt/htcondor -ENV GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ +ENV GALAXY_STATIC_DIR=$GALAXY_ROOT_DIR/static \ GALAXY_EXPORT=$EXPORT_DIR/galaxy \ - GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \ + GALAXY_CONFIG_DIR=$GALAXY_ROOT_DIR/config \ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \ - GALAXY_CONFIG_TOOL_PATH=$GALAXY_ROOT/tools \ - GALAXY_CONFIG_TOOL_DATA_PATH=$GALAXY_ROOT/tool-data \ - GALAXY_VIRTUAL_ENV=$GALAXY_ROOT/.venv \ - GALAXY_DATABASE_PATH=$GALAXY_ROOT/database + GALAXY_CONFIG_TOOL_PATH=$GALAXY_ROOT_DIR/tools \ + GALAXY_CONFIG_TOOL_DATA_PATH=$GALAXY_ROOT_DIR/tool-data \ + GALAXY_VIRTUAL_ENV=$GALAXY_ROOT_DIR/.venv \ + GALAXY_DATABASE_PATH=$GALAXY_ROOT_DIR/database ENV GALAXY_USER=galaxy \ GALAXY_GROUP=galaxy \ @@ -24,7 +24,7 @@ ENV GALAXY_USER=galaxy \ GALAXY_HOME=/home/galaxy ENV GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda \ - MINICONDA_VERSION=py38_4.9.2 + MINICONDA_VERSION=py310_24.5.0-0 RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \ @@ -50,24 +50,24 @@ RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSI FROM build_base as build_galaxy -ARG GALAXY_RELEASE=release_20.09 +ARG GALAXY_RELEASE=release_24.1 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Galaxy RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libssl-dev python3-dev python3-pip -y \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3 9 \ - && mkdir "${GALAXY_ROOT}" \ - && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ - && cd $GALAXY_ROOT \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \ + && mkdir "${GALAXY_ROOT_DIR}" \ + && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT_DIR \ + && cd $GALAXY_ROOT_DIR \ && ./scripts/common_startup.sh \ - && . $GALAXY_ROOT/.venv/bin/activate \ - && pip3 install drmaa psycopg2 pycurl pykube \ + && . $GALAXY_ROOT_DIR/.venv/bin/activate \ + && pip3 install drmaa psycopg2 pycurl pykube redis \ && pip3 install importlib-metadata importlib-resources pathlib2 ruamel.yaml.clib typing zipp \ && deactivate \ && rm -rf .ci .circleci .coveragerc .gitignore .travis.yml CITATION CODE_OF_CONDUCT.md CONTRIBUTING.md CONTRIBUTORS.md \ LICENSE.txt Makefile README.rst SECURITY_POLICY.md pytest.ini tox.ini \ - client contrib doc config/plugins lib/galaxy_test test test-data \ + contrib doc config/plugins lib/galaxy_test test test-data \ .venv/lib/node_modules .venv/src/node-v10.15.3-linux-x64 \ .venv/include/node .venv/bin/node .venv/bin/nodeenv \ && /usr/bin/common_cleanup.sh @@ -79,20 +79,20 @@ COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py ENV EXPORT_DIR=/export \ - GALAXY_ROOT=/galaxy \ + GALAXY_ROOT_DIR=/galaxy \ GALAXY_PYTHON=/usr/bin/python3 \ HTCONDOR_ROOT=/opt/htcondor -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_24.1} \ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ - GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ + GALAXY_STATIC_DIR=$GALAXY_ROOT_DIR/static \ GALAXY_EXPORT=$EXPORT_DIR/galaxy \ - GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \ + GALAXY_CONFIG_DIR=$GALAXY_ROOT_DIR/config \ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \ - GALAXY_CONFIG_TOOL_PATH=$GALAXY_ROOT/tools \ - GALAXY_CONFIG_TOOL_DATA_PATH=$GALAXY_ROOT/tool-data \ - GALAXY_VIRTUAL_ENV=$GALAXY_ROOT/.venv \ - GALAXY_DATABASE_PATH=$GALAXY_ROOT/database + GALAXY_CONFIG_TOOL_PATH=$GALAXY_ROOT_DIR/tools \ + GALAXY_CONFIG_TOOL_DATA_PATH=$GALAXY_ROOT_DIR/tool-data \ + GALAXY_VIRTUAL_ENV=$GALAXY_ROOT_DIR/.venv \ + GALAXY_DATABASE_PATH=$GALAXY_ROOT_DIR/database ENV GALAXY_USER=galaxy \ GALAXY_GROUP=galaxy \ @@ -111,7 +111,7 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 liblzma-dev libbz2-dev libpq-dev \ libcurl4-openssl-dev libssl-dev \ mercurial make netcat python3-dev python3-setuptools python3-pip \ - zlib1g-dev -y \ + zlib1g-dev sudo -y \ # Cython and wheel are needed to later install pysam.. && pip3 install Cython wheel \ && pip3 install pysam \ @@ -121,13 +121,22 @@ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 l RUN usermod -aG docker $GALAXY_USER # Make Python3 standard -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT} +COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT_DIR} ${GALAXY_ROOT_DIR} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_HOME} ${GALAXY_HOME} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda /etc/profile.d/conda.sh /etc/profile.d/conda.sh +# The following commands will be executed as the galaxy user +USER $GALAXY_USER + +# We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan) +ENV HOME=$GALAXY_HOME + +# Switch back to User root +USER root + COPY ./files/start.sh /usr/bin/start.sh EXPOSE 80 diff --git a/compose/galaxy-server/files/create_galaxy_user.py b/compose/galaxy-server/files/create_galaxy_user.py index 3f0faa56e..8d2b05f23 100644 --- a/compose/galaxy-server/files/create_galaxy_user.py +++ b/compose/galaxy-server/files/create_galaxy_user.py @@ -34,6 +34,7 @@ def add_user(sa_session, security_agent, email, password, key=None, username="ad api_key.key = key sa_session.add(api_key) sa_session.flush() + sa_session.commit() return user @@ -57,4 +58,4 @@ def add_user(sa_session, security_agent, email, password, key=None, username="ad sa_session = mapping.context security_agent = mapping.security_agent - add_user(sa_session, security_agent, options.user, options.password, key=options.key, username=options.username) \ No newline at end of file + add_user(sa_session, security_agent, options.user, options.password, key=options.key, username=options.username) diff --git a/compose/galaxy-server/files/start.sh b/compose/galaxy-server/files/start.sh index 860280856..0ea058c18 100755 --- a/compose/galaxy-server/files/start.sh +++ b/compose/galaxy-server/files/start.sh @@ -25,8 +25,8 @@ cp -rf $tools_dir/* $exp_dir echo "Initialization: Check if files already exist, export otherwise." -# Create initial $GALAXY_ROOT in $EXPORT_DIR if not already existent -mkdir -p "$EXPORT_DIR/$GALAXY_ROOT" +# Create initial $GALAXY_ROOT_DIR in $EXPORT_DIR if not already existent +mkdir -p "$EXPORT_DIR/$GALAXY_ROOT_DIR" declare -A exports=( ["$GALAXY_STATIC_DIR"]="$EXPORT_DIR/$GALAXY_STATIC_DIR" \ ["$GALAXY_CONFIG_TOOL_PATH"]="$EXPORT_DIR/$GALAXY_CONFIG_TOOL_PATH" \ @@ -109,8 +109,9 @@ if [ -f "/htcondor_config/galaxy.conf" ]; then echo "HTCondor config file found" cp -f "/htcondor_config/galaxy.conf" /etc/condor/condor_config.local + condor_store_cred -p "$HTCONDOR_POOL_PASSWORD" -f /var/lib/condor/pool_password echo "Starting HTCondor.." - service condor start + /usr/sbin/condor_master -b fi if [ -f /etc/munge/munge.key ]; then @@ -129,5 +130,5 @@ fi chown -RL "$GALAXY_USER:$GALAXY_GROUP" "$GALAXY_CONFIG_DIR" echo "Starting Galaxy now.." -cd "$GALAXY_ROOT" || { echo "Error: Could not change to $GALAXY_ROOT"; exit 1; } -"$GALAXY_VIRTUAL_ENV/bin/uwsgi" --yaml "$GALAXY_CONFIG_DIR/galaxy.yml" --uid "$GALAXY_UID" --gid "$GALAXY_GID" +cd "$GALAXY_ROOT_DIR" || { echo "Error: Could not change to $GALAXY_ROOT_DIR"; exit 1; } +sudo -E -u $GALAXY_USER "$GALAXY_VIRTUAL_ENV/bin/galaxy" --config-file "$GALAXY_CONFIG_FILE" diff --git a/compose/galaxy-slurm-node-discovery/Dockerfile b/compose/galaxy-slurm-node-discovery/Dockerfile index af2c6d5fa..558a328fb 100644 --- a/compose/galaxy-slurm-node-discovery/Dockerfile +++ b/compose/galaxy-slurm-node-discovery/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.11 +FROM alpine:3.17 RUN apk add curl jq diff --git a/compose/galaxy-slurm-node-discovery/run.sh b/compose/galaxy-slurm-node-discovery/run.sh index d80a50224..03e270ee5 100755 --- a/compose/galaxy-slurm-node-discovery/run.sh +++ b/compose/galaxy-slurm-node-discovery/run.sh @@ -5,17 +5,17 @@ # include '_', which is the case for docker-compose. sleep 5 echo "Waiting for Galaxy configurator to finish and release lock" -until [ ! -f /etc/slurm-llnl/configurator.lock ] && echo Lock released; do +until [ ! -f /etc/slurm/configurator.lock ] && echo Lock released; do sleep 0.1; done; -grep < /etc/slurm-llnl/slurm.conf "NodeName=" | while read -r line; do +grep < /etc/slurm/slurm.conf "NodeName=" | while read -r line; do node=$(echo "$line" | sed "s/NodeName=\(.*\) \(NodeAddr.*\)/\1/") node_hostname=$(curl -s --unix-socket /var/run/docker.sock -XGET \ -H "Content-Type: application/json" http://v1.40/containers/json \ -G --data-urlencode "filters={\"name\":[\"$node\"]}" \ | jq -r '.[0] | .["Id"]' | head -c 12) - sed -i "s/$node/$node_hostname/g" /etc/slurm-llnl/slurm.conf + sed -i "s/$node/$node_hostname/g" /etc/slurm/slurm.conf done sleep infinity diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 6f54338b2..e761a1428 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -2,21 +2,21 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:20.04 as galaxy_dependencies +FROM buildpack-deps:22.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.09 +ARG GALAXY_RELEASE=release_24.1 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy -ENV GALAXY_ROOT=/galaxy +ENV GALAXY_ROOT_DIR=/galaxy # Download Galaxy source, but only keep necessary dependencies -RUN mkdir "${GALAXY_ROOT}" \ - && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ - && cd $GALAXY_ROOT \ +RUN mkdir "${GALAXY_ROOT_DIR}" \ + && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT_DIR \ + && cd $GALAXY_ROOT_DIR \ && ls . | grep -v "lib" | xargs rm -rf \ - && cd $GALAXY_ROOT/lib \ + && cd $GALAXY_ROOT_DIR/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ - && cd $GALAXY_ROOT/lib/galaxy \ + && cd $GALAXY_ROOT_DIR/lib/galaxy \ && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf @@ -27,7 +27,7 @@ ENV GALAXY_USER=galaxy \ GALAXY_UID=1450 \ GALAXY_GID=1450 \ GALAXY_HOME=/home/galaxy \ - GALAXY_ROOT=/galaxy + GALAXY_ROOT_DIR=/galaxy RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \ @@ -38,20 +38,20 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ ENV SLURM_USER=galaxy \ SLURM_UID=1450 \ SLURM_GID=1450 \ - MUNGER_USER=munge \ + MUNGE_USER=munge \ MUNGE_UID=1200 \ MUNGE_GID=1200 -RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ - && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ +RUN groupadd -r $MUNGE_USER -g $MUNGE_GID \ + && useradd -u $MUNGE_UID -r -g $MUNGE_USER $MUNGE_USER \ && apt update \ && apt install --no-install-recommends gosu munge python3 python3-dev slurm-wlm -y \ && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/* && find / -name '*.pyc' -delete # Copy Galaxy dependencies -COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $GALAXY_ROOT +COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT_DIR $GALAXY_ROOT_DIR # Make Python3 standard -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 COPY start.sh /usr/bin/start.sh diff --git a/compose/galaxy-slurm/start.sh b/compose/galaxy-slurm/start.sh index cbb8e0b7a..df9cf0e44 100755 --- a/compose/galaxy-slurm/start.sh +++ b/compose/galaxy-slurm/start.sh @@ -4,13 +4,14 @@ sleep 10 # ToDo: Use locking or so to be sure we really have the newest version echo "Waiting for Slurm config" -until [ -f /etc/slurm-llnl/slurm.conf ] && echo Config found; do +until [ -f /etc/slurm/slurm.conf ] && echo Config found; do sleep 0.5; done; if [ "$1" = "slurmctld" ]; then if [ ! -f /etc/munge/munge.key ]; then - gosu "$MUNGE_USER" /usr/sbin/create-munge-key + chown -R "$MUNGE_USER":"$MUNGE_USER" /etc/munge + gosu "$MUNGE_USER" /usr/sbin/mungekey fi echo "Starting Munge.." /etc/init.d/munge start diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index dac9b1503..ce300f25c 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -2,9 +2,9 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:20.04 as build_pulsar +FROM buildpack-deps:22.04 as build_pulsar -ARG PULSAR_RELEASE=0.14.0 +ARG PULSAR_RELEASE=0.15.6 ARG PULSAR_REPO=https://github.com/galaxyproject/pulsar ENV PULSAR_ROOT=/pulsar @@ -19,7 +19,7 @@ RUN mkdir /tmp/pulsar \ && pip3 install wheel \ && python3 -m venv $PULSAR_VIRTUALENV \ && . $PULSAR_VIRTUALENV/bin/activate \ - && pip3 install drmaa kombu pastescript pastedeploy pycurl uwsgi \ + && pip3 install drmaa kombu pastescript pastedeploy pycurl uwsgi pydantic "aiohttp==3.10.9" \ && cd /tmp/pulsar \ && python3 /tmp/pulsar/setup.py install diff --git a/compose/tests/docker-compose.test.bioblend.yml b/compose/tests/docker-compose.test.bioblend.yml index 6b77fb405..009727ba2 100644 --- a/compose/tests/docker-compose.test.bioblend.yml +++ b/compose/tests/docker-compose.test.bioblend.yml @@ -1,10 +1,10 @@ -version: "3.7" services: galaxy-bioblend-test: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-bioblend-test:${IMAGE_TAG:-latest} build: tests/galaxy-bioblend-test environment: - - GALAXY_VERSION=${GALAXY_VERSION:-release_20.09} # TODO: Change to GALAXY_RELEASE - - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-}/ + - GALAXY_VERSION=${GALAXY_VERSION:-release_24.1} # TODO: Change to GALAXY_RELEASE + - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-} + - EXTRA_SKIP_TESTS_BIOBLEND=${EXTRA_SKIP_TESTS_BIOBLEND:-} networks: - galaxy diff --git a/compose/tests/docker-compose.test.selenium.yml b/compose/tests/docker-compose.test.selenium.yml index 12a7f9355..fee6e0187 100644 --- a/compose/tests/docker-compose.test.selenium.yml +++ b/compose/tests/docker-compose.test.selenium.yml @@ -1,11 +1,12 @@ -version: "3.7" services: galaxy-selenium-test: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-selenium-test:${IMAGE_TAG:-latest} build: tests/galaxy-selenium-test environment: - TESTS=${TESTS:-navigates_galaxy.py,login.py} - - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-}/ + - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-} + - SE_ENABLE_TRACING=false + - SE_SESSION_REQUEST_TIMEOUT=1800 volumes: - ${EXPORT_DIR:-./../export}/galaxy/database:/galaxy/database networks: diff --git a/compose/tests/docker-compose.test.workflows.yml b/compose/tests/docker-compose.test.workflows.yml index 8183d3f74..5e9b62889 100644 --- a/compose/tests/docker-compose.test.workflows.yml +++ b/compose/tests/docker-compose.test.workflows.yml @@ -1,11 +1,10 @@ -version: "3.7" services: galaxy-workflow-test: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-workflow-test:${IMAGE_TAG:-latest} build: tests/galaxy-workflow-test environment: - - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-}/ - - WORKFLOWS=${WORKFLOWS:-training/variant-analysis/mapping-by-sequencing/mapping_by_sequencing.ga,sklearn/ard/ard.ga,GraphClust2/GC-lite.ga} + - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-} + - WORKFLOWS=${WORKFLOWS:-training/sequence-analysis/quality-control/quality_control.ga,sklearn/ard/ard.ga,example1/wf3-shed-tools.ga} volumes: - ${EXPORT_DIR:-./../export}/galaxy/database:/galaxy/database networks: diff --git a/compose/tests/docker-compose.test.yml b/compose/tests/docker-compose.test.yml index 275d2c0f0..9c8f4ae87 100644 --- a/compose/tests/docker-compose.test.yml +++ b/compose/tests/docker-compose.test.yml @@ -1,15 +1,14 @@ -version: "3.7" services: galaxy-configurator: environment: - GALAXY_CONFIG_CLEANUP_JOB=never - - NGINX_UWSGI_READ_TIMEOUT=3600 + - NGINX_PROXY_READ_TIMEOUT=3600 - DONT_EXIT=true # Terminates the container after $TIMEOUT minutes # which results in the whole setup terminating if --exit-code-from # is set (see CI) timeout: - image: alpine:3.11 + image: alpine:3.17 environment: - TIMEOUT=${TIMEOUT:-120} command: sh -c "echo \"Setting timeout to $$TIMEOUT minutes\" && sleep $$((( $$TIMEOUT * 60 ))) && echo \"Timeout after $$TIMEOUT minutes!\" && exit 1" diff --git a/compose/tests/galaxy-bioblend-test/Dockerfile b/compose/tests/galaxy-bioblend-test/Dockerfile index a635b421c..d95b84884 100644 --- a/compose/tests/galaxy-bioblend-test/Dockerfile +++ b/compose/tests/galaxy-bioblend-test/Dockerfile @@ -1,10 +1,11 @@ -FROM alpine:3.11 as build +FROM alpine:3.17 as build -ENV BIOBLEND_VERSION=0.13.0 +ENV BIOBLEND_VERSION=1.3.0 ADD "https://github.com/galaxyproject/bioblend/archive/v$BIOBLEND_VERSION.zip" /src/bioblend.zip RUN apk update && apk add curl python3-dev unzip \ - && pip3 install pep8 tox \ + && python3 -m ensurepip --upgrade \ + && pip3 install pep8 tox "aiohttp==3.10.9" \ && cd /src \ && unzip bioblend.zip && rm bioblend.zip \ && mv "bioblend-$BIOBLEND_VERSION" bioblend \ @@ -13,7 +14,7 @@ RUN apk update && apk add curl python3-dev unzip \ WORKDIR /src/bioblend -RUN tox -e py38 --notest +RUN tox -e py310 --notest COPY ./run.sh /src/bioblend/run.sh diff --git a/compose/tests/galaxy-bioblend-test/run.sh b/compose/tests/galaxy-bioblend-test/run.sh index 04eeb3a56..631281f60 100755 --- a/compose/tests/galaxy-bioblend-test/run.sh +++ b/compose/tests/galaxy-bioblend-test/run.sh @@ -9,4 +9,13 @@ export BIOBLEND_GALAXY_URL=${GALAXY_URL:-http://nginx} export BIOBLEND_GALAXY_API_KEY=${GALAXY_DEFAULT_ADMIN_KEY:-fakekey} export BIOBLEND_TEST_JOB_TIMEOUT=${BIOBLEND_TEST_JOB_TIMEOUT:-240} -tox -e py38 -- -k "not test_dataset_peek and not test_create_quota and not test_get_quotas and not test_delete_undelete_quota and not test_update_quota and not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_existing_history and not test_new_history and not test_params and not test_tool_dependency_install and not test_download_history and not test_export_and_download" +# default skip tests +DEFAULT_SKIP_TESTS="not test_rerun_and_remap and not test_create_quota and not test_get_quotas and not test_delete_undelete_quota and not test_update_quota and not test_update_non_default_quota and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_existing_history and not test_new_history and not test_params and not test_tool_dependency_install and not test_download_history and not test_export_and_download and not test_cancel_invocation and not test_run_step_actions and not test_extract_workflow_from_history" + +EXTRA_SKIP_TESTS_BIOBLEND=${EXTRA_SKIP_TESTS_BIOBLEND:-""} + +# Combine default skip tests with extra skip tests, if provided +SKIP_TESTS="$DEFAULT_SKIP_TESTS" +[ -n "$EXTRA_SKIP_TESTS_BIOBLEND" ] && SKIP_TESTS="$SKIP_TESTS and $EXTRA_SKIP_TESTS_BIOBLEND" + +tox -e py310 -- -k "$SKIP_TESTS" diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index 370264852..0ff7f9f6f 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -1,20 +1,20 @@ -FROM selenium/standalone-chrome:3.141.59 +FROM selenium/standalone-chrome:4.25.0 -ARG GALAXY_RELEASE=release_20.09 +ARG GALAXY_RELEASE=release_24.1 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy -ENV GALAXY_ROOT=/galaxy +ENV GALAXY_ROOT_DIR=/galaxy ENV GALAXY_PYTHON=/usr/bin/python3 USER root -RUN apt update && apt install --no-install-recommends python3-dev python3-pip -y && rm -rf /var/lib/apt/lists/* \ - && mkdir "${GALAXY_ROOT}" \ - && chown seluser "${GALAXY_ROOT}" +RUN apt update && apt install --no-install-recommends python3-dev python3-pip libpq-dev 2to3 -y && rm -rf /var/lib/apt/lists/* \ + && mkdir "${GALAXY_ROOT_DIR}" \ + && chown seluser "${GALAXY_ROOT_DIR}" USER seluser -RUN mkdir -p $GALAXY_ROOT && \ - curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ - && cd "${GALAXY_ROOT}" \ +RUN mkdir -p $GALAXY_ROOT_DIR && \ + curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT_DIR \ + && cd "${GALAXY_ROOT_DIR}" \ && ./scripts/common_startup.sh --skip-client-build --dev-wheels COPY run.sh /usr/bin/run.sh diff --git a/compose/tests/galaxy-selenium-test/run.sh b/compose/tests/galaxy-selenium-test/run.sh index 9fbc16650..f372aad01 100755 --- a/compose/tests/galaxy-selenium-test/run.sh +++ b/compose/tests/galaxy-selenium-test/run.sh @@ -15,7 +15,7 @@ export GALAXY_TEST_SELENIUM_REMOTE_HOST=localhost export GALAXY_TEST_SELENIUM_REMOTE_PORT=4444 export GALAXY_TEST_EXTERNAL_FROM_SELENIUM=${GALAXY_URL:-http://nginx} export GALAXY_TEST_EXTERNAL=${GALAXY_URL:-http://nginx} -export GALAXY_CONFIG_MASTER_API_KEY=${GALAXY_DEFAULT_ADMIN_KEY:-fakekey} +export GALAXY_CONFIG_BOOTSTRAP_ADMIN_API_KEY=${GALAXY_DEFAULT_ADMIN_KEY:-fakekey} for test in $(echo "$TESTS" | sed "s/,/ /g"); do diff --git a/compose/tests/galaxy-workflow-test/Dockerfile b/compose/tests/galaxy-workflow-test/Dockerfile index 1b9385da9..0a0e1a2e6 100644 --- a/compose/tests/galaxy-workflow-test/Dockerfile +++ b/compose/tests/galaxy-workflow-test/Dockerfile @@ -1,18 +1,15 @@ -FROM alpine:3.11 +FROM alpine:3.17 -ENV TEST_REPO=${TEST_REPO:-https://github.com/jonas27/workflow-testing} \ - TEST_RELEASE=${TEST_RELEASE:-20.09-comment-filetype} +ENV TEST_REPO=${TEST_REPO:-https://github.com/jyotipm29/workflow-testing} \ + TEST_RELEASE=${TEST_RELEASE:-24.1} -RUN apk add --no-cache bash python3 curl \ - && apk add --no-cache --virtual build-dep gcc libxml2-dev libxslt-dev musl-dev linux-headers python3-dev \ +RUN apk add --no-cache bash python3 py3-pip curl \ + && apk add --no-cache --virtual build-dep gcc make libc-dev xz-dev bzip2-dev hdf5-dev musl-dev linux-headers python3-dev \ && pip3 install planemo \ && mkdir /src && cd /src \ && curl -L -s $TEST_REPO/archive/$TEST_RELEASE.tar.gz | tar xzf - --strip-components=1 \ && apk del build-dep -# Make Python3 standard -RUN ln /usr/bin/python3 /usr/bin/python && ln /usr/bin/python3 /usr/bin/python2 - ADD ./run.sh /usr/bin/run.sh WORKDIR /src diff --git a/compose/tests/galaxy-workflow-test/run.sh b/compose/tests/galaxy-workflow-test/run.sh index 969ce8e16..4d056ade4 100755 --- a/compose/tests/galaxy-workflow-test/run.sh +++ b/compose/tests/galaxy-workflow-test/run.sh @@ -14,7 +14,7 @@ do --galaxy_admin_key "${GALAXY_USER_KEY:-fakekey}" \ --shed_install \ --engine external_galaxy \ - --test_output ${GALAXY_ROOT:-/galaxy}/database/tool_test_output.html \ - --test_output_json ${GALAXY_ROOT:-/galaxy}/database/tool_test_output.json \ + --test_output ${GALAXY_ROOT_DIR:-/galaxy}/database/tool_test_output.html \ + --test_output_json ${GALAXY_ROOT_DIR:-/galaxy}/database/tool_test_output.json \ "$workflow"; done From 130be065ebad48bbffdff117ebecb722213b383a Mon Sep 17 00:00:00 2001 From: jyotipm29 Date: Wed, 30 Oct 2024 16:55:53 +0100 Subject: [PATCH 2/4] added rustus service --- compose/base_config.yml | 1 + compose/docker-compose.yml | 25 +++++++++++++++++++ .../templates/nginx/nginx.conf.j2 | 17 +++++++++++++ 3 files changed, 43 insertions(+) diff --git a/compose/base_config.yml b/compose/base_config.yml index 324220a10..baa631ff7 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -20,6 +20,7 @@ gravity: galaxy: tool_dependency_dir: /tool_deps tool_data_table_config_path: /cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml + tus_upload_store: /tus_upload_store enable_celery_tasks: true celery_conf: result_backend: redis://redis:6379/0 diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 17f2a10a6..3a5604203 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -14,11 +14,13 @@ services: # This is the directory where all your files from Galaxy will be stored # on your host system - ${EXPORT_DIR:-./export}/:/export/:delegated + - ${EXPORT_DIR:-./export}/tus_upload_store:/tus_upload_store:delegated - /var/run/docker.sock:/var/run/docker.sock depends_on: - postgres - rabbitmq - redis + - rustus networks: - galaxy # The galaxy-configurator is responsible for the whole configuration of @@ -95,5 +97,28 @@ services: - ${EXPORT_DIR:-./export}/redis:/data:delegated networks: - galaxy + # For file uploads + rustus: + image: s3rius/rustus:0.7.6-alpine + container_name: galaxy-rustus + hostname: rustus + environment: + - RUSTUS_STORAGE=file-storage + - RUSTUS_DATA_DIR=/data/ + - RUSTUS_URL=${GALAXY_PROXY_PREFIX:-}/api/upload/resumable_upload + - RUSTUS_HOOKS_HTTP_URLS=http://nginx${GALAXY_PROXY_PREFIX:-}/api/upload/hooks + - RUSTUS_HOOKS_HTTP_PROXY_HEADERS=X-Api-Key,Cookie + - RUSTUS_HOOKS=pre-create + - RUSTUS_HOOKS_FORMAT=tusd + - RUSTUS_INFO_STORAGE=redis-info-storage + - RUSTUS_INFO_DB_DSN=redis://redis:6379/1 + - RUSTUS_MAX_BODY_SIZE=20000000 + - RUSTUS_BEHIND_PROXY=true + volumes: + - ${EXPORT_DIR:-./export}/tus_upload_store:/data:delegated + depends_on: + - redis + networks: + - galaxy networks: galaxy: diff --git a/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 b/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 index 256a6d927..d4c11b30a 100644 --- a/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 +++ b/compose/galaxy-configurator/templates/nginx/nginx.conf.j2 @@ -61,6 +61,23 @@ http { $vis_d/static/$static_file =404; } + # delegated uploads + location {{ GALAXY_PROXY_PREFIX | regex_replace("/$", "") }}/api/upload/resumable_upload { + # Disable request and response buffering + proxy_request_buffering off; + proxy_buffering off; + proxy_http_version 1.1; + + # Add X-Forwarded-* headers + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + client_max_body_size 0; + proxy_pass http://rustus:1081; + } + rewrite ^/{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }}$ /{{ GALAXY_PROXY_PREFIX | regex_replace("^/", "") | regex_replace("/$", "") }}/ last; } } From 148dfca5ee9d5dc6448015873a541fbe51162487 Mon Sep 17 00:00:00 2001 From: jyotipm29 Date: Tue, 12 Nov 2024 11:43:14 +0100 Subject: [PATCH 3/4] install conda from conda-forge (miniforge) --- .github/workflows/compose.yml | 1 + .github/workflows/pull-request.yml | 1 + compose/galaxy-server/Dockerfile | 31 +++++++------------ .../galaxy-server/files/create_galaxy_user.py | 2 +- compose/galaxy-server/files/start.sh | 2 +- 5 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index c45feefbb..e743557fe 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -2,6 +2,7 @@ name: build-and-test on: [push] jobs: build_container_base: + if: false # Temporarily disable workflow runs-on: ubuntu-22.04 steps: - name: Checkout diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 2505f0a7d..625724733 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -2,6 +2,7 @@ name: pr-test on: pull_request jobs: test: + if: false # Temporarily disable workflow runs-on: ubuntu-22.04 strategy: matrix: diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 3b8b8f5a5..032bd7200 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -24,28 +24,28 @@ ENV GALAXY_USER=galaxy \ GALAXY_HOME=/home/galaxy ENV GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda \ - MINICONDA_VERSION=py310_24.5.0-0 + MINIFORGE_VERSION=24.3.0-0 RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \ && mkdir $GALAXY_HOME \ && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME -FROM build_base as build_miniconda +FROM build_base as build_miniforge COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh -# Install Miniconda -RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \ - && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX \ - && rm ~/miniconda.sh \ + +# Install Miniforge +RUN curl -s -L https://github.com/conda-forge/miniforge/releases/download/$MINIFORGE_VERSION/Miniforge3-$MINIFORGE_VERSION-Linux-x86_64.sh > ~/miniforge.sh \ + && /bin/bash ~/miniforge.sh -b -p $GALAXY_CONDA_PREFIX/ \ + && rm ~/miniforge.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ && echo ". $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh" >> $GALAXY_HOME/.bashrc \ && echo "conda activate base" >> $GALAXY_HOME/.bashrc \ && export PATH=$GALAXY_CONDA_PREFIX/bin/:$PATH \ - && conda config --add channels defaults \ && conda config --add channels bioconda \ - && conda config --add channels conda-forge \ && conda install virtualenv pip ephemeris \ && conda clean --packages -t -i \ + && cp -r ~/.conda $GALAXY_HOME && cp ~/.condarc $GALAXY_HOME \ && /usr/bin/common_cleanup.sh FROM build_base as build_galaxy @@ -124,18 +124,9 @@ RUN usermod -aG docker $GALAXY_USER RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT_DIR} ${GALAXY_ROOT_DIR} -COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} -COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_HOME} ${GALAXY_HOME} -COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda /etc/profile.d/conda.sh /etc/profile.d/conda.sh - -# The following commands will be executed as the galaxy user -USER $GALAXY_USER - -# We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan) -ENV HOME=$GALAXY_HOME - -# Switch back to User root -USER root +COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniforge ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} +COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniforge ${GALAXY_HOME} ${GALAXY_HOME} +COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniforge /etc/profile.d/conda.sh /etc/profile.d/conda.sh COPY ./files/start.sh /usr/bin/start.sh diff --git a/compose/galaxy-server/files/create_galaxy_user.py b/compose/galaxy-server/files/create_galaxy_user.py index 8d2b05f23..a94f983eb 100644 --- a/compose/galaxy-server/files/create_galaxy_user.py +++ b/compose/galaxy-server/files/create_galaxy_user.py @@ -17,7 +17,7 @@ def add_user(sa_session, security_agent, email, password, key=None, username="ad if query.count() > 0: return query.first() else: - User.use_pbkdf2 = False + User.use_pbkdf2 = True user = User(email) user.username = username user.set_password_cleartext(password) diff --git a/compose/galaxy-server/files/start.sh b/compose/galaxy-server/files/start.sh index 0ea058c18..349ee6038 100755 --- a/compose/galaxy-server/files/start.sh +++ b/compose/galaxy-server/files/start.sh @@ -131,4 +131,4 @@ chown -RL "$GALAXY_USER:$GALAXY_GROUP" "$GALAXY_CONFIG_DIR" echo "Starting Galaxy now.." cd "$GALAXY_ROOT_DIR" || { echo "Error: Could not change to $GALAXY_ROOT_DIR"; exit 1; } -sudo -E -u $GALAXY_USER "$GALAXY_VIRTUAL_ENV/bin/galaxy" --config-file "$GALAXY_CONFIG_FILE" +sudo -E -H -u $GALAXY_USER "$GALAXY_VIRTUAL_ENV/bin/galaxy" --config-file "$GALAXY_CONFIG_FILE" From 4fd2351e5c156145684a52dc9abc2d6b7f2d632d Mon Sep 17 00:00:00 2001 From: jyotipm29 Date: Tue, 12 Nov 2024 11:50:47 +0100 Subject: [PATCH 4/4] fix ubuntu version --- .github/workflows/compose.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 7bd71d0a6..f3fd1d604 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -3,7 +3,7 @@ on: [push] jobs: build_container_base: if: false # Temporarily disable workflow - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 70a13cb2c..625724733 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -3,7 +3,7 @@ on: pull_request jobs: test: if: false # Temporarily disable workflow - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: infrastructure: