Skip to content

Commit 08ef14d

Browse files
bgrueningbgruening
authored
Merge pull request #569 from bgruening/20.09
20.09
2 parents f9bff09 + 7d122a6 commit 08ef14d

File tree

27 files changed

+460
-160
lines changed

27 files changed

+460
-160
lines changed

‎.github/workflows/compose.yml

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: build-and-test
22
on: [push]
33
jobs:
44
build_container_base:
5-
runs-on: ubuntu-latest
5+
runs-on: ubuntu-20.04
66
steps:
77
- name: Checkout
88
uses: actions/checkout@v2
@@ -120,7 +120,7 @@ jobs:
120120
--build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \
121121
--build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \
122122
--build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \
123-
--build-arg GALAXY_REPO=https://github.com/andreassko/galaxy \
123+
--build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \
124124
${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying"
125125
done;
126126
shell: bash
@@ -164,6 +164,7 @@ jobs:
164164
files: -f docker-compose.yml -f docker-compose.k8s.yml
165165
exclude_test:
166166
- bioblend
167+
- workflow_example1
167168
- workflow_ard
168169
- workflow_mapping_by_sequencing
169170
- selenium
@@ -232,13 +233,13 @@ jobs:
232233
- name: Checkout
233234
uses: actions/checkout@v2
234235
- name: Set image tag in env
235-
run: echo "::set-env name=IMAGE_TAG::${GITHUB_REF#refs/heads/}"
236+
run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
236237
- name: Master branch - Set image to to 'latest'
237238
if: github.ref == 'refs/heads/master'
238-
run: echo "::set-env name=IMAGE_TAG::latest"
239+
run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV
239240
- name: Set WORKFLOWS env for worfklows-test
240241
if: matrix.test.workflow
241-
run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}"
242+
run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV
242243
- name: Run tests for the first time
243244
if: steps.run_check.outputs.run
244245
run: |

‎.github/workflows/lint.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
level: warning
1717
pattern: "*.sh"
1818
- name: Run hadolint with reviewdog
19-
uses: reviewdog/action-hadolint@v1
19+
uses: reviewdog/action-hadolint@v1.16.0
2020
with:
2121
github_token: ${{ secrets.GITHUB_TOKEN }}
2222
reporter: github-check

‎.github/workflows/pull-request.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: pr-test
22
on: pull_request
33
jobs:
44
test:
5-
runs-on: ubuntu-latest
5+
runs-on: ubuntu-20.04
66
strategy:
77
matrix:
88
infrastructure:
@@ -30,6 +30,7 @@ jobs:
3030
exclude_test:
3131
- workflow_example1
3232
- workflow_mapping_by_sequencing
33+
- workflow_ard
3334
# - name: galaxy-pulsar-mq
3435
# files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml
3536
# exclude_test:
@@ -41,6 +42,7 @@ jobs:
4142
- bioblend
4243
- workflow_ard
4344
- workflow_mapping_by_sequencing
45+
- workflow_example1
4446
- selenium
4547
- name: galaxy-singularity
4648
files: -f docker-compose.yml -f docker-compose.singularity.yml
@@ -108,7 +110,7 @@ jobs:
108110
uses: actions/checkout@v2
109111
- name: Set WORKFLOWS env for worfklows-test
110112
if: matrix.test.workflow
111-
run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}"
113+
run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV
112114
- name: Build galaxy-container-base
113115
env:
114116
image_name: galaxy-container-base
@@ -142,7 +144,7 @@ jobs:
142144
echo "Removing export directory if existent";
143145
sudo rm -rf export
144146
set +e
145-
docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy
147+
docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy
146148
docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }}
147149
test_exit_code=$?
148150
error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1)

‎.github/workflows/single.sh

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
export GALAXY_HOME=/home/galaxy
5+
export [email protected]
6+
export [email protected]
7+
export GALAXY_USER_PASSWD=password
8+
export BIOBLEND_GALAXY_API_KEY=fakekey
9+
export BIOBLEND_GALAXY_URL=http://localhost:8080
10+
11+
sudo apt-get update -qq
12+
#sudo apt-get install docker-ce --no-install-recommends -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew"
13+
sudo apt-get install sshpass --no-install-recommends -y
14+
15+
pip3 install ephemeris
16+
17+
docker --version
18+
docker info
19+
20+
# start building this repo
21+
git submodule update --init --recursive
22+
sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
23+
24+
## define a container size check function, first parameter is the container name, second the max allowed size in MB
25+
container_size_check () {
26+
27+
# check that the image size is not growing too much between releases
28+
# the 19.05 monolithic image was around 1.500 MB
29+
size="${docker image inspect $1 --format='{{.Size}}'}"
30+
size_in_mb=$(($size/(1024*1024)))
31+
if [[ $size_in_mb -ge $2 ]]
32+
then
33+
echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2"
34+
sleep 2
35+
#exit
36+
fi
37+
}
38+
39+
export WORKING_DIR=${GITHUB_WORKSPACE:-$PWD}
40+
41+
export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
42+
SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
43+
cd "$WORKING_DIR"
44+
docker build -t quay.io/bgruening/galaxy galaxy/
45+
#container_size_check quay.io/bgruening/galaxy 1500
46+
47+
mkdir local_folder
48+
docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \
49+
--name galaxy \
50+
--privileged=true \
51+
-v "$(pwd)/local_folder:/export/" \
52+
-e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \
53+
-e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \
54+
-e GALAXY_CONFIG_ENABLE_USER_DELETION=True \
55+
-e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \
56+
-v /tmp/:/tmp/ \
57+
quay.io/bgruening/galaxy
58+
59+
sleep 30
60+
docker logs galaxy
61+
# Define start functions
62+
docker_exec() {
63+
cd "$WORKING_DIR"
64+
docker exec galaxy "$@"
65+
}
66+
docker_exec_run() {
67+
cd "$WORKING_DIR"
68+
docker run quay.io/bgruening/galaxy "$@"
69+
}
70+
docker_run() {
71+
cd "$WORKING_DIR"
72+
docker run "$@"
73+
}
74+
75+
docker ps
76+
77+
# Test submitting jobs to an external slurm cluster
78+
cd "${WORKING_DIR}/test/slurm/" && bash test.sh && cd "$WORKING_DIR"
79+
80+
# Test submitting jobs to an external gridengine cluster
81+
# TODO 19.05, need to enable this again!
82+
# - cd $WORKING_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR
83+
84+
echo 'Waiting for Galaxy to come up.'
85+
galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 300
86+
87+
curl -v --fail $BIOBLEND_GALAXY_URL/api/version
88+
89+
# Test self-signed HTTPS
90+
docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAINER
91+
# TODO 19.05
92+
# - sleep 90s && curl -v -k --fail https://127.0.0.1:443/api/version
93+
#- echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned
94+
95+
docker logs httpstest && docker stop httpstest && docker rm httpstest
96+
97+
# Test FTP Server upload
98+
date > time.txt
99+
# FIXME passive mode does not work, it would require the container to run with --net=host
100+
#curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true
101+
# Test FTP Server get
102+
#curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
103+
104+
# Test CVMFS
105+
docker_exec bash -c "service autofs start"
106+
docker_exec bash -c "cvmfs_config chksetup"
107+
docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand"
108+
109+
# Test SFTP Server
110+
sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt'
111+
112+
# Run a ton of BioBlend test against our servers.
113+
cd "$WORKING_DIR/test/bioblend/" && . ./test.sh && cd "$WORKING_DIR/"
114+
115+
# not working anymore in 18.01
116+
# executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid --log-file=galaxy_install.log --pid-file=galaxy_install.pid
117+
# [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml
118+
# /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log'
119+
# getopt_long() error
120+
# cat: galaxy_install.pid: No such file or directory
121+
# tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory
122+
#- |
123+
# if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ]
124+
# then
125+
# # Test without install-repository wrapper
126+
# sleep 10
127+
# docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools'
128+
# fi
129+
130+
131+
# Test the 'new' tool installation script
132+
docker_exec install-tools "$SAMPLE_TOOLS"
133+
# Test the Conda installation
134+
docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/bin/:$PATH && conda --version && conda install samtools -c bioconda --yes'
135+
136+
137+
docker stop galaxy
138+
docker rm -f galaxy
139+
docker rmi -f $DOCKER_RUN_CONTAINER

‎.github/workflows/single_container.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: Single Container Test
2+
on: [push]
3+
jobs:
4+
build_and_test:
5+
runs-on: ubuntu-latest
6+
strategy:
7+
matrix:
8+
python-version: [3.7]
9+
steps:
10+
- name: Checkout
11+
uses: actions/checkout@v2
12+
- uses: actions/setup-python@v1
13+
with:
14+
python-version: ${{ matrix.python-version }}
15+
- name: Build and Test
16+
run: bash .github/workflows/single.sh

‎.travis.yml

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ before_install:
4040
# start building this repo
4141
- git submodule update --init --recursive
4242
- sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
43+
- export WORKING_DIR="$TRAVIS_BUILD_DIR"
44+
- export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
45+
- export INSTALL_REPO_ARG=""
46+
- export SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
47+
- travis_wait 30 cd "$WORKING_DIR" && docker build -t quay.io/bgruening/galaxy galaxy/
4348
- |
4449
## define a container size check function, first parameter is the container name, second the max allowed size in MB
4550
container_size_check () {
@@ -55,14 +60,6 @@ before_install:
5560
#exit
5661
fi
5762
}
58-
59-
export WORKING_DIR="$TRAVIS_BUILD_DIR"
60-
export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
61-
INSTALL_REPO_ARG=""
62-
SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
63-
cd "$WORKING_DIR"
64-
docker build -t quay.io/bgruening/galaxy galaxy/
65-
6663
container_size_check quay.io/bgruening/galaxy 1500
6764
6865
mkdir local_folder

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -993,6 +993,8 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro
993993
- Featuring Galaxy 20.05
994994
- Completely reworked compose setup
995995
- The default admin password and apikey (`GALAXY_DEFAULT_ADMIN_PASSWORD` and `GALAXY_DEFAULT_ADMIN_KEY`) have changed: the password is now `password` (instead of `admin`) and the apikey `fakekey` (instead of `admin`).
996+
- 20.09:
997+
- Featuring Galaxy 20.09
996998
997999
# Support & Bug Reports <a name="Support-Bug-Reports" /> [[toc]](#toc)
9981000

‎compose/README.md

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,9 +151,9 @@ All options are discussed under [configuration reference](#configuration-referen
151151

152152
### Use specific Galaxy version or Docker images
153153
The `IMAGE_TAG` environment variable allows to use specific versions of the
154-
setup. Say, you want to stay with Galaxy v20.05 for now:
154+
setup. Say, you want to stay with Galaxy v20.09 for now:
155155

156-
> export IMAGE_TAG=20.05
156+
> export IMAGE_TAG=20.09
157157
> docker-compose up
158158
159159
Without setting this variable, you will always get updated to the newest
@@ -378,7 +378,7 @@ The following are settings specific to this docker-compose setup:
378378
| `GALAXY_K8S_DOCKER_REPO_DEFAULT` | The Docker Repo/Registry to use if the resolver could not resolve the proper image for a job. Defaults to `docker.io`. |
379379
| `GALAXY_K8S_DOCKER_OWNER_DEFAULT` | The Owner/Username to use if the resolver could not resolve the proper image for a job. Is not set by default. |
380380
| `GALAXY_K8S_DOCKER_IMAGE_DEFAULT` | The Image to use if the resolver could not resolve the proper image for a job. Defaults to `ubuntu`. |
381-
| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `18.04`. |
381+
| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `20.04`. |
382382

383383
### HTCondor
384384
| Variable | Description |
@@ -393,3 +393,21 @@ The following are settings specific to this docker-compose setup:
393393
| `SLURM_NODE_CPUS` | Number of CPUs per node. Defaults to 1. |
394394
| `SLURM_NODE_MEMORY` | Amount of memory per node. Defaults to 1024. |
395395
| `SLURM_NODE_HOSTNAME` | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. |
396+
397+
### Github Workflow Tests (Branch 20.09)
398+
| Setup | bioblend | workflow ard | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium |
399+
|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------|
400+
| Galaxy Base | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
401+
| Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
402+
| HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
403+
| Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
404+
| Pulsar | :heavy_check_mark: | :x: | :x: | :x: | :heavy_check_mark: |
405+
| k8s | :x: | :x: | :x: | :x: | :x: |
406+
| Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: |
407+
| Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: |
408+
| HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: |
409+
410+
411+
Implemented: :heavy_check_mark:
412+
Not Implemented: :x:
413+

‎compose/base-images/galaxy-cluster-base/Dockerfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@ ENV MUNGER_USER=munge \
2020
MUNGE_GID=1200
2121
RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \
2222
&& useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \
23-
&& echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \
24-
&& echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \
23+
&& echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \
24+
&& echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \
2525
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \
2626
&& apt update \
27-
&& apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \
27+
&& apt install --no-install-recommends python3-distutils slurm-client slurmd slurmctld slurm-drmaa1 -y \
2828
&& apt --no-install-recommends install munge libmunge-dev -y \
2929
&& ln -s /usr/lib/slurm-drmaa/lib/libdrmaa.so.1 /usr/lib/slurm-drmaa/lib/libdrmaa.so \
3030
&& /usr/bin/common_cleanup.sh

‎compose/base-images/galaxy-container-base/Dockerfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM buildpack-deps:18.04 as build_singularity
1+
FROM buildpack-deps:20.04 as build_singularity
22

33
COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
44

@@ -21,12 +21,12 @@ RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_
2121

2222

2323
# --- Final image ---
24-
FROM ubuntu:18.04 as final
24+
FROM ubuntu:20.04 as final
2525

2626
COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
2727

2828
# Base dependencies
29-
RUN apt update && apt install --no-install-recommends ca-certificates squashfs-tools -y \
29+
RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \
3030
&& /usr/bin/common_cleanup.sh
3131

3232
# Install Docker

0 commit comments

Comments
 (0)