Merge pull request #269 from bgruening/16.10

Release 16.10

Author: bgruening

.travis.yml

@@ -7,10 +7,11 @@ services:
   - docker
 
 env:
+  matrix:
+    - TOX_ENV=py27
+    - COMPOSE=True
   global:
-        - TOX_ENV=py27
-        - COMPOSE=True
-        - secure: "PgEA6vDYZWAIZ3KyUTeA9SNSNR+eJU3kBkoEIsTnmnkyta0Z0VbIv98B9ApYeiC0VHpVzY1gblimmkXSY5+4oD9tUxpA630cUhE9iLNuo5o3MpRFun8nf8H6Jz2Y/PaPw3VPLKsH49LJYgJXeqgzoV7FHrpUP9Q3/5dycWybV2I="
+    - secure: "PgEA6vDYZWAIZ3KyUTeA9SNSNR+eJU3kBkoEIsTnmnkyta0Z0VbIv98B9ApYeiC0VHpVzY1gblimmkXSY5+4oD9tUxpA630cUhE9iLNuo5o3MpRFun8nf8H6Jz2Y/PaPw3VPLKsH49LJYgJXeqgzoV7FHrpUP9Q3/5dycWybV2I="
 
 git:
   submodules: false
@@ -41,8 +42,11 @@ before_install:
         cd compose
         docker build -t quay.io/galaxy/proftpd ./proftpd
         docker tag quay.io/galaxy/proftpd quay.io/galaxy/proftpd:compose
+        docker build -t quay.io/galaxy/postgres ./postgres-galaxy
+        docker tag quay.io/galaxy/postgres quay.io/galaxy/postgres:compose
         docker tag quay.io/bgruening/galaxy quay.io/bgruening/galaxy:compose
         docker-compose --project-name galaxy_compose up -d
+        sleep 50
     else
         mkdir local_folder
         docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \
@@ -69,12 +73,16 @@ script:
   # Test submitting jobs to an external gridengine cluster
   - cd $TRAVIS_BUILD_DIR/test/gridengine/ && bash test.sh && cd $TRAVIS_BUILD_DIR
   # Test FTP Server
-  - curl --fail $BIOBLEND_GALAXY_URL/api/version
-  - date > time.txt && curl --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
-  - curl --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
+  - curl -v --fail $BIOBLEND_GALAXY_URL/api/version
+  - date > time.txt && curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
+  - curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
 
   # Test SFTP Server
   - sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt'
+  # Test self-signed HTTPS
+  - docker run -d -p 443:443 -e "USE_HTTPS=True" quay.io/bgruening/galaxy
+  - sleep 60s && curl -v -k --fail https://127.0.0.1:443/api/version
+  - echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned
 
   - cd $TRAVIS_BUILD_DIR/test/bioblend/ && bash test.sh && cd $TRAVIS_BUILD_DIR/
   # Test the 'old' tool installation script

README.md

@@ -233,7 +233,7 @@ The Galaxy welcome screen can be changed by providing a `welcome.html` page in `
 
 ## Deactivating services <a name="Deactivating-services" /> [[toc]](#toc)
 
-Non-essential services can be deactivated during startup. Set the environment variable `NONUSE` to a comma separated list of services. Currently, `nodejs`, `proftp`, `reports`, `slurmd` and `slurmctld` are supported.
+Non-essential services can be deactivated during startup. Set the environment variable `NONUSE` to a comma separated list of services. Currently, `nodejs`, `postgres`, `proftp`, `reports`, `slurmd` and `slurmctld` are supported.
 
 ```sh
 docker run -d -p 8080:80 -p 9002:9002 \
@@ -578,6 +578,8 @@ git submodule update --init --recursive
   - documentation and tests updates for SLURM integration by @mvdbeek
   - first version with initial Docker compose support (proftpd ✔️)
   - SFTP support by @zfrenchee
+ - 16.10:
+  - [HTTPS support](https://github.com/bgruening/docker-galaxy-stable/pull/240 ) by @zfrenchee and @mvdbeek
 
 # Support & Bug Reports <a name="Support-Bug-Reports" /> [[toc]](#toc)
 

compose/Readme.md

@@ -0,0 +1,80 @@
+Galaxy Docker Compose
+=====================
+
+# Table of Contents <a name="toc" />
+
+- [Usage](#Usage)
+- [Build](#Build)
+- [Roadmap](#Roadmap)
+- [Advanced](#Advanced)
+  - [postgres](#postgres)
+    - [Configuration](#postgres-Configuration)
+  - [proftpd](#proftpd)
+    - [Configuration](#proftpd-Configuration)
+
+# Usage <a name="Usage" /> [[toc]](#toc)
+
+At first you need to install docker with compose.
+- [docker](https://docs.docker.com/installation/)
+- [docker-compose](https://docs.docker.com/compose/install/)
+
+# Build <a name="Build" /> [[toc]](#toc)
+
+Checkout the repository:
+```
+git checkout https://github.com/bgruening/docker-galaxy-stable.git
+cd docker-galaxy-stable/compose
+```
+
+Build the compose containers:
+```sh
+./buildlocal.sh
+```
+After successful installation you can run the compose file:
+```sh
+docker-compose up -d
+```
+
+
+# Roadmap <a name="Roadmap" /> [[toc]](#toc)
+
+We wish to split the monolithic container into its components, i.e. postgres, proftpd, slurm, nginx (more?)
+So far postgres and proftpd are working.
+
+# Advanced <a name="Advanced" /> [[toc]](#toc)
+
+## postgres <a name="postgres" /> [[toc]](#toc)
+You can theoretically use any external database. The included postgres build is based on the [official postgres container](https://hub.docker.com/_/postgres/) and adds an initialization script which loads a vanilla dump of the initial database state on first startup.
+
+In case you want to generate an initial database dump yourself:
+```sh
+./dumpsql.sh
+./buildlocal.sh
+```
+
+To manually initialize a database without the dump connected via docker-compose.yml, before starting the galaxy container you can run:
+```sh
+docker-compose run galaxy install_db.sh
+```
+which will perform database migration.
+
+### Configuration <a name="postgres-Configuration" /> [[toc]](#toc)
+See [official postgres container](https://hub.docker.com/_/postgres/).
+
+## proftpd <a name="proftpd" /> [[toc]](#toc)
+
+Proftpd uses the [ansible galaxy extras project](https://github.com/galaxyproject/ansible-galaxy-extras) to configurate proftpd.
+
+### Configuration <a name="proftpd-Configuration" /> [[toc]](#toc)
+
+- *proftpd\_db\_connection*=_database@host_: Configurates the database name and hostname. Hostname can be a linked database container.
+- *proftpd\_db\_username*=_dbuser_: User in the database.
+- *proftpd\_db\_password*=_dbpass_: Password of the user in the database.
+- *proftpd\_files\_dir*=_/export/ftp_: Directory where the user files are to be placed. Should be synchronized with volumes.
+- *proftpd\_sql\_auth\_type*=_SHA1_: Authentication type used in the galaxy database (SHA1|PBKDF2).
+- *proftpd\_welcome*=_Public Galaxy FTP_: Welcome message.
+- *proftpd\_passive\_port\_low*=_30000_, *proftpd\_passive\_port\_high*=_40000_: Passive mode port range. This should be kept small for docker exposing the range (30000-30010 or so), because docker allocates each port separately which makes the process stale. Should be same as exposed ports.
+- *proftpd\_use\_sftp*=_false_: Enable sftp.
+- *proftpd\_nat\_masquerade*=_false_: Set masquearade to true if host is NAT'ed.
+- *proftpd\_masquerade\_address*=_ip_: Refers to the ip that clients use to establish an ftp connection. Can be a command that returns an IP or an IP address and applies only if proftpd\_nat\_masquerade is true. `\`ec2metadata --public-ipv4\`` returns the public ip for amazon's ec2 service.
+

compose/buildlocal.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+docker build -t quay.io/bgruening/galaxy ../galaxy/
+docker tag quay.io/bgruening/galaxy quay.io/bgruening/galaxy:compose
+docker build -t quay.io/galaxy/proftpd ./proftpd
+docker tag quay.io/galaxy/proftpd quay.io/galaxy/proftpd:compose
+docker build -t quay.io/galaxy/postgres ./postgres-galaxy
+docker tag quay.io/galaxy/postgres quay.io/galaxy/postgres:compose

compose/docker-compose.yml

@@ -1,46 +1,76 @@
 version: '2'
 services:
   # proftpd container
-  proftp:
+  proftpd:
+#    build: proftpd
     image: quay.io/galaxy/proftpd:compose
-    container_name: galaxy-proftp
+    environment:
+        - proftpd_db_connection=galaxy@db
+        - proftpd_db_username=galaxy
+        - proftpd_db_password=chaopagoosaequuashie
+        - proftpd_files_dir=/export/ftp
+        - proftpd_use_sftp=True
+        - proftpd_passive_port_low=30000
+        - proftpd_passive_port_high=30010
+    container_name: galaxy-proftpd
     volumes:
-        - ftp_config:/etc/proftpd/
         - ftp_data:/export/ftp
     expose:
         - 21
         - 22
     ports:
         - "8021:21"
         - "8022:22"
-    # proftpd config files from Galaxy are needed
-    depends_on:
-        - galaxy
+        - "30000-30010:30000-30010"
     links:
-        - galaxy:galaxy.local
+        - postgres:db
     restart: always
-    hostname: galaxy-proftp
+    hostname: galaxy-proftpd
+
+  postgres:
+# Using the official postgres image. This needs to be populated by calling
+#  docker-compose run galaxy install_db.sh
+# on first run
+#    image: postgres
+# This comes with an initialization to quickly populate the database on first start
+#    build: postgres-galaxy
+    image: quay.io/galaxy/postgres:compose
+    container_name: galaxy-postgres
+    environment:
+        - POSTGRES_PASSWORD=chaopagoosaequuashie
+        - POSTGRES_USER=galaxy
+        - POSTGRES_DB=galaxy
+    volumes:
+        - ./postgres-storage:/var/lib/postgresql/data
+    hostname: galaxy-postgres
+
 
 
   # Main Galaxy container
   galaxy:
+#    build: ../galaxy
     image: quay.io/bgruening/galaxy:compose
     environment:
         #    Insert here your environment variables to change Galaxy's behaviour.
         #    For example to personalise Galaxy by changing the branding with
         #    - GALAXY_CONFIG_BRAND=FOO
-        - NONUSE=proftp
+        - NONUSE=proftp postgres
         - GALAXY_CONFIG_FTP_UPLOAD_DIR=/export/ftp
         - GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True
         - GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True
         - GALAXY_CONFIG_ENABLE_USER_DELETION=True
         - GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True
+        - GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:chaopagoosaequuashie@db/galaxy?client_encoding=utf8
+        - [email protected]
+        - GALAXY_DEFAULT_ADMIN_PASSWORD=admin
+        - GALAXY_DEFAULT_ADMIN_KEY=admin
     ports:
         - "8080:80" # nginx
         - "9002:9002" # supervisor web-interface - not super useful in compose mode
+    links:
+        - postgres:db
     container_name: galaxy
     volumes:
-        - ftp_config:/etc/proftpd/
         - ftp_data:/export/ftp
         # This is the directory where all your files from Galaxy will be stored
         # on your host system
@@ -49,5 +79,4 @@ services:
     privileged: True
 
 volumes:
-  ftp_config: {}
   ftp_data: {}

compose/dumpsql.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Sets the image of postgres to use
+POSTGRES=postgres
+POSTGRES_USER=galaxy
+POSTGRES_PASSWORD=chaopagoosaequuashie
+POSTGRES_DB=galaxy
+# Create postgres in detached mode
+pg_start=`date +%s`
+docker run -d --name "dumpsql_postgres" -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" -e "POSTGRES_USER=$POSTGRES_USER" -e "POSTGRES_DB=$POSTGRES_DB" $POSTGRES
+# Output postgres log
+docker logs -f dumpsql_postgres &
+# Wait until postgres has initialized
+until docker run --rm --link dumpsql_postgres:pg $POSTGRES pg_isready -U postgres -h pg >/dev/null; do sleep 1; done
+pg_end=`date +%s`
+init_start=`date +%s`
+docker run -i --rm --name "dumpsql_galaxy_installdb" -e "GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@db/$POSTGRES_DB?client_encoding=utf8" --link "dumpsql_postgres:db" quay.io/bgruening/galaxy:compose install_db.sh
+init_end=`date +%s`
+dump_start=`date +%s`
+docker exec "dumpsql_postgres" pg_dump --no-tablespace --no-acl --no-owner -U postgres galaxy > postgres-galaxy/init-galaxy-db.sql.in
+dump_end=`date +%s`
+docker rm -f dumpsql_postgres
+echo "Stats:"
+echo "Startup postgres: $((pg_end-pg_start)) sec"
+echo "install_db: $((init_end-init_start)) sec"
+echo "pg_dumpall: $((dump_end-dump_start)) sec"
+echo "Total: $((dump_end-pg_start)) sec"

compose/postgres-galaxy/Dockerfile

@@ -0,0 +1,4 @@
+FROM library/postgres
+ADD init-galaxy-db.sql.in /docker-entrypoint-initdb.d/
+ADD init-galaxy-db.sh /docker-entrypoint-initdb.d/
+

compose/postgres-galaxy/init-galaxy-db.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "Initializing galaxy database with defaults"
+"${psql[@]}" < /docker-entrypoint-initdb.d/init-galaxy-db.sql.in >/dev/null
+echo "Successfully initialized galaxy database"