Merging changes from cedadev fork (#45)

* 2.0 integration (#44)

* Changes from cedadev fork (#43)

* Postgres container working

* Improvements to solr, tomcat and index-node

* Improved postgres container

* SLCS is working

* idp-node seems to be working

* Changes to support URL prefix

* Proxy -> CoG working

* Solr + Zookeeper working

* Index node working + certificates

* Revamped configuration + index node

* Use a common config container

* ORP seems to be working

* Setup container + fixes to compose

* SLCS working

* Auth app seems to run, but isn't working properly

* Should have been in last commit

* Set theme jekyll-theme-cayman

* Testing GitHub pages for project

* Testing Github pages site

* Create architecture.md

* Create comparison.md

* Quick start done

* Comparison with previous impl

* Container architecture docs

* Add version and hub to docker-compose

* Docs on configuration

* Changes while doing Helm integration

* Changes to support OpenShift deployment

* Fix problem with ORP trusting IDP

* Changes to make hub and version flexible

* Fix certificates, move generation of Java certs into containers

* Publishing seems to be working apart from some issues with facets

* Fix description of init

* Get index node to query Solr for facets correctly

* Add Travis support

* Extend Travis timeout

* Fix timeout but still get output

* Fixed omission of while loop

* Bring docker push into actual command

* Bring back-to-top link to top

* Small modifications to configs in prep for future work

* Added helm-values command + pinned conda version in publisher

* Make HTTPS everywhere the default

* Fix compose file for newly required variable

* Make containers work as non-root

* Improved postgres containers for permissions and separated cogdb

* Revert to master + slave for Solr

* Set up CI for ESGF-based repository

* Update Travis script

* More tweaks to Travis job

* Update .travis.yml

Author: mkjpryor-stfc

.env

@@ -1,8 +1,6 @@
 # Default environment values used by ESGF/Docker
 # Note that any value defined in the shell takes precedence over values defined in this file.
 
-ESGF_HOSTNAME=my-node.esgf.org
-ESGF_CONFIG=~/esgf_config
-ESGF_DATA_DIR=/tmp
-ESGF_VERSION=1.0
-ESGF_IMAGES_HUB=esgfhub
+ESGF_HUB=esgfhub
+ESGF_PREFIX=
+ESGF_VERSION=latest

.travis.yml

@@ -0,0 +1,65 @@
+sudo: required
+
+services:
+  - docker
+
+addons:
+  apt:
+    packages:
+      - docker-ce
+
+branches:
+  only:
+    - master
+    - devel
+
+script:
+  - |
+    function periodic_echo {
+        while true; do
+            sleep 60
+            echo "Ensuring execution continues..."
+        done
+    }
+    function keep_alive {
+        periodic_echo &
+        local echo_pid=$!
+        $@ &
+        local command_pid=$!
+        wait $command_pid
+        local command_status=$?
+        kill -9 $echo_pid
+        if [ "$command_status" -ne "0" ]; then
+            exit $command_status
+        fi
+        return 0
+    }
+    # If the build is for a tag, use that as the image tag
+    # If not, use the output of git-describe
+    export ESGF_VERSION="${TRAVIS_TAG:-"$(git describe --always --tags)"}"
+    keep_alive docker-compose build
+    # If this is a pull request, we are done - all we wanted to do is check it builds
+    if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
+        echo "PR #${TRAVIS_PULL_REQUEST} built successfully - exiting"
+        exit
+    fi
+    # If it is not a pull request, then push the tag to Docker Hub
+    echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+    keep_alive docker-compose push
+    # If building master, retag with "latest" and push to Docker Hub
+    # If building devel, retag with "devel" and push to Docker Hub
+    # Any other branch (which shouldn't happen due to the whitelist above), we are done
+    if [ "$TRAVIS_BRANCH" = "master" ]; then
+        export ESGF_VERSION="latest"
+    elif [ "$TRAVIS_BRANCH" = "devel" ]; then
+        export ESGF_VERSION="devel"
+    else
+        exit
+    fi
+    keep_alive docker-compose build
+    keep_alive docker-compose push
+
+notifications:
+  email: false
+  slack:
+    secure: v3c0zFQbSa5D+tE6CpMZwztWCt7jYeMZeQlgwCrGHCTWduCQhbeeHjzjWLyqqEag/ecvm2k40zWOSFF9HBDwStnXDaW0LmRRhtAQgapNlCvtAxa5IQhTcPtMfki0CrA8xj58SCfHdGbVDRquwNUuvahlhg45ddJBO1mw8hrMS1KRutE3AAZUVTdVJ2N7D6x7qA7i2hKVKjGfGlmHN+eA5ja/nW0J28XRKRXX6ztfelWIy7RNMWnhy/bsjkHNeiaIV6UztTKRi0mIzZSu/DXGM9VvrUleNUTqn/w/CumNnIh+Oboktl1jarGojnW0njRPqZbQ0Xm9mxc/vAcA0vFjLUVdae/fTNKojW2j8GZ2dOf9bFabdLflup6KAlF1+8PoSW9vDtSquYt8Ez1KpTm5YxkwGtf74KklEzDNqVoCrOK8uZWYOd05g0A32FbRZDlPGdouk1oz8Tw0ShOTtUd96pd69l8PsViXN7PkLh8UQ3yQv8E/A3Y7SHYUnFbQPN0uANwhqE9S0m9X/FQHRL2KGNXdnBTUKXaa7nRGxol57lXwWTIQbRwyBfYQf174yfMroZZO12lhmdBDzau5gtKBnPt33lbbjoGQBgRrF2zWukRKuMWOg8H8UiHMgJhcK6MFfMQeB/B0SOw/3t3sLGNm/1grbkiS/8lOOs1d2ZBdK5s=

README.md

@@ -1,2 +1,7 @@
 # esgf-docker
-ESGF software stack as Docker images
+
+ESGF software stack as Docker images.
+
+## Documentation
+
+For documentation, please visit [cedadev.github.io/esgf-docker](https://cedadev.github.io/esgf-docker).

auth/Dockerfile

@@ -1,50 +1,81 @@
-# ESGF-Auth client application.
+#####
+## Image for running the ESGF Auth client Django application
+##
+## To avoid including unnecessary build dependencies in the final image, we use
+## a builder image to check out the code and build wheels for all our dependencies
+#####
 
-ARG ESGF_IMAGES_HUB=esgfhub
-ARG ESGF_VERSION=devel
-ARG ESGF_REPO=http://distrib-coffee.ipsl.jussieu.fr/pub/esgf
+ARG ESGF_HUB=esgfhub
+ARG ESGF_PREFIX=
+ARG ESGF_VERSION=latest
 
-FROM $ESGF_IMAGES_HUB/esgf-node:$ESGF_VERSION
-MAINTAINER Earth System Grid Federation <[email protected]>
+# This build stage is required because COPY --from=$ARG is not supported
+# https://github.com/moby/moby/issues/34482
+FROM ${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION} as configuration
+
+FROM python:2.7-slim AS esgf-auth-build
+
+# Install git for checking out the source code and pulling dependencies
+RUN apt-get update && \
+    apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/*
+
+# Clone the github repo and checkout the specified version
+ARG ESGF_AUTH_GIT_VERSION=master
+RUN git clone https://github.com/ESGF/esgf-auth.git /application && \
+    cd /application && \
+    git checkout $ESGF_AUTH_GIT_VERSION && \
+    rm -rf .git && \
+    cd -
 
-# installation directory
-ENV ESGF_AUTH_INSTALL_DIR=/usr/local/esgf-auth
+# Create wheels from the requirements
+RUN mkdir /pip-wheels && \
+    pip wheel --wheel-dir /pip-wheels git+https://github.com/philipkershaw/crypto-cookie.git && \
+    pip wheel --wheel-dir /pip-wheels psycopg2 && \
+    pip wheel --wheel-dir /pip-wheels -r /application/requirements.txt
 
-# library dependencies
-RUN yum -y update; yum -y install libffi-devel; yum clean all
 
-# create python virtual environment
-RUN mkdir -p $ESGF_AUTH_INSTALL_DIR && \
-    cd $ESGF_AUTH_INSTALL_DIR && \
-    virtualenv venv
+#############
+## Stage 2 ##
+#############
+
+FROM ${ESGF_HUB}/${ESGF_PREFIX}django:${ESGF_VERSION}
+
+MAINTAINER Earth System Grid Federation <[email protected]>
 
-# install source code
-RUN mkdir -p $ESGF_AUTH_INSTALL_DIR/src
+# Copy the application and wheels from the build stage
+COPY --from=esgf-auth-build /application /application
+COPY --from=esgf-auth-build /pip-wheels  /pip-wheels
 
-# install crypto cookie
-RUN cd $ESGF_AUTH_INSTALL_DIR && \
-    source venv/bin/activate && \
-    cd src && \
-    git clone https://github.com/philipkershaw/crypto-cookie.git && \
-    cd crypto-cookie && \
-    python setup.py install
+# Install the dependency wheels (remove the wheels when we are done)
+RUN pip install --no-index --find-links=/pip-wheels -r /application/requirements.txt && \
+    pip install --no-index --find-links=/pip-wheels crypto-cookie && \
+    pip install --no-index --find-links=/pip-wheels psycopg2 && \
+    rm -rf /pip-wheels
 
-# copy mock configuration file which is read by settings.py during the database creation
-COPY conf/esgf_auth_config.json /esg/config/esgf_auth_config.json
+# Because esgf-auth has no setup.py, we can't use pip install -e /application
+# Instead, we add /application to the PYTHONPATH
+ENV PYTHONPATH /application:$PYTHONPATH
 
-# install esgf-auth
-RUN cd $ESGF_AUTH_INSTALL_DIR && \
-    source venv/bin/activate && \
-    cd src && \
-    git clone https://github.com/lukaszlacinski/esgf-auth.git && \
-    cd esgf-auth && \
-    pip install -r requirements.txt && \
-    python ./manage.py migrate
+# Install ESGF Auth settings wrappers
+COPY python/*.py /usr/local/lib/python2.7/site-packages/
 
-# change permission on parent directory and database file so user apache can write
-RUN chmod a+w /usr/local/esgf-auth/src/esgf-auth && \
-    chmod a+w /usr/local/esgf-auth/src/esgf-auth/db.sqlite3
+# Install configuration files
+# Ensure the config directory is owned by the Django user and root group
+RUN mkdir -p /esg/config && chown 1001:0 /esg/config && chmod 775 /esg/config
+COPY --chown=1001:0 --from=configuration /esg/config /esg/config
+# Make sure the config interpolation runs first by giving it a low ordinal
+COPY --from=configuration /opt/esgf-docker/scripts/interpolate-configs.sh /django-init.d/01-interpolate.sh
+# Install the template auth config file
+ENV ESGF_AUTH_CONFIG_FILE /esg/auth/esgf_auth_config.json
+# Ensure /esg/auth exists and is owned by the Django user and root group
+RUN mkdir -p /esg/auth && chown 1001:0 /esg/auth && chmod 775 /esg/auth
+COPY conf/esgf_auth_config.json.template "$ESGF_AUTH_CONFIG_FILE.template"
+# This script interpolates the auth config files with additional environment variables
+COPY scripts/auth-configure.sh /django-init.d/02-auth-configure.sh
 
-VOLUME /usr/local/esgf-auth
+# Run as the Django user unless otherwise specified
+USER 1001
 
-CMD ["supervisord", "--nodaemon", "-c", "/etc/supervisord.conf"]
+# The "command" is just the settings module to use
+CMD ["esgf_auth_settings"]

auth/conf/esgf_auth_config.json

@@ -0,0 +1,5 @@
+{
+    "ESGF_HOSTNAME": "$ESGF_HOSTNAME",
+    "ESGF_SECRET_KEY": "$ESGF_COOKIE_SECRET_KEY",
+    "WEBAPP_SECRET_KEY": "$ESGF_AUTH_SECRET_KEY"
+}