added cvmfs provision playbook and removed g++

Author: jyotipm29

galaxy/Dockerfile

@@ -11,7 +11,6 @@ MAINTAINER Björn A. Grüning, [email protected]
 # * README: only Docker next to Docker is supported
 # * NodeJS is getting globally installed via the playbook, this is not needed anymore isn't it?
 # * the playbooks are not cleaning anything up
-# * autofs is big and should be installed during startup, only in case --priv is enables ?
 #
 ARG GALAXY_RELEASE
 ARG GALAXY_REPO
@@ -28,8 +27,8 @@ ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_24.1} \
 ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \
     GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \
     GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \
-    GALAXY_CONFIG_TOOL_CONFIG_FILE=/etc/galaxy/tool_conf.xml \
-    GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \
+    GALAXY_CONFIG_TOOL_CONFIG_FILE=$GALAXY_CONFIG_DIR/tool_conf.xml \
+    GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=$GALAXY_CONFIG_DIR/tool_data_table_conf.xml \
     GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \
     GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \
     GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \
@@ -163,8 +162,10 @@ ADD sample_tool_list.yaml $GALAXY_HOME/ephemeris/sample_tool_list.yaml
 # Activate Interactive Tools during runtime
 ADD ./tools_conf_interactive.xml.sample $GALAXY_INTERACTIVE_TOOLS_CONFIG_FILE
 
-RUN chown $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_DIR/reports.yml $GALAXY_HOME/ephemeris/sample_tool_list.yaml $GALAXY_INTERACTIVE_TOOLS_CONFIG_FILE
-RUN chmod 0644 $GALAXY_CONFIG_DIR/reports.yml $GALAXY_HOME/ephemeris/sample_tool_list.yaml $GALAXY_INTERACTIVE_TOOLS_CONFIG_FILE
+RUN curl -o $GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH \
+    -L https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/8adb1f82c94fe95b09df2a2816440ce2420b7d39/env/main/files/galaxy/config/tool_data_table_conf.xml \
+    && chown $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_DIR/reports.yml $GALAXY_HOME/ephemeris/sample_tool_list.yaml $GALAXY_INTERACTIVE_TOOLS_CONFIG_FILE $GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH \
+    && chmod 0644 $GALAXY_CONFIG_DIR/reports.yml $GALAXY_HOME/ephemeris/sample_tool_list.yaml $GALAXY_INTERACTIVE_TOOLS_CONFIG_FILE $GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH
 
 # Make python3 standard
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
@@ -216,12 +217,14 @@ RUN ansible-playbook /ansible/proftpd_provision.yml \
     --extra-vars proftpd_sql_password=$GALAXY_USER \
     --extra-vars galaxy_ftp_upload_dir=$EXPORT_DIR/ftp
 
+# Install cvmfs
+RUN ansible-playbook /ansible/cvmfs_provision.yml
+
 RUN mkdir -p /shed_tools \
     && chown $GALAXY_USER:$GALAXY_USER /shed_tools \
     && ln -s /tool_deps/ $EXPORT_DIR/tool_deps \
     # Configure Galaxy to use the Tool Shed
     && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps \
-    && apt update -qq && apt install --no-install-recommends -y g++ make \
     && ansible-playbook /ansible/provision.yml \
     --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \
     --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \
@@ -247,15 +250,15 @@ RUN mkdir -p /shed_tools \
     --extra-vars tus_upload_store_path=$GALAXY_CONFIG_TUS_UPLOAD_STORE \
     --extra-vars gx_it_proxy_sessions_path=$GALAXY_CONFIG_INTERACTIVETOOLS_MAP \
     --extra-vars use_pbkdf2=True \
-    --tags=galaxyextras,cvmfs --skip-tags=tusd,flower,redis,slurm,condor,proftpd -c local \
+    --tags=galaxyextras --skip-tags=tusd,flower,redis,slurm,condor,proftpd,cvmfs -c local \
     && . $GALAXY_VIRTUAL_ENV/bin/activate \
     && pip install WeasyPrint \
     && deactivate \
     # TODO: no clue why this is needed here again
     && cd $GALAXY_ROOT_DIR && ./scripts/common_startup.sh \
     && cd config && find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + \
     && find . -name '.cache' -type d -prune -exec rm -rf '{}' + \
-    && apt purge software-properties-common dirmngr gpg gpg-agent g++ make gcc -y && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ~/.cache/ \
+    && apt purge software-properties-common dirmngr gpg gpg-agent -y && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ~/.cache/ \
     # cleanup dance
     && find $GALAXY_ROOT_DIR/ -name '*.pyc' -delete | true \
     && find /usr/lib/ -name '*.pyc' -delete | true \

galaxy/ansible/cvmfs_provision.yml

@@ -0,0 +1,81 @@
+# Setup of the CernVM-File system (CVMFS) and configure so that the reference
+# data hosted by Galaxy on usegalaxy.org is available to the remote target.
+- hosts: localhost
+  connection: local
+  remote_user: root
+  vars_files:
+    - vars/main.yml
+  tasks:
+    - name: Install CernVM apt key
+      apt_key:
+        url: https://cvmrepo.web.cern.ch/cvmrepo/apt/cernvm.gpg
+
+    # Install & setup CermVM-FS
+    - name: Configure CernVM apt repository
+      apt_repository:
+        filename: "cernvm.list"
+        mode: 422
+        repo: deb [allow-insecure=true] https://cvmrepo.web.cern.ch/cvmrepo/apt/ {{ ansible_distribution_release }}-prod main
+
+    - name: Install CernVM-FS client (apt)
+      apt:
+        name: ['cvmfs', 'cvmfs-config']
+        state: present
+        update_cache: yes
+
+    - name: Make CernVM-FS key directories
+      file:
+        state: directory
+        path: "{{ item }}"
+        owner: "root"
+        group: "root"
+        mode: "0755"
+      loop: "{{ cvmfs_keys | map(attribute='path') | map('dirname') | unique }}"
+
+    - name: Install CernVM-FS keys
+      copy:
+        content: "{{ item.key }}"
+        dest: "{{ item.path }}"
+        owner: "root"
+        group: "root"
+        mode: "0444"
+      with_items: "{{ cvmfs_keys }}"
+
+    - name: Perform AutoFS and FUSE configuration for CernVM-FS
+      command: cvmfs_config setup
+
+    - name: Configure CernVM-FS config repository
+      block:
+
+        - name: Create config repo config
+          copy:
+            content: |
+              CVMFS_SERVER_URL="{{ cvmfs_config_repo.urls | join(';') }}"
+              CVMFS_PUBLIC_KEY="{{ cvmfs_config_repo.key.path }}"
+            dest: "/etc/cvmfs/config.d/{{ cvmfs_config_repo.repository.repository }}.conf"
+            owner: "root"
+            group: "root"
+            mode: "0444"
+
+        - name: Set config repo defaults
+          copy:
+            content: |
+              CVMFS_CONFIG_REPOSITORY="{{ cvmfs_config_repo.repository.repository }}"
+              CVMFS_DEFAULT_DOMAIN="{{ cvmfs_config_repo.domain }}"
+              CVMFS_USE_GEOAPI="{{ cvmfs_config_repo.use_geoapi | default('yes') }}"
+            dest: "/etc/cvmfs/default.d/80-galaxyproject-cvmfs.conf"
+            owner: "root"
+            group: "root"
+            mode: "0444"
+
+    - name: Configure CernVM-FS global client settings
+      copy:
+        content: |
+          CVMFS_HTTP_PROXY="{{ cvmfs_http_proxies | default(['DIRECT']) | join(';') }}"
+          CVMFS_QUOTA_LIMIT="{{ cvmfs_quota_limit | default('4000') }}"
+          CVMFS_CACHE_BASE="{{ cvmfs_cache_base | default('/var/lib/cvmfs') }}"
+          CVMFS_USE_GEOAPI="{{ cvmfs_use_geoapi | default('yes') }}"
+        dest: "/etc/cvmfs/default.local"
+        owner: "root"
+        group: "root"
+        mode: "0644"

galaxy/ansible/vars/main.yml

@@ -0,0 +1,39 @@
+cvmfs_config_repo:
+  domain: galaxyproject.org
+  key:
+    path: /etc/cvmfs/keys/galaxyproject.org/cvmfs-config.galaxyproject.org.pub
+    key: |
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJZTWTY3/dBfspFKifv8
+      TWuuT2Zzoo1cAskKpKu5gsUAyDFbZfYBEy91qbLPC3TuUm2zdPNsjCQbbq1Liufk
+      uNPZJ8Ubn5PR6kndwrdD13NVHZpXVml1+ooTSF5CL3x/KUkYiyRz94sAr9trVoSx
+      THW2buV7ADUYivX7ofCvBu5T6YngbPZNIxDB4mh7cEal/UDtxV683A/5RL4wIYvt
+      S5SVemmu6Yb8GkGwLGmMVLYXutuaHdMFyKzWm+qFlG5JRz4okUWERvtJ2QAJPOzL
+      mAG1ceyBFowj/r3iJTa+Jcif2uAmZxg+cHkZG5KzATykF82UH1ojUzREMMDcPJi2
+      dQIDAQAB
+      -----END PUBLIC KEY-----
+  urls:
+    - http://cvmfs1-psu0.galaxyproject.org/cvmfs/@fqrn@
+    - http://cvmfs1-iu0.galaxyproject.org/cvmfs/@fqrn@
+    - http://cvmfs1-tacc0.galaxyproject.org/cvmfs/@fqrn@
+    - http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@
+    - http://cvmfs1-mel0.gvl.org.au/cvmfs/@fqrn@
+  repository:
+    repository: cvmfs-config.galaxyproject.org
+    stratum0: cvmfs0-psu0.galaxyproject.org
+    owner: "root"
+    server_options: []
+    client_options: []
+
+cvmfs_keys:
+  - path: /etc/cvmfs/keys/galaxyproject.org/cvmfs-config.galaxyproject.org.pub
+    key: |
+      -----BEGIN PUBLIC KEY-----
+      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJZTWTY3/dBfspFKifv8
+      TWuuT2Zzoo1cAskKpKu5gsUAyDFbZfYBEy91qbLPC3TuUm2zdPNsjCQbbq1Liufk
+      uNPZJ8Ubn5PR6kndwrdD13NVHZpXVml1+ooTSF5CL3x/KUkYiyRz94sAr9trVoSx
+      THW2buV7ADUYivX7ofCvBu5T6YngbPZNIxDB4mh7cEal/UDtxV683A/5RL4wIYvt
+      S5SVemmu6Yb8GkGwLGmMVLYXutuaHdMFyKzWm+qFlG5JRz4okUWERvtJ2QAJPOzL
+      mAG1ceyBFowj/r3iJTa+Jcif2uAmZxg+cHkZG5KzATykF82UH1ojUzREMMDcPJi2
+      dQIDAQAB
+      -----END PUBLIC KEY-----

galaxy/startup.sh

@@ -488,10 +488,6 @@ fi
 
 if $PRIVILEGED; then
     # in privileged mode autofs and CVMFS is available
-    # install autofs
-    echo "Installing autofs to enable automatic CVMFS mounts"
-    apt-get install autofs --no-install-recommends -y
-    apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/*
     export GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH="/etc/galaxy/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml"
 
     echo "Enable Galaxy Interactive Tools."