Skip to content

Commit

Permalink
Merge pull request #126 from hic-infra/r-bioconductor
Browse files Browse the repository at this point in the history
Add reverse proxy for bioconductor (R)
  • Loading branch information
AaronJackson authored Oct 17, 2024
2 parents 6072911 + f20218a commit ca91ae1
Show file tree
Hide file tree
Showing 6 changed files with 282 additions and 1 deletion.
9 changes: 9 additions & 0 deletions reverse_proxy/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ zone: tre.example.org
cran_mirror: https://cran.ma.imperial.ac.uk/
cran_regex: ^/(bin/windows|src)/contrib/(4\.1|4\.3|4\.4|3\.6)*/*[a-zA-Z0-9_\-\.]*(\.tar\.gz|\.zip|PACKAGES)$

# bioconductor
bioconductor_mirror: https://bioconductor.statistik.tu-dortmund.de/
bioc_repo_regex: ^\/packages\/[0-9\.]+\/(bioc\/)?(src|bin\/windows)\/contrib\/[a-zA-Z0-9_\-\.]*(\.tar\.gz|\.zip)$
bioc_index_regex: ^\/packages\/[0-9\.]+\/(bioc|workflows)\/src\/contrib\/PACKAGES$
bioc_data_regex: ^\/packages\/[0-9\.]+\/(workflows|data\/(annotation|experiment))\/src\/contrib\/PACKAGES$
bioconductor_regex: '({{bioc_repo_regex}}|{{bioc_index_regex}}|{{bioc_data_regex}})'

# conda-forge
conda_repo_regex: ^/(conda-forge|bioconda)/(linux-64|linux-aarch64|win-64|noarch)/(current_)?repodata.json$
conda_pkg_regex: ^/(conda-forge|bioconda)/(linux-64|linux-aarch64|win-64|noarch)/[a-zA-Z0-9\-\._!]*(\.tar\.bz2|\.conda)$
Expand All @@ -24,3 +31,5 @@ cran_cert: ""
cran_cert_key: ""
conda_cert: ""
conda_cert_key: ""
bioconductor_cert: ""
bioconductor_cert_key: ""
172 changes: 172 additions & 0 deletions reverse_proxy/molecule/default/converge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,20 @@
line: 127.0.0.1 conda.{{zone}}
unsafe_writes: true # issue with testing inside a container

- name: add testing endpoint to /etc/hosts
become: true
lineinfile:
path: /etc/hosts
line: 127.0.0.1 bioconductor.{{zone}}
unsafe_writes: true # issue with testing inside a container

- name: break access to bioconductor.org for testing
become: true
lineinfile:
path: /etc/hosts
line: 255.255.255.255 bioconductor.org
unsafe_writes: true # issue with testing inside a container

- name: install R for testing package installation
become: true
apt:
Expand Down Expand Up @@ -73,3 +87,161 @@
channels:
- http://conda.{{ zone}}/conda-forge
ssl_verify: False
- name: create a bioconductor config for testing
copy:
dest: /root/config.yaml
content: |
---
output_dir: output
staging_deploy_root: [email protected]:/loc/www/bioconductor-test-new
production_deploy_root: [email protected]:/extra/www/bioc
staging_dir: /loc/www/bioconductor-test-new
release_version: "3.19"
r_version_associated_with_release: "4.4.0"
r_version_associated_with_devel: "4.4.0"
devel_version: "3.20"
versions:
- "3.19"
- "3.20"
devel_repos:
- "bioc"
r_ver_for_bioc_ver:
"1.6": "2.1"
"1.7": "2.2"
"1.8": "2.3"
"1.9": "2.4"
"2.0": "2.5"
"2.1": "2.6"
"2.2": "2.7"
"2.3": "2.8"
"2.4": "2.9"
"2.5": "2.10"
"2.6": "2.11"
"2.7": "2.12"
"2.8": "2.13"
"2.9": "2.14"
"2.10": "2.15"
"2.11": "2.15" # R switching to yearly releases, BioC remaining the same
"2.12": "3.0"
"2.13": "3.0"
"2.14": "3.1"
"3.0": "3.1"
"3.1": "3.2"
"3.2": "3.2"
"3.3": "3.3"
"3.4": "3.3"
"3.5": "3.4"
"3.6": "3.4"
"3.7": "3.5"
"3.8": "3.5"
"3.9": "3.6"
"3.10": "3.6"
"3.11": "4.0"
"3.12": "4.0"
"3.13": "4.1"
"3.14": "4.1"
"3.15": "4.2"
"3.16": "4.2"
"3.17": "4.3"
"3.18": "4.3"
"3.19": "4.4"
"3.20": "4.4"
# UPDATE THIS when we release a version
release_dates: # old info from http://en.wikipedia.org/wiki/Bioconductor#Milestones
"1.0": "1/5/2001"
"1.1": "11/19/2002"
"1.2": "5/29/2003"
"1.3": "10/30/2003"
"1.4": "5/17/2004"
"1.5": "10/25/2004"
"1.6": "5/18/2005"
"1.7": "10/14/2005"
"1.8": "4/27/2006"
"1.9": "10/4/2006"
"2.0": "4/26/2007"
"2.1": "10/8/2007"
"2.2": "5/1/2008"
"2.3": "10/22/2008"
"2.4": "4/21/2009"
"2.5": "10/28/2009"
"2.6": "4/23/2010"
"2.7": "10/18/2010"
"2.8": "4/14/2011"
"2.9": "11/1/2011"
"2.10": "4/2/2012"
"2.11": "10/3/2012"
"2.12": "4/4/2013"
"2.13": "10/15/2013"
"2.14": "4/14/2014"
"3.0": "10/14/2014"
"3.1": "4/17/2015"
"3.2": "10/14/2015"
"3.3": "5/4/2016"
"3.4": "10/18/2016"
"3.5": "4/25/2017"
"3.6": "10/31/2017"
"3.7": "05/01/2018"
"3.8": "10/31/2018"
"3.9": "05/03/2019"
"3.10": "10/30/2019"
"3.11": "04/28/2020"
"3.12": "10/28/2020"
"3.13": "05/20/2021"
"3.14": "10/27/2021"
"3.15": "04/27/2022"
"3.16": "11/02/2022"
"3.17": "04/26/2023"
"3.18": "10/25/2023"
"3.19": "05/01/2024"
release_last_built_dates:
'2.0': 09/12/2007
'2.1': 04/09/2008
'2.2': 10/16/2008
'2.3': 04/18/2009
'2.4': 10/20/2009
'2.5': 04/03/2010
'2.6': 10/05/2010
'2.7': 04/07/2011
'2.8': 10/21/2011
'2.9': 03/24/2012
'2.10': 09/24/2012
'2.11': 03/25/2013
'2.12': 10/09/2013
'2.13': 04/05/2014
'2.14': 10/08/2014
'3.0': 04/10/2015
'3.1': 10/09/2015
'3.2': 04/23/2016
'3.3': 10/13/2016
'3.4': 04/15/2017
'3.5': 10/18/2017
'3.6': 04/12/2018
'3.7': 10/17/2018
'3.8': 04/16/2019
'3.9': 10/16/2019
'3.10': 04/15/2020
'3.11': 10/17/2020
'3.12': 05/06/2021
'3.13': 10/15/2021
'3.14': 04/13/2022
'3.15': 10/19/2022
'3.16': 04/12/2023
'3.17': 10/16/2023
'3.18': 04/17/2024
mirrors:
- 0-Bioconductor:
- institution: Bioconductor, automatic redirection to servers worldwide
institution_url: https://bioconductor.org
contact: Bioconductor Maintainer
contact_email: [email protected]
city: World-wide
mirror_url: http://bioconductor.localhost/
#https_mirror_url: https://bioconductor.org/
rsync: secure_mirror_from_master
country_code: us
84 changes: 84 additions & 0 deletions reverse_proxy/molecule/default/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,90 @@
fail_msg: "GET queries should fail"
success_msg: "GET query was sucessfully blocked"


######################################################################
# bioconductor tests

- name: test connection to /
uri:
url: http://bioconductor.{{zone}}/
force: true # avoid hitting cache
failed_when: false
changed_when: false
register: bioconductor_curl_root

- name: test connection to /
assert:
that: bioconductor_curl_root.status == 403
fail_msg: "bioconductor mirror requests without file extensions should fail"
success_msg: "bioconductor request without file extension was successfully blocked"

- name: test connection to packages/3.20/bioc/src/contrib/PACKAGES
uri:
url: http://bioconductor.{{zone}}/packages/3.20/bioc/src/contrib/PACKAGES
force: true # avoid hitting cache
failed_when: false
changed_when: false
register: bioconductor_curl_packages

- name: test connection to /packages/3.20/bioc/src/contrib/PACKAGES
assert:
that: bioconductor_curl_packages.status == 200
fail_msg: "Request to valid file extension was blocked."
success_msg: "Request to valid file extension was not blocked."

- name: test connection data/annotation/src/contrib/chicken.db_3.13.0.tar.gz
uri:
url: http://bioconductor.{{zone}}/packages/3.20/data/annotation/src/contrib/chicken.db_3.13.0.tar.gz
force: true # avoid hitting cache
failed_when: false
changed_when: false
register: bioconductor_data1

- name: test connection data/annotation/src/contrib/chicken.db_3.13.0.tar.gz
assert:
that: bioconductor_data1.status == 403
fail_msg: "bioconductor requests to annotation data should fail"
success_msg: "bioconductor requests to annotation data failed correctly"

- name: test connection data/experiment/src/contrib/colonCA_1.46.0.tar.gz
uri:
url: http://bioconductor.{{zone}}/packages/release/data/experiment/src/contrib/colonCA_1.46.0.tar.gz
force: true # avoid hitting cache
failed_when: false
changed_when: false
register: bioconductor_data2

- name: test connection data/experiment/src/contrib/colonCA_1.46.0.tar.gz
assert:
that: bioconductor_data2.status == 403
fail_msg: "bioconductor requests to experiment data should fail"
success_msg: "bioconductor requests to experiment data failed correctly"

- name: bioconductor package installation
shell:
cmd: |
rm -rf /usr/local/lib/R/site-library/BiocVersion # remove if installed
/usr/bin/R --no-save <<RPROG
install.packages("BiocManager")
options(BIOCONDUCTOR_CONFIG_FILE="file:///root/config.yaml")
options(BioC_mirror="http://bioconductor.localhost")
options(BIOCMANAGER_SITE_REPOSITORY="http://bioconductor.localhost")
library(BiocManager)
BiocManager::install("BiocVersion")
q()
RPROG
ls /usr/local/lib/R/site-library/BiocVersion
failed_when: false
changed_when: false
register: bioc_install

- name: ensure bioconductor package installed successfully
assert:
that: bioc_install.rc == 0
fail_msg: "Failed to install BiocManager from local proxy"
success_msg: "BiocManager was successfully installed via local proxy"

######################################################################
# conda-forge tests

Expand Down
15 changes: 15 additions & 0 deletions reverse_proxy/tasks/bioconductor.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---

- name: add our bioconductor reverse proxy config
become: true
template:
dest: /etc/nginx/sites-enabled/bioconductor.conf
src: generic.conf.j2
vars:
generic_name: bioconductor
upstream_endpoint: "{{ bioconductor_mirror }}"
eicar_path: packages/src/contrib/eicar.tar.gz
endpoint_regex: "{{ bioconductor_regex }}"
certificate_file: "{{ bioconductor_cert }}"
certificate_key_file: "{{ bioconductor_cert_key }}"
notify: restart nginx
1 change: 1 addition & 0 deletions reverse_proxy/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@
state: absent

- include_tasks: cran.yml
- include_tasks: bioconductor.yml
- include_tasks: conda.yml
2 changes: 1 addition & 1 deletion squid/molecule/default/molecule.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ provisioner:
group_vars:
all:
allowed_domains:
- dundee.ac.uk
- eu.httpbin.org
verifier:
name: ansible

0 comments on commit ca91ae1

Please sign in to comment.