Merge pull request #40 from hic-infra/condaproxy

Conda reverse proxy

reverse_proxy/defaults/main.yml

@@ -5,3 +5,10 @@ zone: tre.example.org
 # cran mirror
 cran_mirror: https://cran.ma.imperial.ac.uk/
 cran_regex: ^/(bin/windows|src)/contrib/(4\.1|4\.3|3\.6)*/*[a-zA-Z0-9_\-\.]*(\.tar\.gz|\.zip|PACKAGES)$
+
+# conda-forge
+conda_repo_regex: ^/conda-forge/(linux-64|linux-aarch64|win-64|noarch)/(current_)?repodata.json$
+conda_pkg_regex: ^/conda-forge/(linux-64|linux-aarch64|win-64|noarch)/[a-zA-Z0-9\-\._!]*(\.tar\.bz2|\.conda)$
+
+# av scanning
+av_scanner: /opt/sophos-spl/plugins/av/bin/avscanner

reverse_proxy/molecule/default/converge.yml

@@ -13,6 +13,13 @@
         line: 127.0.0.1 cran.{{zone}}
         unsafe_writes: true # issue with testing inside a container
 
+    - name: add testing endpoint to /etc/hosts
+      become: true
+      lineinfile:
+        path: /etc/hosts
+        line: 127.0.0.1 conda.{{zone}}
+        unsafe_writes: true # issue with testing inside a container
+
     - name: install R for testing package installation
       become: true
       apt:
@@ -55,3 +62,22 @@
           /usr/bin/clamscan -d /tmp/clamav.hdb $1
         dest: /opt/sophos-spl/plugins/av/bin/avscanner
         mode: u+rwx,g+rx,o+rx
+
+    - name: download conda for testing
+      get_url:
+        # ansible_arch is likely either x86_64 or aarch64 which is the same naming format as miniconda uses.
+        url: https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-{{ ansible_architecture }}.sh
+        dest: /Miniconda3.sh
+      register: conda_download
+
+    - name: install conda
+      shell: bash /Miniconda3.sh -b
+      when: conda_download.changed
+
+    - name: setup conda to use local proxy
+      copy:
+        dest: /root/.condarc
+        content: |
+          channels:
+            - http://conda.{{ zone}}/conda-forge
+          ssl_verify: False

reverse_proxy/molecule/default/verify.yml

@@ -150,3 +150,51 @@
       that: r_curl_query.status == 403
       fail_msg: "GET queries should fail"
       success_msg: "GET query was sucessfully blocked"
+
+######################################################################
+# conda-forge tests
+
+  - name: test connection to /
+    uri:
+      url: http://conda.{{zone}}/
+      force: true  # avoid hitting cache
+    failed_when: false
+    changed_when: false
+    register: conda_curl_root
+
+  - name: test connection to /
+    assert:
+      that: conda_curl_root.status == 403
+      fail_msg: "Conda mirror requests without file extensions should fail"
+      success_msg: "Conda request without file extension was successfully blocked"
+
+  - name: test connection to /conda-forge/noarch/current_repodata.json
+    uri:
+      url: http://conda.{{zone}}/conda-forge/noarch/current_repodata.json
+      force: true  # avoid hitting cache
+    failed_when: false
+    changed_when: false
+    register: conda_curl_repodata
+
+  - name: test connection to /conda-forge/noarch/current_repodata.json
+    assert:
+      that: conda_curl_repodata.status == 200
+      fail_msg: "Request to valid repodata was blocked."
+      success_msg: "Request to valid repodata was not allowed."
+
+  - name: perform installation of a package (numpy)
+    shell: |
+      eval "$(/root/miniconda3/bin/conda shell.bash hook)"
+      env=$(cat /proc/sys/kernel/random/uuid)
+      conda create -n "$env" numpy
+      conda activate "$env"
+      python -c 'import numpy'
+    failed_when: false
+    changed_when: false
+    register: conda_install
+
+  - name: ensure numpy installation was successful
+    assert:
+      that: conda_install.rc == 0
+      fail_msg: "Failed to install conda package (numpy) from local proxy"
+      success_msg: "Conda package (numpy) was successfully installed via local proxy"

reverse_proxy/tasks/cloudwatch.yml

@@ -0,0 +1,3 @@
+---
+
+- name: add nginx config to awslogs

reverse_proxy/tasks/conda.yml

@@ -0,0 +1,13 @@
+---
+
+- name: add our conda-forge reverse proxy config
+  become: true
+  template:
+    dest: /etc/nginx/sites-enabled/conda.conf
+    src: generic.conf.j2
+  vars:
+    generic_name: conda
+    upstream_endpoint: https://conda.anaconda.org/
+    eicar_path: conda-forge/eicar/download/noarch/eicar.conda
+    endpoint_regex: "({{ conda_pkg_regex }}|{{ conda_repo_regex }})"
+  notify: restart nginx

reverse_proxy/tasks/cran.yml

@@ -4,5 +4,10 @@
   become: true
   template:
     dest: /etc/nginx/sites-enabled/cran.conf
-    src: cran.conf.j2
+    src: generic.conf.j2
+  vars:
+    generic_name: cran
+    upstream_endpoint: "{{ cran_mirror }}"
+    eicar_path: src/contrib/eicar.tar.gz
+    endpoint_regex: "{{ cran_regex }}"
   notify: restart nginx

reverse_proxy/tasks/main.yml

@@ -13,3 +13,4 @@
     state: absent
 
 - include_tasks: cran.yml
+- include_tasks: conda.yml

reverse_proxy/templates/cran.conf.j2 → reverse_proxy/templates/generic.conf.j2

@@ -1,23 +1,23 @@
 server {
   listen 80;
-  server_name cran.{{zone}};
+  server_name {{ generic_name }}.{{zone}};
 
   # Internal only endpoint for performing a proxy pass to the cran mirror.
-  location /cran-fetch {
+  location /{{ generic_name }}-fetch {
     internal;
-    proxy_pass {{cran_mirror}};
+    proxy_pass {{ upstream_endpoint }};
   }
 
   # Define an EICAR endpoint for verifying that the reverse proxy is
   # scanning correctly.
-  location /cran-fetch/src/contrib/eicar.tar.gz {
+  location /{{ generic_name }}-fetch/{{ eicar_path }} {
     internal;
     rewrite_by_lua_block {
       ngx.say("X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*")
     }
   }
 
-  location ~ {{cran_regex}} {
+  location ~ {{ endpoint_regex }} {
     # ensure we are only processing GET requests
     rewrite_by_lua_block {
       function reject ()
@@ -43,7 +43,14 @@ server {
 
       -- Perform a fetch using the internal cran-fetch endpoint. Everything after the
       -- "/cran-fetch" is passed to the configured cran mirror.
-      local res = ngx.location.capture("/cran-fetch" .. ngx.var.uri);
+      ngx.log(ngx.STDERR, ngx.var.uri)
+      local res = ngx.location.capture("/{{ generic_name }}-fetch" .. ngx.var.uri);
+
+      -- Get the headers to ensure we return the correct Content-Type
+      local ctype = res.header['Content-Type']
+      local cenc  = res.header['Content-Encoding']
+      ngx.header["Content-Type"] = ctype
+      ngx.header["Content-Encoding"] = cenc
 
       local path = os.tmpname()
 
@@ -53,7 +60,7 @@ server {
 
       -- Launch the antivirus scanner on the temporary file.
       local t_start = os.clock()
-      local ret = os.execute("/opt/sophos-spl/plugins/av/bin/avscanner " .. path)
+      local ret = os.execute("{{ av_scanner }} " .. path)
       local t_end = os.clock()
 
       os.remove(path)
@@ -64,7 +71,7 @@ server {
       ngx.header["X-Sophos-Scan-Timer"] = tostring(t_end - t_start) .. " cpu seconds"
 
       if ret == 0 then
-        ngx.say(res.body)
+        ngx.print(res.body)
       else
         ngx.status = ngx.HTTP_BAD_GATEWAY
         ngx.header["Content-Type"] = "text/plain"