[MRG] Adding Turing cluster to the Federation (#1203)

[MRG] Adding Turing cluster to the Federation

.travis.yml

@@ -32,6 +32,9 @@ before_deploy:
 - |
   # Stage 1: Install gcloud SDK
   curl -L https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-173.0.0-linux-x86_64.tar.gz | tar --directory ${HOME} --extract --gzip --file -
+- |
+  # Stage 1: Install Azure CLI
+  curl -L https://aka.ms/InstallAzureCli | bash
 - |
   # Stage 1: Install Kubectl
   mkdir -p ${HOME}/bin
@@ -59,6 +62,7 @@ before_deploy:
   # Stage 2, Step 2: Set up helm!
   helm init --client-only
   helm repo add jupyterhub https://jupyterhub.github.io/helm-chart
+  helm repo add jetstack https://charts.jetstack.io
   helm repo update
   (cd mybinder && helm dep up)
 - |
@@ -91,11 +95,17 @@ before_deploy:
   # Stage 5, Step 3: Deploy to production on ovh k8s
   python ./deploy.py ovh binder-ovh
 - |
-  # Stage 5, Step 4: Verify production works
+  # Stage 5, Step 4: Deploy to production on Turing k8s
+  python ./deploy.py turing turing
+- |
+  # Stage 5, Step 5: Verify production works
   travis_retry py.test -vx -n 2 --binder-url=https://gke.mybinder.org --hub-url=https://hub.gke.mybinder.org
 - |
-  # Stage 5, Step 5: Verify production on ovh k8s works
+  # Stage 5, Step 6: Verify production on ovh k8s works
   travis_retry py.test -vx -n 2 --binder-url=https://ovh.mybinder.org --hub-url=https://hub-binder.mybinder.ovh
+- |
+  # Stage 5, Step 7: Verify production on Turing k8s works
+  travis_retry py.test -vx -n 2 --binder-url=https://turing.mybinder.org --hub-url=https://hub.mybinder.turing.ac.uk
 
 
 env:

config/staging.yaml

@@ -122,5 +122,6 @@ federationRedirect:
       weight: 1
       health: https://gke2.staging.mybinder.org/health
       versions: https://gke2.staging.mybinder.org/versions
-    # unset the gesis entry
+    # unset the gesis and turing entries
     gesis: null
+    turing: null

config/turing.yaml

@@ -0,0 +1,200 @@
+projectName: turing
+
+tags:
+  kubelego: false
+  certmanager: true
+
+letsencrypt:
+  contactEmail: [email protected]
+
+binderhub:
+  extraConfig:
+    01-eventlog: ""
+  config:
+    BinderHub:
+      pod_quota: 20
+      hub_url: https://hub.mybinder.turing.ac.uk
+      badge_base_url: https://mybinder.org
+      image_prefix: turingmybinderregistry.azurecr.io/binder-prod/binder-prod-
+      sticky_builds: true
+    DockerRegistry:
+      token_url: https://turingmybinderregistry.azurecr.io/oauth2/token?service=turingmybinderregistry.azurecr.io
+  registry:
+    url: https://turingmybinderregistry.azurecr.io
+
+  replicas: 1
+
+  resources:
+    requests:
+      cpu: "0.25"
+      memory: 1Gi
+    limits:
+      cpu: "2"
+      memory: 1Gi
+
+  ingress:
+    enabled: true
+    hosts:
+      - binder.mybinder.turing.ac.uk
+      - turing.mybinder.org
+    annotations:
+      cert-manager.io/cluster-issuer: letsencrypt-prod
+      kubernetes.io/ingress.class: nginx
+      kubernetes.io/tls-acme: "true"
+      https:
+        enabled: true
+        type: nginx
+    tls:
+      - secretName: turing-binder-tls-crt
+        hosts:
+          - binder.mybinder.turing.ac.uk
+          - turing.mybinder.org
+
+  jupyterhub:
+    hub:
+      resources:
+        requests:
+          cpu: "0.25"
+          memory: 1Gi
+        limits:
+          cpu: "2"
+          memory: 1Gi
+    singleuser:
+      memory:
+        guarantee: 550M
+        limit: 2G
+      cpu:
+        guarantee: 0.01
+        limit: 1
+    proxy:
+      chp:
+        resources:
+          requests:
+            memory: 320Mi
+            cpu: "0.1"
+          limits:
+            memory: 320Mi
+            cpu: "0.5"
+      nginx:
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: "0.25"
+          limits:
+            memory: 512Mi
+            cpu: 1
+
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+        kubernetes.io/ingress.class: nginx
+        kubernetes.io/tls-acme: "true"
+        https:
+          enabled: true
+          type: nginx
+      hosts:
+        - hub.mybinder.turing.ac.uk
+      tls:
+        - secretName: turing-hub-tls-crt
+          hosts:
+            - hub.mybinder.turing.ac.uk
+
+    scheduling:
+      userScheduler:
+        enabled: false
+      podPriority:
+        enabled: true
+      userPlaceholder:
+        enabled: true
+        replicas: 5
+
+grafana:
+  ingress:
+    annotations:
+      cert-manager.io/cluster-issuer: letsencrypt-prod
+      kubernetes.io/ingress.class: nginx
+      kubernetes.io/tls-acme: 'true'
+    hosts:
+      - grafana.mybinder.turing.ac.uk
+    tls:
+      - secretName: turing-grafana-tls-crt
+        hosts:
+          - grafana.mybinder.turing.ac.uk
+
+  datasources:
+    datasources.yaml:
+      apiVersion: 1
+      datasources:
+        - name: prometheus
+          orgId: 1
+          type: prometheus
+          url: https://prometheus.mybinder.turing.ac.uk
+          access: direct
+          isDefault: true
+          editable: false
+
+prometheus:
+  server:
+    ingress:
+      annotations:
+        kubernetes.io/ingress.class: nginx
+        kubernetes.io/tls-acme: 'false'
+      hosts:
+        - prometheus.mybinder.turing.ac.uk
+      tls:
+        - hosts:
+            - prometheus.mybinder.turing.ac.uk
+          secretName: turing-prometheus-tls-crt
+
+
+nginx-ingress:
+  controller:
+    service:
+      loadBalancerIP: 51.105.120.231
+    config:
+      proxy-body-size: 64m
+
+
+static:
+  ingress:
+    annotations:
+      kubernetes.io/ingress.class: nginx
+      kubernetes.io/tls-acme: 'false'
+    hosts:
+      - static.mybinder.turing.ac.uk
+    tls:
+      - hosts:
+          - static.mybinder.turing.ac.uk
+        secretName: turing-static-tls-crt
+
+redirector:
+  redirects:
+    - type: host
+      host:
+        from: docs-mybinder.turing.10.0.0.1.xip.io
+        to: mybinder.readthedocs.io
+
+matomo:
+  enabled: false
+  db:
+    instanceName: binder-staging:us-central1:matomo
+    serviceAccountKey: ""
+  trustedHosts:
+    - staging-mybinder.turing.10.0.0.1.xip.io
+  ingress:
+    hosts:
+      - staging-mybinder.turing.10.0.0.1.xip.io
+
+analyticsPublisher:
+  enabled: false
+  project: binder-turing
+  events:
+    sourceBucket: mybinder-staging-events-raw-export
+    destinationBucket: mybinder-staging-events-archive
+
+gcsProxy:
+  enabled: false
+  buckets:
+    - name: mybinder-staging-events-archive
+      host: archive-analytics-staging-mybinder.turing.10.0.0.1.xip.io

deploy.py

@@ -14,6 +14,35 @@
 ABSOLUTE_HERE = os.path.dirname(os.path.realpath(__file__))
 
 
+def setup_auth_turing(cluster):
+    """
+    Set up athentication with Turing k8s cluster on Azure.
+    """
+    # Read in auth info
+    azure_file = os.path.join(ABSOLUTE_HERE, "secrets", "turing-auth-key-prod.json")
+    with open(azure_file, "r") as stream:
+        azure = json.load(stream)
+
+    # Login in to Azure
+    login_cmd = [
+        "az", "login", "--service-principal",
+        "--username", azure["sp-app-id"],
+        "--password", azure["sp-app-key"],
+        "--tenant", azure["tenant-id"]
+    ]
+    subprocess.check_output(login_cmd)
+
+    # Set kubeconfig
+    creds_cmd = [
+        "az", "aks", "get-credentials",
+        "--name", cluster,
+        "--resource-group", "binder-prod"
+
+    ]
+    stdout = subprocess.check_output(creds_cmd)
+    print(stdout.decode('utf-8'))
+
+
 def setup_auth_ovh(release, cluster):
     """
     Set up authentication with 'binder-ovh' K8S from the ovh-kubeconfig.yml
@@ -122,6 +151,7 @@ def deploy(release):
     subprocess.check_call([
         "python3",
         "secrets/ban.py",
+        release,
     ])
 
     print(BOLD + GREEN + f"Starting helm upgrade for {release}" + NC, flush=True)
@@ -170,7 +200,7 @@ def main():
     argparser.add_argument(
         'release',
         help="Release to deploy",
-        choices=['staging', 'prod', 'ovh']
+        choices=['staging', 'prod', 'ovh', 'turing']
     )
     argparser.add_argument(
         'cluster',
@@ -181,6 +211,8 @@ def main():
 
     if args.cluster == 'binder-ovh':
         setup_auth_ovh(args.release, args.cluster)
+    elif args.cluster == 'turing':
+        setup_auth_turing(args.cluster)
     else:
         setup_auth_gcloud(args.release, args.cluster)
 

mybinder/requirements.yaml

@@ -11,6 +11,13 @@ dependencies:
  - name: kube-lego
    version: 0.4.2
    repository: https://kubernetes-charts.storage.googleapis.com
+   tags:
+     - kubelego
+ - name: cert-manager
+   version: v0.12.0-beta.1
+   repository: https://charts.jetstack.io
+   tags:
+     - certmanager
  - name: binderhub
    version: 0.2.0-068.0b5080a
    repository: https://jupyterhub.github.io/helm-chart

mybinder/templates/cluster-issuer.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.tags.certmanager }}
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+  namespace: turing
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: [email protected]
+    privateKeySecretRef:
+      name: letsencrypt-staging
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+  namespace: turing
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: [email protected]
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
+{{- end }}

mybinder/values.yaml

@@ -1,3 +1,6 @@
+tags:
+  kubelego: true
+
 etcJupyter:
   jupyter_notebook_config.json:
     NotebookApp:
@@ -416,7 +419,7 @@ federationRedirect:
   hosts:
     gke:
       url: https://gke.mybinder.org
-      weight: 66
+      weight: 65
       health: https://gke.mybinder.org/health
       versions: https://gke.mybinder.org/versions
       prime: true
@@ -430,3 +433,8 @@ federationRedirect:
       weight: 15
       health: https://gesis.mybinder.org/health
       versions: https://gesis.mybinder.org/versions
+    turing:
+      url: https://turing.mybinder.org
+      weight: 1
+      health: https://turing.mybinder.org/health
+      versions: https://turing.mybinder.org/versions