Scrape notebook & hub prometheus metrics

- /metrics now responds with prometheus metrics, rather than
  nbresuse metrics (jupyter-server/jupyter-resource-usage#68)
- Notebook Server's /metrics endpoint is now available without
  authentication, so prometheus can scrape those metrics
  (jupyter/notebook#5870)
- Annotations are set up on the user pods to tell prometheus the
  port and path to scrape.
- NetworkPolicy is set up to allow prometheus server to hit the
  singleuser server and the hub pods. Note that this required
  manually editing netpol in prometheus - should add that feature
  to the prometheus chart.

Author: yuvipanda

hub/values.yaml

@@ -19,6 +19,10 @@ etcJupyter:
       # https://github.com/jupyter/jupyter/issues/174
       # https://github.com/ipython/ipython/issues/9163
       db_file: ":memory:"
+    NotebookApp:
+      authenticate_prometheus: false
+    ResourceUseDisplay:
+      disable_legacy_endpoint: true
 
 nfsPVC:
   enabled: true
@@ -114,6 +118,10 @@ jupyterhub:
       SHELL: /bin/bash
     startTimeout: 600 # 10 mins, because sometimes we have too many new nodes coming up together
     defaultUrl: /tree
+    extraAnnotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/path: "/user/{username}/metrics"
+      prometheus.io/port: "8888"
     networkPolicy:
       # In clusters with NetworkPolicy enabled, do not
       # allow outbound internet access that's not DNS, FTP, HTTP or HTTPS
@@ -134,6 +142,20 @@ jupyterhub:
             # Allow FTP access https://github.com/berkeley-dsep-infra/datahub/issues/1789
             - port: 21
               protocol: TCP
+      # Allow ingress from promethetheus scraper in support namespace
+      # This lets us get notebook metrics!
+      ingress:
+        - ports:
+            - port: 8888
+              protocol: TCP
+          from:
+          - namespaceSelector:
+              matchLabels:
+                name: support
+            podSelector:
+              matchLabels:
+                app: prometheus
+                component: server
   auth:
     google:
       hostedDomain: [berkeley.edu]
@@ -161,6 +183,13 @@ jupyterhub:
             - namespaceSelector:
                 matchLabels:
                   name: support
+              podSelector:
+                matchLabels:
+                  app: prometheus
+                  component: server
+          ports:
+            - port: http
+              protocol: TCP
     livenessProbe:
       enabled: true
       initialDelaySeconds: 180