Merge pull request #3903 from consideRatio/pr/res-all-stuff

deployer: fix misc resource allocation script details

Author: consideRatio

deployer/commands/generate/resource_allocation/daemonset_requests.yaml

@@ -25,19 +25,19 @@ gke:
     other_daemon_sets: binder-staging-dind,binder-staging-image-cleaner,imagebuilding-demo-binderhub-service-docker-api
     cpu_requests: 344m
     memory_requests: 596Mi
-    k8s_version: v1.27.4-gke.900
+    k8s_version: v1.27.10-gke.1055000
   2i2c-uk:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,gke-metrics-agent,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 344m
     memory_requests: 596Mi
-    k8s_version: v1.27.7-gke.1056000
+    k8s_version: v1.27.10-gke.1055000
   awi-ciroh:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,gke-metrics-agent,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 344m
     memory_requests: 596Mi
-    k8s_version: v1.27.4-gke.900
+    k8s_version: v1.27.10-gke.1055000
   callysto:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,gke-metrics-agent,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
@@ -49,126 +49,162 @@ gke:
     other_daemon_sets: ""
     cpu_requests: 338m
     memory_requests: 496Mi
-    k8s_version: v1.27.7-gke.1056000
+    k8s_version: v1.27.10-gke.1055000
   cloudbank:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,gke-metrics-agent,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 344m
     memory_requests: 596Mi
-    k8s_version: v1.27.5-gke.200
+    k8s_version: v1.27.10-gke.1055000
   hhmi:
     requesting_daemon_sets: fluentbit-gke,gke-metadata-server,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 228m
     memory_requests: 480Mi
-    k8s_version: v1.27.7-gke.1056000
+    k8s_version: v1.27.10-gke.1055000
   leap:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,gke-metrics-agent,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 344m
     memory_requests: 596Mi
-    k8s_version: v1.27.4-gke.900
+    k8s_version: v1.27.10-gke.1055000
   linked-earth:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,gke-metrics-agent,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 344m
     memory_requests: 596Mi
-    k8s_version: v1.27.4-gke.900
+    k8s_version: v1.27.10-gke.1055000
   meom-ige:
     requesting_daemon_sets: fluentbit-gke,gke-metadata-server,gke-metrics-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 234m
     memory_requests: 580Mi
-    k8s_version: v1.27.4-gke.900
+    k8s_version: v1.27.10-gke.1055000
   pangeo-hubs:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,gke-metrics-agent,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 344m
     memory_requests: 596Mi
-    k8s_version: v1.27.5-gke.200
+    k8s_version: v1.27.10-gke.1055000
   qcl:
     requesting_daemon_sets: calico-node,fluentbit-gke,gke-metadata-server,ip-masq-agent,netd,pdcsi-node,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 338m
     memory_requests: 496Mi
-    k8s_version: v1.27.7-gke.1056000
+    k8s_version: v1.27.10-gke.1055000
 eks:
   2i2c-aws-us:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
+  bican:
+    requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
+    other_daemon_sets: ""
+    cpu_requests: 195m
+    memory_requests: 250Mi
+    k8s_version: v1.29.1-eks-b9c9ed7
   catalystproject-africa:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
+  dandi:
+    requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
+    other_daemon_sets: ""
+    cpu_requests: 195m
+    memory_requests: 250Mi
+    k8s_version: v1.29.1-eks-b9c9ed7
+  earthscope:
+    requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
+    other_daemon_sets: ""
+    cpu_requests: 195m
+    memory_requests: 250Mi
+    k8s_version: v1.28.6-eks-508b6b3
   gridsst:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
+  jupyter-health:
+    requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
+    other_daemon_sets: ""
+    cpu_requests: 195m
+    memory_requests: 250Mi
+    k8s_version: v1.29.1-eks-b9c9ed7
   jupyter-meets-the-earth:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
+  linc:
+    requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
+    other_daemon_sets: ""
+    cpu_requests: 195m
+    memory_requests: 250Mi
+    k8s_version: v1.29.1-eks-b9c9ed7
   nasa-cryo:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
   nasa-esdis:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
   nasa-ghg:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
   nasa-veda:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
   openscapes:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
+  opensci:
+    requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
+    other_daemon_sets: ""
+    cpu_requests: 195m
+    memory_requests: 250Mi
+    k8s_version: v1.28.6-eks-508b6b3
   smithsonian:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
   ubc-eoas:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
   victor:
     requesting_daemon_sets: aws-node,ebs-csi-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: ""
     cpu_requests: 170m
     memory_requests: 250Mi
-    k8s_version: v1.27.8-eks-8cb36c9
+    k8s_version: v1.27.10-eks-508b6b3
 aks:
   utoronto:
     requesting_daemon_sets: cloud-node-manager,csi-azuredisk-node,csi-azurefile-node,kube-proxy,support-cryptnono,support-prometheus-node-exporter
     other_daemon_sets: calico-node
     cpu_requests: 226m
     memory_requests: 300Mi
-    k8s_version: v1.26.3
+    k8s_version: v1.28.3

deployer/commands/generate/resource_allocation/instance_capacities.py

@@ -132,11 +132,11 @@ def instance_capacities(
         props = ["cpu_capacity", "cpu_allocatable", "mem_capacity", "mem_allocatable"]
         for p in props:
             lp = f"{p}_low"
-            if new_cap[lp] < cap[lp]:
+            if parse_quantity(new_cap[lp]) < parse_quantity(cap[lp]):
                 cap[lp] = new_cap[lp]
         for p in props:
             lp = f"{p}_high"
-            if new_cap[lp] > cap[lp]:
+            if parse_quantity(new_cap[lp]) > parse_quantity(cap[lp]):
                 cap[lp] = new_cap[lp]
 
     # write

deployer/commands/generate/resource_allocation/instance_capacities.yaml

@@ -61,24 +61,6 @@ n2-highmem-32:
   mem_capacity_high: 251.897Gi
   mem_allocatable_low: 240.079Gi
   mem_allocatable_high: 240.079Gi
-n1-highmem-4:
-  cpu_capacity_low: 4.0
-  cpu_capacity_high: 4.0
-  cpu_allocatable_low: 3.92
-  cpu_allocatable_high: 3.92
-  mem_capacity_low: 25.451Gi
-  mem_capacity_high: 25.451Gi
-  mem_allocatable_low: 22.192Gi
-  mem_allocatable_high: 22.192Gi
-e2-highmem-16:
-  cpu_capacity_low: 16.0
-  cpu_capacity_high: 16.0
-  cpu_allocatable_low: 15.89
-  cpu_allocatable_high: 15.89
-  mem_capacity_low: 125.807Gi
-  mem_capacity_high: 125.807Gi
-  mem_allocatable_low: 116.549Gi
-  mem_allocatable_high: 116.549Gi
 n1-standard-2:
   cpu_capacity_low: 2.0
   cpu_capacity_high: 2.0
@@ -113,10 +95,10 @@ r5.xlarge:
   cpu_capacity_high: 4.0
   cpu_allocatable_low: 3.92
   cpu_allocatable_high: 3.92
-  mem_capacity_low: 30.887Gi
-  mem_capacity_high: 30.907Gi
-  mem_allocatable_low: 29.917Gi
-  mem_allocatable_high: 29.937Gi
+  mem_capacity_low: 30.886Gi
+  mem_capacity_high: 30.887Gi
+  mem_allocatable_low: 29.916Gi
+  mem_allocatable_high: 29.917Gi
 r5.2xlarge:
   cpu_capacity_low: 8.0
   cpu_capacity_high: 8.0
@@ -135,15 +117,24 @@ r5.4xlarge:
   mem_capacity_high: 124.364Gi
   mem_allocatable_low: 121.492Gi
   mem_allocatable_high: 121.504Gi
+r5.16xlarge:
+  cpu_capacity_low: 64.0
+  cpu_capacity_high: 64.0
+  cpu_allocatable_low: 63.77
+  cpu_allocatable_high: 63.77
+  mem_capacity_low: 498.372Gi
+  mem_capacity_high: 498.372Gi
+  mem_allocatable_low: 490.108Gi
+  mem_allocatable_high: 490.108Gi
 m5.large:
   cpu_capacity_low: 2.0
   cpu_capacity_high: 2.0
   cpu_allocatable_low: 1.93
   cpu_allocatable_high: 1.93
-  mem_capacity_low: 7.473Gi
-  mem_capacity_high: 7.473Gi
-  mem_allocatable_low: 6.815Gi
-  mem_allocatable_high: 6.815Gi
+  mem_capacity_low: 7.461Gi
+  mem_capacity_high: 7.543Gi
+  mem_allocatable_low: 6.803Gi
+  mem_allocatable_high: 6.885Gi
 m5.xlarge:
   cpu_capacity_low: 4.0
   cpu_capacity_high: 4.0
@@ -158,20 +149,29 @@ g4dn.xlarge:
   cpu_capacity_high: 4.0
   cpu_allocatable_low: 3.92
   cpu_allocatable_high: 3.92
-  mem_capacity_low: 15.333Gi
+  mem_capacity_low: 15.324Gi
   mem_capacity_high: 15.333Gi
-  mem_allocatable_low: 14.675Gi
+  mem_allocatable_low: 14.666Gi
   mem_allocatable_high: 14.675Gi
+g4dn.2xlarge:
+  cpu_capacity_low: 8.0
+  cpu_capacity_high: 8.0
+  cpu_allocatable_low: 7.91
+  cpu_allocatable_high: 7.91
+  mem_capacity_low: 30.886Gi
+  mem_capacity_high: 30.886Gi
+  mem_allocatable_low: 30.227Gi
+  mem_allocatable_high: 30.227Gi
 
 # AKS instance types
 Standard_E4s_v3:
   cpu_capacity_low: 4.0
   cpu_capacity_high: 4.0
   cpu_allocatable_low: 3.86
   cpu_allocatable_high: 3.86
-  mem_capacity_low: 31.354Gi
+  mem_capacity_low: 31.342Gi
   mem_capacity_high: 31.354Gi
-  mem_allocatable_low: 27.062Gi
+  mem_allocatable_low: 27.05Gi
   mem_allocatable_high: 27.062Gi
 Standard_E8s_v3:
   cpu_capacity_low: 8.0
@@ -182,3 +182,12 @@ Standard_E8s_v3:
   mem_capacity_high: 62.806Gi
   mem_allocatable_low: 56.594Gi
   mem_allocatable_high: 56.594Gi
+Standard_E8s_v5:
+  cpu_capacity_low: 8.0
+  cpu_capacity_high: 8.0
+  cpu_allocatable_low: 7.82
+  cpu_allocatable_high: 7.82
+  mem_capacity_low: 62.793Gi
+  mem_capacity_high: 62.793Gi
+  mem_allocatable_low: 56.581Gi
+  mem_allocatable_high: 56.581Gi