From 66eb7e5b7b214d9d28f3c36f70eb16f91672fae5 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 7 Oct 2023 13:41:55 +0100 Subject: [PATCH] Add notes/config on AWS EKS VPC CNI --- .prettierignore | 1 + terraform/aws/curvenote/cni/README.md | 20 + .../curvenote/cni/aws-k8s-cni-us-east-2.yaml | 564 ++++++++++++++++++ 3 files changed, 585 insertions(+) create mode 100644 terraform/aws/curvenote/cni/README.md create mode 100644 terraform/aws/curvenote/cni/aws-k8s-cni-us-east-2.yaml diff --git a/.prettierignore b/.prettierignore index 9df8843e7..0f7ad7494 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,2 @@ mybinder/templates/ +terraform/aws/curvenote/cni/ diff --git a/terraform/aws/curvenote/cni/README.md b/terraform/aws/curvenote/cni/README.md new file mode 100644 index 000000000..0f4f175e9 --- /dev/null +++ b/terraform/aws/curvenote/cni/README.md @@ -0,0 +1,20 @@ +# Enable NetworkPolicies on EKS + +EKS automatically installs the VPC CNI plugin, but by default NetworkPolicies are not enabled. + +1. Find the recommended version of the VPC CNI plugin + https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html +2. Download the VPC-CNI Kubernetes manifest, replacing `1.15.0` with the recommended version + ``` + curl -O https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.15.0/config/master/aws-k8s-cni.yaml + ``` +3. Edit `aws-k8s-cni.yaml`: + - Change all mentions of `us-west-2` to your region + - Update the manifest following the `kubectl` instructions in + https://docs.aws.amazon.com/eks/latest/userguide/cni-network-policy.html + - Add `enable-network-policy-controller: "true"` to the `aws-node` ConfigMap + - Set `--enable-network-policy=true` in the `aws-node` DaemonSet `aws-network-policy-agent` container +4. Apply: + ``` + kubectl apply -f cni/aws-k8s-cni.yaml + ``` diff --git a/terraform/aws/curvenote/cni/aws-k8s-cni-us-east-2.yaml b/terraform/aws/curvenote/cni/aws-k8s-cni-us-east-2.yaml new file mode 100644 index 000000000..f7886dc0d --- /dev/null +++ b/terraform/aws/curvenote/cni/aws-k8s-cni-us-east-2.yaml @@ -0,0 +1,564 @@ +--- +# Source: crds/customresourcedefinition.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: eniconfigs.crd.k8s.amazonaws.com +spec: + scope: Cluster + group: crd.k8s.amazonaws.com + preserveUnknownFields: false + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + names: + plural: eniconfigs + singular: eniconfig + kind: ENIConfig +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + labels: + app.kubernetes.io/name: amazon-network-policy-controller-k8s + name: policyendpoints.networking.k8s.aws +spec: + group: networking.k8s.aws + names: + kind: PolicyEndpoint + listKind: PolicyEndpointList + plural: policyendpoints + singular: policyendpoint + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: PolicyEndpoint is the Schema for the policyendpoints API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PolicyEndpointSpec defines the desired state of PolicyEndpoint + properties: + egress: + description: Egress is the list of egress rules containing resolved + network addresses + items: + description: EndpointInfo defines the network endpoint information + for the policy ingress/egress + properties: + cidr: + description: CIDR is the network address(s) of the endpoint + type: string + except: + description: Except is the exceptions to the CIDR ranges mentioned + above. + items: + type: string + type: array + ports: + description: Ports is the list of ports + items: + description: Port contains information about the transport + port/protocol + properties: + endPort: + description: Endport specifies the port range port to + endPort port must be defined and an integer, endPort + > port + format: int32 + type: integer + port: + description: Port specifies the numerical port for the + protocol. If empty applies to all ports + format: int32 + type: integer + protocol: + default: TCP + description: Protocol specifies the transport protocol, + default TCP + type: string + type: object + type: array + required: + - cidr + type: object + type: array + ingress: + description: Ingress is the list of ingress rules containing resolved + network addresses + items: + description: EndpointInfo defines the network endpoint information + for the policy ingress/egress + properties: + cidr: + description: CIDR is the network address(s) of the endpoint + type: string + except: + description: Except is the exceptions to the CIDR ranges mentioned + above. + items: + type: string + type: array + ports: + description: Ports is the list of ports + items: + description: Port contains information about the transport + port/protocol + properties: + endPort: + description: Endport specifies the port range port to + endPort port must be defined and an integer, endPort + > port + format: int32 + type: integer + port: + description: Port specifies the numerical port for the + protocol. If empty applies to all ports + format: int32 + type: integer + protocol: + default: TCP + description: Protocol specifies the transport protocol, + default TCP + type: string + type: object + type: array + required: + - cidr + type: object + type: array + podIsolation: + description: PodIsolation specifies whether the pod needs to be isolated + for a particular traffic direction Ingress or Egress, or both. If + default isolation is not specified, and there are no ingress/egress + rules, then the pod is not isolated from the point of view of this + policy. This follows the NetworkPolicy spec.PolicyTypes. + items: + description: PolicyType string describes the NetworkPolicy type + This type is beta-level in 1.8 + type: string + type: array + podSelector: + description: PodSelector is the podSelector from the policy resource + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelectorEndpoints: + description: PodSelectorEndpoints contains information about the pods + matching the podSelector + items: + description: PodEndpoint defines the summary information for the + pods + properties: + hostIP: + description: HostIP is the IP address of the host the pod is + currently running on + type: string + name: + description: Name is the pod name + type: string + namespace: + description: Namespace is the pod namespace + type: string + podIP: + description: PodIP is the IP address of the pod + type: string + required: + - hostIP + - name + - namespace + - podIP + type: object + type: array + policyRef: + description: PolicyRef is a reference to the Kubernetes NetworkPolicy + resource. + properties: + name: + description: Name is the name of the Policy + type: string + namespace: + description: Namespace is the namespace of the Policy + type: string + required: + - name + - namespace + type: object + required: + - policyRef + type: object + status: + description: PolicyEndpointStatus defines the observed state of PolicyEndpoint + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: aws-vpc-cni/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aws-node + namespace: kube-system + labels: + app.kubernetes.io/name: aws-node + app.kubernetes.io/instance: aws-vpc-cni + k8s-app: aws-node + app.kubernetes.io/version: "v1.15.0" +--- +# Source: aws-vpc-cni/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: amazon-vpc-cni + namespace: kube-system + labels: + app.kubernetes.io/name: aws-node + app.kubernetes.io/instance: aws-vpc-cni + k8s-app: aws-node + app.kubernetes.io/version: "v1.15.0" +data: + enable-windows-ipam: "false" + enable-network-policy-controller: "true" +--- +# Source: aws-vpc-cni/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aws-node + labels: + app.kubernetes.io/name: aws-node + app.kubernetes.io/instance: aws-vpc-cni + k8s-app: aws-node + app.kubernetes.io/version: "v1.15.0" +rules: + - apiGroups: + - crd.k8s.amazonaws.com + resources: + - eniconfigs + verbs: ["list", "watch", "get"] + - apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch", "get"] + - apiGroups: [""] + resources: + - pods + verbs: ["list", "watch", "get"] + - apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch", "get"] + - apiGroups: ["", "events.k8s.io"] + resources: + - events + verbs: ["create", "patch", "list"] + - apiGroups: ["networking.k8s.aws"] + resources: + - policyendpoints + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.aws"] + resources: + - policyendpoints/status + verbs: ["get"] + - apiGroups: + - vpcresources.k8s.aws + resources: + - cninodes + verbs: ["get", "list", "patch"] +--- +# Source: aws-vpc-cni/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: aws-node + labels: + app.kubernetes.io/name: aws-node + app.kubernetes.io/instance: aws-vpc-cni + k8s-app: aws-node + app.kubernetes.io/version: "v1.15.0" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-node +subjects: + - kind: ServiceAccount + name: aws-node + namespace: kube-system +--- +# Source: aws-vpc-cni/templates/daemonset.yaml +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: aws-node + namespace: kube-system + labels: + app.kubernetes.io/name: aws-node + app.kubernetes.io/instance: aws-vpc-cni + k8s-app: aws-node + app.kubernetes.io/version: "v1.15.0" +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + selector: + matchLabels: + k8s-app: aws-node + template: + metadata: + labels: + app.kubernetes.io/name: aws-node + app.kubernetes.io/instance: aws-vpc-cni + k8s-app: aws-node + spec: + priorityClassName: "system-node-critical" + serviceAccountName: aws-node + hostNetwork: true + initContainers: + - name: aws-vpc-cni-init + image: "602401143452.dkr.ecr.us-east-2.amazonaws.com/amazon-k8s-cni-init:v1.15.0" + env: + - name: DISABLE_TCP_EARLY_DEMUX + value: "false" + - name: ENABLE_IPv6 + value: "false" + securityContext: + privileged: true + resources: + requests: + cpu: 25m + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + terminationGracePeriodSeconds: 10 + tolerations: + - operator: Exists + securityContext: + {} + containers: + - name: aws-node + image: "602401143452.dkr.ecr.us-east-2.amazonaws.com/amazon-k8s-cni:v1.15.0" + ports: + - containerPort: 61678 + name: metrics + livenessProbe: + exec: + command: + - /app/grpc-health-probe + - -addr=:50051 + - -connect-timeout=5s + - -rpc-timeout=5s + initialDelaySeconds: 60 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /app/grpc-health-probe + - -addr=:50051 + - -connect-timeout=5s + - -rpc-timeout=5s + initialDelaySeconds: 1 + timeoutSeconds: 10 + env: + - name: ADDITIONAL_ENI_TAGS + value: "{}" + - name: AWS_VPC_CNI_NODE_PORT_SUPPORT + value: "true" + - name: AWS_VPC_ENI_MTU + value: "9001" + - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG + value: "false" + - name: AWS_VPC_K8S_CNI_EXTERNALSNAT + value: "false" + - name: AWS_VPC_K8S_CNI_LOGLEVEL + value: "DEBUG" + - name: AWS_VPC_K8S_CNI_LOG_FILE + value: "/host/var/log/aws-routed-eni/ipamd.log" + - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT + value: "prng" + - name: AWS_VPC_K8S_CNI_VETHPREFIX + value: "eni" + - name: AWS_VPC_K8S_PLUGIN_LOG_FILE + value: "/var/log/aws-routed-eni/plugin.log" + - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL + value: "DEBUG" + - name: DISABLE_INTROSPECTION + value: "false" + - name: DISABLE_METRICS + value: "false" + - name: DISABLE_NETWORK_RESOURCE_PROVISIONING + value: "false" + - name: ENABLE_IPv4 + value: "true" + - name: ENABLE_IPv6 + value: "false" + - name: ENABLE_POD_ENI + value: "false" + - name: ENABLE_PREFIX_DELEGATION + value: "false" + - name: WARM_ENI_TARGET + value: "1" + - name: WARM_PREFIX_TARGET + value: "1" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + resources: + requests: + cpu: 25m + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + - mountPath: /host/var/log/aws-routed-eni + name: log-dir + - mountPath: /var/run/aws-node + name: run-dir + - mountPath: /run/xtables.lock + name: xtables-lock + - name: aws-eks-nodeagent + image: "602401143452.dkr.ecr.us-east-2.amazonaws.com/amazon/aws-network-policy-agent:v1.0.2" + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + args: + - --enable-ipv6=false + - --enable-network-policy=true + - --enable-cloudwatch-logs=false + - --metrics-bind-addr=:8162 + - --health-probe-bind-addr=:8163 + resources: + requests: + cpu: 25m + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: true + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /sys/fs/bpf + name: bpf-pin-path + - mountPath: /var/log/aws-routed-eni + name: log-dir + - mountPath: /var/run/aws-node + name: run-dir + volumes: + - name: bpf-pin-path + hostPath: + path: /sys/fs/bpf + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + - name: log-dir + hostPath: + path: /var/log/aws-routed-eni + type: DirectoryOrCreate + - name: run-dir + hostPath: + path: /var/run/aws-node + type: DirectoryOrCreate + - name: xtables-lock + hostPath: + path: /run/xtables.lock + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate