helm: Add chart for cloud credentials and helm values for env variables

Signed-off-by: Aaron Wilson <[email protected]>

helm/README.md

@@ -63,10 +63,11 @@ helmfile delete --environment <your-env>
 
 If you only want to modify one part of the installation, it is possible to run the charts individually in `./ais/charts` with their own `values.yaml` files.
 
-| Chart             | Description                                                                                       |
-|-------------------|---------------------------------------------------------------------------------------------------|
-| [ais-operator](https://github.com/NVIDIA/ais-k8s/releases)  | Deploy the AIS operator -- our helmfile deploys the chart generated from our latest AIS operator release |
-| [ais-cluster](./ais/charts/ais-cluster/Chart.yaml)  | Create an AIS cluster resource, with the expectation the operator is already deployed           |
-| [ais-create-pv](./ais/charts/create-pv/Chart.yaml)  | Create persistent volumes to be used by AIS targets           |
-| [tls-issuer](./ais/charts/tls-issuer/Chart.yaml)  | Create a cert-manager Issuer for self-signed certs           |
-| [tls-cert](./ais/charts/tls-cert/Chart.yaml)  | Create a cert-manager certificate           |
+| Chart                                                      | Description                                                                                              |
+|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|
+| [ais-cloud-secrets](./ais/charts/cloud-secrets/Chart.yaml) | Create k8s secrets from local files for cloud backends                                                   |
+| [ais-cluster](./ais/charts/ais-cluster/Chart.yaml)         | Create an AIS cluster resource, with the expectation the operator is already deployed                    |
+| [ais-create-pv](./ais/charts/create-pv/Chart.yaml)         | Create persistent volumes to be used by AIS targets                                                      |
+| [ais-operator](https://github.com/NVIDIA/ais-k8s/releases) | Deploy the AIS operator -- our helmfile deploys the chart generated from our latest AIS operator release |
+| [tls-cert](./ais/charts/tls-cert/Chart.yaml)               | Create a cert-manager certificate                                                                        |
+| [tls-issuer](./ais/charts/tls-issuer/Chart.yaml)           | Create a cert-manager Issuer for self-signed certs                                                       |

helm/ais/charts/ais-cluster/templates/ais.yaml

@@ -18,6 +18,10 @@ spec:
     {{- end }}
   enablePromExporter: false
   proxySpec:
+    env:
+      {{- with .Values.proxySpec.env }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
     {{- if .Values.proxySpec.size }}
     size: {{ .Values.proxySpec.size }}
     {{- end }}
@@ -37,6 +41,10 @@ spec:
       {{- toYaml .Values.proxySpec.resources | nindent 6 }}
     {{- end }}
   targetSpec:
+    env:
+      {{- with .Values.targetSpec.env }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
     {{- if .Values.targetSpec.size }}
     size: {{ .Values.targetSpec.size }}
     {{- end }}

helm/ais/charts/cloud-secrets/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v2
+name: ais-cloud-secrets
+version: 0.1.0

helm/ais/charts/cloud-secrets/templates/aws-secret.yaml

@@ -0,0 +1,11 @@
+{{- if and .Values.aws_config .Values.aws_credentials }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.cloud.awsSecretName }}
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  config: {{ .Values.aws_config | b64enc }}
+  credentials: {{ .Values.aws_credentials | b64enc }}
+{{- end }}

helm/ais/charts/cloud-secrets/templates/gcp-secret.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.gcp_json }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.cloud.gcpSecretName }}
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  gcp.json: {{ .Values.gcp_json | b64enc }}
+{{- end }}

helm/ais/config/ais/sjc11.yaml

@@ -36,16 +36,16 @@ initImage:
   name: aistorage/ais-init
   tag: v3.26-a7ac713
 cloud:
-  awsSecretName: ""
-  gcpSecretName: ""
+  awsSecretName: "aws-creds"
+  gcpSecretName: "gcp-creds"
 multihome:
   enabled: false
   hostnameMap: 
   networkAttachment:
 hostNetwork: false
 shutdownCluster: false
 cleanupData: false
-cleanupMetadata: false
+cleanupMetadata: true
 stateStorageClass: "local-path"
 authNSecretName:
 logsDir:
@@ -56,6 +56,9 @@ proxySpec:
     limits:
       cpu: "12"
 targetSpec:
+  env: 
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/var/gcp/gcp.json"
   resources:
     requests:
       cpu: "24"

helm/ais/config/ais/values-sample.yaml

@@ -46,5 +46,9 @@ proxySpec:
   size: 3
   resources:
 targetSpec:
+  # currently REQUIRED to be provided IF cloud.gcpSecretName is set above
+  env: 
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/var/gcp/gcp.json"
   size: 3
   resources:

helm/ais/config/cloud/sjc11.yaml.gotmpl

@@ -0,0 +1,8 @@
+aws_config: |-
+{{ readFile (printf "%s/.aws/sjc11/config" (env "HOME")) | indent 2 }}
+
+aws_credentials: |-
+{{ readFile (printf "%s/.aws/sjc11/credentials" (env "HOME")) | indent 2 }}
+
+gcp_json: |-
+{{ readFile (printf "%s/.gcp/sjc11/gcp.json" (env "HOME")) | indent 2 }}

helm/ais/helmfile.yaml

@@ -7,6 +7,9 @@ environments:
     values: 
     - https:
         enabled: true
+    # Only controls secret creation, update ais config "cloud" to use 
+    - cloud-secrets:
+        enabled: true
     kubeContext: sjc11
   oci-iad:
     values: 
@@ -43,7 +46,18 @@ releases:
     values:
       - "./config/tls-cert/{{ .Environment.Name }}.yaml"
       - "./config/ais/{{ .Environment.Name }}.yaml"
-
+
+  # Sets up cloud secrets for AIS pods to mount and use for accessing remote backends
+  - name: ais-cloud-secrets
+    namespace: ais
+    createNamespace: true
+    chart: charts/cloud-secrets
+    version: 0.1.0
+    condition: cloud-secrets.enabled
+    values:
+      - "./config/cloud/{{ .Environment.Name }}.yaml.gotmpl"
+      - "./config/ais/{{ .Environment.Name }}.yaml"
+
   - name: ais-cluster
     namespace: ais
     createNamespace: true

operator/README.md

@@ -155,9 +155,38 @@ spec:
 
 ### Config Backend Provider for GCP & AWS
 
-AIS operator supports GCP and AWS as the config backend provider. To enable the config backend provider, you need to create a secret with the corresponding credential file.
+AIS operator supports configuring  GCP and AWS as cloud providers for buckets. To enable the config for these providers, you need to create a secret with the corresponding credential file.
 
-```yaml
+#### Helm
+Helm deployments include a [chart](../helm/ais/charts/cloud-secrets/Chart.yaml) for generating these secrets based on local config and credentials. 
+  1. Update the environment in the provided [helmfile](../helm/ais/helmfile.yaml) with the `cloud-secrets` variable: 
+  ```yaml 
+      - cloud-secrets:
+        enabled: true
+  ```
+
+  2. Create a `<env_name>.yaml.gotmpl` file in [helm/ais/config/cloud/](../helm/ais/config/cloud/)
+
+  3. Add references to the local files you want to use. Example for sjc11 (be sure to update your paths correctly):
+  ```yaml
+  aws_config: |-
+  {{ readFile (printf "%s/.aws/sjc11/config" (env "HOME")) | indent 2 }}
+
+  aws_credentials: |-
+  {{ readFile (printf "%s/.aws/sjc11/credentials" (env "HOME")) | indent 2 }}
+
+  gcp_json: |-
+  {{ readFile (printf "%s/.gcp/sjc11/gcp.json" (env "HOME")) | indent 2 }}
+  ```
+
+#### Ansible
+For ansible deployments, see the [ais_aws_config](../playbooks/cloud/ais_aws_config.yml) and [ais_gcp_config](../playbooks/cloud/ais_gcp_config.yml) playbooks and the associated [README](../playbooks/cloud/README.md).
+
+
+#### Manual
+You can also create the secrets manually:
+
+```bash
 kubectl create secret -n ais-operator-system generic aws-creds \
   --from-file=config=$HOME/.aws/config \
   --from-file=credentials=$HOME/.aws/credentials
@@ -166,6 +195,8 @@ kubectl create secret -n ais-operator-system generic gcp-creds \
   --from-file=gcp.json=<path-to-gcp-credential-file>.json
 ```
 
+Once the secrets are created, update the AIS config yaml to reference the secrets:
+
 ```yaml
 # config/samples/ais_v1beta1_sample.yaml
 apiVersion: ais.nvidia.com/v1beta1
@@ -178,6 +209,16 @@ spec:
 ...
 ```
 
+Finally, for **GCP** configs, the environment variable for the location **MUST** be provided through the `targetSpec.Env` section (For ansible, it is included in the default template). As of writing, the operator will always mount the provided secret to `/var/gcp`, so for a secret with `data.gcp.json` the resulting file location in the pod will be `var/gcp/gcp.json` :
+
+```yaml
+targetSpec:
+  env: 
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/var/gcp/gcp.json"
+```
+
+
 ### Enabling HTTPS for AIStore Deployment in Kubernetes
 
 While the examples above demonstrate running web servers that accept plain HTTP requests, you may want to enhance the security of your AIStore deployment by enabling HTTPS in a Kubernetes environment.

playbooks/ais-deployment/roles/ais_deploy_cluster/templates/ais.yaml.j2

@@ -40,6 +40,9 @@ spec:
 {% endif %}
 
   targetSpec:
+    env: 
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: /var/gcp/gcp.json
     hostPort: 51081
     servicePort: 51081
     portPublic: 51081