add support to push to remote registry (#18)

* add support to push to remote registry * ensure default to deploy is true Signed-off-by: vsoch <[email protected]>
converged-computing · Nov 30, 2023 · 320b000 · 320b000
1 parent 2627284
commit 320b000
Show file tree

Hide file tree

Showing 18 changed files with 344 additions and 31 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@
 # Binaries for programs and plugins
 ./vendor
 *.exe
+.env
 *.exe~
 *.dll
 *.so

diff --git a/api/v1alpha1/orascache_types.go b/api/v1alpha1/orascache_types.go
@@ -23,9 +23,29 @@ type OrasCacheSpec struct {
  // +optional
  Image string `json:"image"`
 
+ // Names of secrets for the operator
+ // +optional
+ Secrets Secrets `json:"secrets"`
+
+ // Skip deploying the registry (stateful set) implying all references
+ // are for a remote (existing) registry
+ // +kubebuilder:default=true
+ // +default=true
+ // +optional
+ Deploy bool `json:"deploy"`
+}
+
+type Secrets struct {
+
+ // Secrets for the environment for the ORAS operator sidecar pod to push
+ // e.g., oras pull -u username -p password myregistry.io/myimage:latest
+ // This should have ORAS_USER and ORAS_PASS
+ // +optional
+ OrasEnv string `json:"orasEnv"`
+
  // Secret for the registry REGISTRY_HTTP_SECRET
  // +optional
- Secret string `json:"secret"`
+ RegistryHttp string `json:"registryHttp"`
 }
 
 // OrasCacheStatus defines the observed state of OrasCache

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/cache.converged-computing.github.io_orascaches.yaml b/config/crd/bases/cache.converged-computing.github.io_orascaches.yaml
@@ -35,13 +35,28 @@ spec:
  spec:
  description: OrasCacheSpec defines the desired state of OrasCache
  properties:
+ deploy:
+ default: true
+ description: Skip deploying the registry (stateful set) implying all
+ references are for a remote (existing) registry
+ type: boolean
  image:
  default: ghcr.io/oras-project/registry:latest
  description: Image is the oras registry to deploy
  type: string
- secret:
- description: Secret for the registry REGISTRY_HTTP_SECRET
- type: string
+ secrets:
+ description: Names of secrets for the operator
+ properties:
+ orasEnv:
+ description: Secrets for the environment for the ORAS operator
+ sidecar pod to push e.g., oras pull -u username -p password
+ myregistry.io/myimage:latest This should have ORAS_USER and
+ ORAS_PASS
+ type: string
+ registryHttp:
+ description: Secret for the registry REGISTRY_HTTP_SECRET
+ type: string
+ type: object
  type: object
  status:
  description: OrasCacheStatus defines the observed state of OrasCache

diff --git a/controllers/oras/orascache_controller.go b/controllers/oras/orascache_controller.go
@@ -107,9 +107,12 @@ func (r *OrasCacheReconciler) ensureOrasCache(
  }
 
  // The service running the oras registry is a stateful set
- _, result, _, err = r.getStatefulSet(ctx, spec)
- if err != nil {
- return result, err
+ // But only deploy if we are requested to!
+ if spec.Spec.Deploy {
+ _, result, _, err = r.getStatefulSet(ctx, spec)
+ if err != nil {
+ return result, err
+ }
  }
  return ctrl.Result{}, nil
 }

diff --git a/controllers/oras/statefulset.go b/controllers/oras/statefulset.go
@@ -120,15 +120,30 @@ func (r *OrasCacheReconciler) createStatefulSet(
  },
  }
 
+ // Prepare environment variables
  // Do we have a secret?
- if spec.Spec.Secret != "" {
- env := []corev1.EnvVar{{
+ if spec.Spec.Secrets.RegistryHttp != "" {
+ env := []corev1.EnvVar{}
+ env = append(env, corev1.EnvVar{
  Name: "REGISTRY_HTTP_SECRET",
- Value: spec.Spec.Secret,
- }}
+ Value: spec.Spec.Secrets.RegistryHttp,
+ })
  set.Spec.Template.Spec.Containers[0].Env = env
  }
 
+ // Are we also adding environment variables from a secret?
+ if spec.Spec.Secrets.OrasEnv != "" {
+ set.Spec.Template.Spec.Containers[0].EnvFrom = []v1.EnvFromSource{
+ {
+ SecretRef: &corev1.SecretEnvSource{
+ LocalObjectReference: corev1.LocalObjectReference{
+ Name: spec.Spec.Secrets.OrasEnv,
+ },
+ },
+ },
+ }
+ }
+
  // Controller reference always needs to be set before creation
  ctrl.SetControllerReference(spec, set, r.Scheme)
  err := r.Client.Create(ctx, set)

diff --git a/docs/getting_started/custom-resource-definition.md b/docs/getting_started/custom-resource-definition.md
@@ -43,14 +43,40 @@ spec:
  image: ghcr.io/oras-project/registry:latest
 ```
 
-#### secret
+#### deploy
 
-In the case of deploying more than one registry, the push secret can be customized.
+If deploy is set to false, this indicates that you don't want the operator to deploy a stateful set local registry.
+This means that you'll need to provide all jobs / pods with a remote `registry` field to replace it, and (likely) a secret to pull or push.
+See [orasEnv](#orasEnv) below for details.
 
 ```yaml
 spec:
- # We can use all the defaults here (this is a default)
- secret: mysecret
+ deploy: false 
+```
+
+#### secrets
+
+There are several secrets that can be added, if needed.
+
+##### orasEnv
+
+If you deploy a secret to the namespace that you want the ORAS operator to use (providing in the environment to the pod)
+you can define the name here.
+
+```yaml
+spec:
+ secrets: 
+ orasEnv: mysecret-name
+```
+
+##### registryHttp
+
+In the case of deploying more than one registry pod (not supported yet), the push secret can be customized.
+
+```yaml
+spec:
+ secrets: 
+ registryHttp: mysecret
 ```
 
 Note that this is not supported yet - likely we would want to add custom volumes (for shared storage) between more than one pod in the stateful set. 

diff --git a/docs/getting_started/user-guide.md b/docs/getting_started/user-guide.md
@@ -149,15 +149,38 @@ once, first for the job, and then for the underlying pod(s) it creates. Annotati
 | output-uri | The output unique resource identifier for the registry step, including repository, name, and tag | false | false |NA will be used if not defined, meaning the step has no outputs |
 | oras-cache | The name of the sidecar orchestrator | false | false | oras |
 | oras-container | The container with oras to run for the service | false | false | ghcr.io/oras-project/oras:v1.1.0 |
+| registry | A custom remote registry URI for all containers | false | false | unset |
+| oras-env | A secret in the same namespace to add to the sidecar | false | false | unset |
 | container | The name of the launcher container | false | false | assumes the first container found requires the launcher |
 | entrypoint | The https address of the application entrypoint to wget | false | false | [entrypoint.sh](https://raw.githubusercontent.com/converged-computing/oras-operator/main/hack/entrypoint.sh) |
 | oras-entrypoint | The https address of the oras cache sidecar entrypoint to wget | false | false | [oras-entrypoint.sh](https://raw.githubusercontent.com/converged-computing/oras-operator/main/hack/oras-entrypoint.sh) |
 | debug | Print all discovered settings in the operator log | false | false | "false" |
 | unpack | Unpack a directory compressed to .tar.gz | false | false | "true" |
 
 There should not be a need to change the oras-cache (sidecar container) unless for some reason you have another container in the pod also called oras. It is exposed for this rare case.
+More details about specific annotations are provided below.
 
-Note that when List is true, this means the annotation can be provided as a list, and more than one value can be added with the pattern `<prefix>/<field>_<count>`. Currently the only supported list field is `input-uri`, anticipating that multiple parent steps might feed into one child step.
+#### Registry
+
+By default, the URIs that are provided are prefixed with the registry that the ORAS Operator has deployed on the same cluster. However, if registry is supplied, 
+it is assumed to be used for the entire job or pod definition, and this registry is used instead. If you absolutely don't need the local registry you can
+set "deploy" to false in the operator CRD (and it will be skipped). Also note that if you are pushing to a remote registry, you'll need to provide
+the name of a secret with your login and password via `oras-env`, discussed next.
+
+#### oras-env
+
+This should be the name of a secret in the same namespace as your pod or job that has one or more secrets to add to the environment. As an example to create such a secret:
+
+```bash
+kubectl create secret generic oras-env --from-literal="ORAS_USER=${ORAS_USER}" --from-literal="ORAS_PUSH_PASS=${ORAS_PASS}" --from-literal="ORAS_PULL_PASS=${ORAS_PASS}"
+```
+The most common use case here is to provide credentials for pushing or pulling to a remote registry (when deploy is false). We provide the same credential twice for pull and push because it could
+be the case that you only want to add for one or the other, and in fact adding to a command that doesn't require it could result in an error. The secret must be in the same namespace
+as your job or pod.
+
+#### Multiple input-uri
+
+Note that when List is true, this means the annotation can be provided as a list, and more than one value can be added with the pattern `<prefix>/<field>_<count>`. Currently the only supported list field is `input-uri`, anticipating that multiple parent steps might feed into one child step. Here is an example with multiple input uris.
 
 ```yaml
 kind: Pod
@@ -215,6 +238,7 @@ kubectl logs -n oras-operator-system oras-operator-controller-manager-ff66845dd-
 You can then try one of the [examples](https://github.com/converged-computing/oras-operator/tree/main/examples) in the repository. A brief description of each is provided here,
 and likely we will add more detail as we develop them.
 
- - [tests/registry](https://github.com/converged-computing/oras-operator/tree/main/examples/tests/registry/roas.yaml): a basic example of creating an ORAS registry cache
+ - [registry](https://github.com/converged-computing/oras-operator/tree/main/examples/registry): demonstrates interaction with a remote registry and disable of a local (namespaced) deployment
+ - [tests/registry](https://github.com/converged-computing/oras-operator/tree/main/examples/tests/registry/oras.yaml): a basic example of creating an ORAS registry cache
  - [tests/hello-world](https://github.com/converged-computing/oras-operator/tree/main/examples/tests/hello-world): Writing one "hello-world" output and saving to an ORAS registry cache
  - [workflows/metrics](https://github.com/converged-computing/oras-operator/tree/main/examples/woirkflow/metrics): Running two Metrics Operator apps/metrics (LAMMPS and HWLOC) and getting results for each!
diff --git a/examples/dist/oras-operator-arm.yaml b/examples/dist/oras-operator-arm.yaml
@@ -53,13 +53,24 @@ spec:
  spec:
  description: OrasCacheSpec defines the desired state of OrasCache
  properties:
+ deploy:
+ default: true
+ description: Skip deploying the registry (stateful set) implying all references are for a remote (existing) registry
+ type: boolean
  image:
  default: ghcr.io/oras-project/registry:latest
  description: Image is the oras registry to deploy
  type: string
- secret:
- description: Secret for the registry REGISTRY_HTTP_SECRET
- type: string
+ secrets:
+ description: Names of secrets for the operator
+ properties:
+ orasEnv:
+ description: Secrets for the environment for the ORAS operator sidecar pod to push e.g., oras pull -u username -p password myregistry.io/myimage:latest This should have ORAS_USER and ORAS_PASS
+ type: string
+ registryHttp:
+ description: Secret for the registry REGISTRY_HTTP_SECRET
+ type: string
+ type: object
  type: object
  status:
  description: OrasCacheStatus defines the observed state of OrasCache

diff --git a/examples/dist/oras-operator-dev.yaml b/examples/dist/oras-operator-dev.yaml
@@ -53,13 +53,24 @@ spec:
  spec:
  description: OrasCacheSpec defines the desired state of OrasCache
  properties:
+ deploy:
+ default: true
+ description: Skip deploying the registry (stateful set) implying all references are for a remote (existing) registry
+ type: boolean
  image:
  default: ghcr.io/oras-project/registry:latest
  description: Image is the oras registry to deploy
  type: string
- secret:
- description: Secret for the registry REGISTRY_HTTP_SECRET
- type: string
+ secrets:
+ description: Names of secrets for the operator
+ properties:
+ orasEnv:
+ description: Secrets for the environment for the ORAS operator sidecar pod to push e.g., oras pull -u username -p password myregistry.io/myimage:latest This should have ORAS_USER and ORAS_PASS
+ type: string
+ registryHttp:
+ description: Secret for the registry REGISTRY_HTTP_SECRET
+ type: string
+ type: object
  type: object
  status:
  description: OrasCacheStatus defines the observed state of OrasCache

diff --git a/examples/dist/oras-operator.yaml b/examples/dist/oras-operator.yaml
@@ -53,13 +53,24 @@ spec:
  spec:
  description: OrasCacheSpec defines the desired state of OrasCache
  properties:
+ deploy:
+ default: true
+ description: Skip deploying the registry (stateful set) implying all references are for a remote (existing) registry
+ type: boolean
  image:
  default: ghcr.io/oras-project/registry:latest
  description: Image is the oras registry to deploy
  type: string
- secret:
- description: Secret for the registry REGISTRY_HTTP_SECRET
- type: string
+ secrets:
+ description: Names of secrets for the operator
+ properties:
+ orasEnv:
+ description: Secrets for the environment for the ORAS operator sidecar pod to push e.g., oras pull -u username -p password myregistry.io/myimage:latest This should have ORAS_USER and ORAS_PASS
+ type: string
+ registryHttp:
+ description: Secret for the registry REGISTRY_HTTP_SECRET
+ type: string
+ type: object
  type: object
  status:
  description: OrasCacheStatus defines the observed state of OrasCache