internal/controller: support custom atlas.hcl (#244)

README.md

@@ -63,6 +63,51 @@ To configure the operator, you can set the following values in the `values.yaml`
 
 - `prewarmDevDB`: The Operator always keeps devdb resources around to speed up the migration process. Set this to `false` to disable this feature.
 
+- `allowCustomConfig`: Enable this to allow custom `atlas.hcl` configuration. To use this feature, you can set the `config` field in the `AtlasSchema` or `AtlasMigration` resource.
+
+```yaml
+  spec:
+    envName: myenv
+    config: |
+      env myenv {}
+    # config from secretKeyRef
+    # configFrom:
+    #   secretKeyRef:
+    #     key: config
+    #     name: my-secret
+```
+
+To use variables in the `config` field:
+
+```yaml
+  spec:
+    envName: myenv
+    variables:
+      - name: db_url
+        value: "mysql://root"
+      # variables from secretKeyRef
+      # - name: db_url
+      #   valueFrom:
+      #     secretKeyRef:
+      #       key: db_url
+      #       name: my-secret
+      # variables from configMapKeyRef
+      # - name: db_url
+      #   valueFrom:
+      #     configMapKeyRef:
+      #       key: db_url
+      #       name: my-configmap
+    config: |
+      variable "db_url" {
+        type = string
+      }
+      env myenv {
+        url = var.db_url
+      }
+```
+
+> Note: Allowing custom configuration enables executing arbitrary commands using the `external` data source as well as arbitrary SQL using the `sql` data source. Use this feature with caution.
+
 - `extraEnvs`: Used to set environment variables for the operator
 
 ```yaml

api/v1alpha1/atlasmigration_types.go

@@ -61,7 +61,8 @@ type (
 	}
 	// AtlasMigrationSpec defines the desired state of AtlasMigration
 	AtlasMigrationSpec struct {
-		TargetSpec `json:",inline"`
+		TargetSpec        `json:",inline"`
+		ProjectConfigSpec `json:",inline"`
 		// EnvName sets the environment name used for reporting runs to Atlas Cloud.
 		EnvName string `json:"envName,omitempty"`
 		// Cloud defines the Atlas Cloud configuration.

api/v1alpha1/atlasschema_types.go

@@ -71,7 +71,8 @@ type (
 	}
 	// AtlasSchemaSpec defines the desired state of AtlasSchema
 	AtlasSchemaSpec struct {
-		TargetSpec `json:",inline"`
+		TargetSpec        `json:",inline"`
+		ProjectConfigSpec `json:",inline"`
 		// Desired Schema of the target.
 		Schema Schema `json:"schema,omitempty"`
 		// Cloud defines the Atlas Cloud configuration.

api/v1alpha1/project_config.go

@@ -0,0 +1,112 @@
+// Copyright 2023 The Atlas Operator Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	"ariga.io/atlas-go-sdk/atlasexec"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclwrite"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type (
+	// ProjectConfigSpec defines the project configuration.
+	ProjectConfigSpec struct {
+		// Config defines the project configuration.
+		// Should be a valid YAML string.
+		Config string `json:"config,omitempty"`
+		// ConfigFrom defines the reference to the secret key that contains the project configuration.
+		ConfigFrom Secret `json:"configFrom,omitempty"`
+		// EnvName defines the environment name that defined in the project configuration.
+		// If not defined, the default environment "k8s" will be used.
+		EnvName string `json:"envName,omitempty"`
+		// Vars defines the input variables for the project configuration.
+		Vars []Variable `json:"vars,omitempty"`
+	}
+	// Variables defines the reference of secret/configmap to the input variables for the project configuration.
+	Variable struct {
+		Key       string    `json:"key,omitempty"`
+		Value     string    `json:"value,omitempty"`
+		ValueFrom ValueFrom `json:"valueFrom,omitempty"`
+	}
+	// ValueFrom defines the reference to the secret key that contains the value.
+	ValueFrom struct {
+		// SecretKeyRef defines the secret key reference to use for the value.
+		SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef,omitempty"`
+		// ConfigMapKeyRef defines the configmap key reference to use for the value.
+		ConfigMapKeyRef *corev1.ConfigMapKeySelector `json:"configMapKeyRef,omitempty"`
+	}
+)
+
+// GetConfig returns the project configuration.
+// The configuration is resolved from the secret reference.
+func (s ProjectConfigSpec) GetConfig(ctx context.Context, r client.Reader, ns string) (*hclwrite.File, error) {
+	rawConfig := s.Config
+	if s.ConfigFrom.SecretKeyRef != nil {
+		cfgFromSecret, err := getSecretValue(ctx, r, ns, s.ConfigFrom.SecretKeyRef)
+		if err != nil {
+			return nil, err
+		}
+		rawConfig = cfgFromSecret
+	}
+	if rawConfig == "" {
+		return nil, nil
+	}
+	config, diags := hclwrite.ParseConfig([]byte(rawConfig), "", hcl.InitialPos)
+	if diags.HasErrors() {
+		return nil, fmt.Errorf("failed to parse project configuration: %v", diags)
+	}
+	return config, nil
+}
+
+// GetVars returns the input variables for the project configuration.
+// The variables are resolved from the secret or configmap reference.
+func (s ProjectConfigSpec) GetVars(ctx context.Context, r client.Reader, ns string) (atlasexec.Vars2, error) {
+	vars := atlasexec.Vars2{}
+	for _, variable := range s.Vars {
+		var (
+			value string
+			err   error
+		)
+		value = variable.Value
+		if variable.ValueFrom.SecretKeyRef != nil {
+			if value, err = getSecretValue(ctx, r, ns, variable.ValueFrom.SecretKeyRef); err != nil {
+				return nil, err
+			}
+		}
+		if variable.ValueFrom.ConfigMapKeyRef != nil {
+			if value, err = getConfigMapValue(ctx, r, ns, variable.ValueFrom.ConfigMapKeyRef); err != nil {
+				return nil, err
+			}
+		}
+		// Resolve variables with the same key by grouping them into a slice.
+		// It's necessary when generating Atlas command for list(string) input type.
+		if existingValue, exists := vars[variable.Key]; exists {
+			if _, ok := existingValue.([]string); ok {
+				vars[variable.Key] = append(existingValue.([]string), value)
+			} else if _, ok := existingValue.(string); ok {
+				vars[variable.Key] = []string{existingValue.(string), value}
+			} else {
+				return nil, fmt.Errorf("invalid variable type for %q", variable.Key)
+			}
+		}
+		vars[variable.Key] = value
+	}
+	return vars, nil
+}

api/v1alpha1/target.go

@@ -66,7 +66,7 @@ type (
 // DatabaseURL returns the database url.
 func (s TargetSpec) DatabaseURL(ctx context.Context, r client.Reader, ns string) (*url.URL, error) {
 	if s.URLFrom.SecretKeyRef != nil {
-		val, err := getSecrectValue(ctx, r, ns, s.URLFrom.SecretKeyRef)
+		val, err := getSecretValue(ctx, r, ns, s.URLFrom.SecretKeyRef)
 		if err != nil {
 			return nil, err
 		}
@@ -76,21 +76,21 @@ func (s TargetSpec) DatabaseURL(ctx context.Context, r client.Reader, ns string)
 		return url.Parse(s.URL)
 	}
 	if s.Credentials.UserFrom.SecretKeyRef != nil {
-		val, err := getSecrectValue(ctx, r, ns, s.Credentials.UserFrom.SecretKeyRef)
+		val, err := getSecretValue(ctx, r, ns, s.Credentials.UserFrom.SecretKeyRef)
 		if err != nil {
 			return nil, err
 		}
 		s.Credentials.User = val
 	}
 	if s.Credentials.PasswordFrom.SecretKeyRef != nil {
-		val, err := getSecrectValue(ctx, r, ns, s.Credentials.PasswordFrom.SecretKeyRef)
+		val, err := getSecretValue(ctx, r, ns, s.Credentials.PasswordFrom.SecretKeyRef)
 		if err != nil {
 			return nil, err
 		}
 		s.Credentials.Password = val
 	}
 	if s.Credentials.HostFrom.SecretKeyRef != nil {
-		val, err := getSecrectValue(ctx, r, ns, s.Credentials.HostFrom.SecretKeyRef)
+		val, err := getSecretValue(ctx, r, ns, s.Credentials.HostFrom.SecretKeyRef)
 		if err != nil {
 			return nil, err
 		}
@@ -99,7 +99,7 @@ func (s TargetSpec) DatabaseURL(ctx context.Context, r client.Reader, ns string)
 	if s.Credentials.Host != "" {
 		return s.Credentials.URL(), nil
 	}
-	return nil, fmt.Errorf("no target database defined")
+	return nil, nil
 }
 
 // URL returns the URL for the database.
@@ -133,7 +133,7 @@ func (c *Credentials) URL() *url.URL {
 	return u
 }
 
-func getSecrectValue(
+func getSecretValue(
 	ctx context.Context,
 	r client.Reader,
 	ns string,
@@ -147,6 +147,20 @@ func getSecrectValue(
 	return string(val.Data[ref.Key]), nil
 }
 
+func getConfigMapValue(
+	ctx context.Context,
+	r client.Reader,
+	ns string,
+	ref *corev1.ConfigMapKeySelector,
+) (string, error) {
+	val := &corev1.ConfigMap{}
+	err := r.Get(ctx, types.NamespacedName{Name: ref.Name, Namespace: ns}, val)
+	if err != nil {
+		return "", err
+	}
+	return string(val.Data[ref.Key]), nil
+}
+
 // Driver defines the database driver.
 type Driver string
 

api/v1alpha1/types_test.go

@@ -53,9 +53,6 @@ func TestTargetSpec_DatabaseURL(t *testing.T) {
 			require.Equal(t, a, u.String())
 		}
 	)
-	// error
-	_, err := target.DatabaseURL(ctx, nil, "default")
-	require.ErrorContains(t, err, "no target database defined")
 
 	// Should return the URL from the credentials
 	target.Credentials = v1alpha1.Credentials{