[FIX] Terraform provider version inconsistency within stages

src/_nebari/constants.py

@@ -30,3 +30,52 @@
 AWS_DEFAULT_REGION = "us-east-1"
 AZURE_DEFAULT_REGION = "Central US"
 GCP_DEFAULT_REGION = "us-central1"
+
+# TERRAFORM REQUIRED PROVIDERS
+REQUIRED_PROVIDERS = {
+    "aws": {
+        "_name": "aws",
+        "source": "hashicorp/aws",
+        "version": "5.12.0",
+    },
+    "azurerm": {
+        "_name": "azurerm",
+        "source": "hashicorp/azurerm",
+        "version": "=3.97.1",
+    },
+    "docker": {
+        "_name": "docker",
+        "source": "kreuzwerker/docker",
+        "version": "2.16.0",
+    },
+    "google": {
+        "_name": "google",
+        "source": "hashicorp/google",
+        "version": "4.83.0",
+    },
+    "helm": {
+        "_name": "helm",
+        "source": "hashicorp/helm",
+        "version": "2.1.2",
+    },
+    "keycloak": {
+        "_name": "keycloak",
+        "source": "mrparkers/keycloak",
+        "version": "3.7.0",
+    },
+    "kind": {
+        "_name": "kind",
+        "source": "tehcyx/kind",
+        "version": "0.4.0",
+    },
+    "kubectl": {
+        "_name": "kubectl",
+        "source": "gavinbunney/kubectl",
+        "version": ">= 1.7.0",
+    },
+    "kubernetes": {
+        "_name": "kubernetes",
+        "source": "hashicorp/kubernetes",
+        "version": ">= 2.35.1",
+    },
+}

src/_nebari/stages/base.py

@@ -12,7 +12,10 @@
 from kubernetes.client.rest import ApiException
 
 from _nebari.provider import helm, kubernetes, kustomize, opentofu
-from _nebari.stages.tf_objects import NebariTerraformState
+from _nebari.stages.tf_objects import (
+    NebariOpentofuRequiredProvider,
+    NebariOpentofuRequiredVersion,
+)
 from nebari.hookspecs import NebariStage
 
 KUSTOMIZATION_TEMPLATE = "kustomization.yaml.tmpl"
@@ -65,7 +68,6 @@ def set_outputs(
     def check(
         self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
     ):
-
         if self.failed_to_create:
             print(
                 f"ERROR: After stage={self.name} "
@@ -143,7 +145,6 @@ def render(self) -> Dict[pathlib.Path, str]:
     def deploy(
         self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
     ):
-
         print(f"Deploying kubernetes resources for {self.name}")
         # get the kubernetes client
         kubernetes_client = self._get_k8s_client(stage_outputs)
@@ -208,7 +209,6 @@ def destroy(
         # destroy each manifest in the reverse order
 
         for manifest in sorted(manifests, reverse=True):
-
             print(f"Destroyed manifest: {manifest}")
             try:
                 kubernetes.delete_from_yaml(kubernetes_client, manifest)
@@ -220,7 +220,6 @@ def destroy(
         # destroy each crd in the reverse order
 
         for crd in sorted(crds, reverse=True):
-
             print(f"Destroyed CRD: {crd}")
             try:
                 kubernetes.delete_from_yaml(kubernetes_client, crd)
@@ -244,7 +243,20 @@ def state_imports(self) -> List[Tuple[str, str]]:
         return []
 
     def tf_objects(self) -> List[Dict]:
-        return [NebariTerraformState(self.name, self.config)]
+        return [
+            NebariOpentofuRequiredVersion(self.config),
+        ]
+
+    def _tf_objects_required_providers(self) -> List[Dict]:
+        contents = [NebariOpentofuRequiredVersion(self.config)]
+        for obj in self.tf_objects():
+            match obj:
+                case {"terraform": {"required_providers": dict(providers)}}:
+                    contents.extend(
+                        NebariOpentofuRequiredProvider(provider, self.config)
+                        for provider in providers.keys()
+                    )
+        return contents
 
     def render(self) -> Dict[pathlib.Path, str]:
         contents = {
@@ -253,8 +265,24 @@ def render(self) -> Dict[pathlib.Path, str]:
             )
         }
         for root, dirs, filenames in os.walk(self.template_directory):
+            root_path = pathlib.Path(root)
+            if any(filename.endswith(".tf") for filename in filenames) and (
+                pathlib.PosixPath("modules")
+                in root_path.relative_to(self.template_directory).parents
+            ):
+                contents[
+                    pathlib.Path(
+                        self.stage_prefix,
+                        pathlib.Path.relative_to(
+                            root_path,
+                            self.template_directory,
+                        ),
+                    )
+                    / "_nebari.tf.json"
+                ] = opentofu.tf_render_objects(self._tf_objects_required_providers())
+
             for filename in filenames:
-                root_filename = pathlib.Path(root) / filename
+                root_filename = root_path / filename
                 with root_filename.open("rb") as f:
                     contents[
                         pathlib.Path(

src/_nebari/stages/infrastructure/__init__.py

@@ -16,7 +16,10 @@
 from _nebari.provider.cloud import amazon_web_services, azure_cloud, google_cloud
 from _nebari.stages.base import NebariTerraformStage
 from _nebari.stages.kubernetes_services import SharedFsEnum
-from _nebari.stages.tf_objects import NebariTerraformState
+from _nebari.stages.tf_objects import (
+    NebariOpentofuRequiredProvider,
+    NebariTerraformState,
+)
 from _nebari.utils import (
     AZURE_NODE_RESOURCE_GROUP_SUFFIX,
     construct_azure_resource_group_name,
@@ -732,8 +735,12 @@ def state_imports(self) -> List[Tuple[str, str]]:
             ]
 
     def tf_objects(self) -> List[Dict]:
+        resources = super().tf_objects()
+
         if self.config.provider == schema.ProviderEnum.gcp:
             return [
+                *resources,
+                NebariOpentofuRequiredProvider("google", self.config),
                 opentofu.Provider(
                     "google",
                     project=self.config.google_cloud_platform.project,
@@ -743,15 +750,27 @@ def tf_objects(self) -> List[Dict]:
             ]
         elif self.config.provider == schema.ProviderEnum.azure:
             return [
+                *resources,
+                NebariOpentofuRequiredProvider("azurerm", self.config),
                 NebariTerraformState(self.name, self.config),
             ]
         elif self.config.provider == schema.ProviderEnum.aws:
             return [
+                *resources,
+                NebariOpentofuRequiredProvider("aws", self.config),
                 opentofu.Provider("aws", region=self.config.amazon_web_services.region),
                 NebariTerraformState(self.name, self.config),
             ]
+        elif self.config.provider == schema.ProviderEnum.local:
+            return [
+                *resources,
+                NebariOpentofuRequiredProvider("kind", self.config),
+                NebariOpentofuRequiredProvider("docker", self.config),
+                NebariOpentofuRequiredProvider("kubectl", self.config),
+                NebariOpentofuRequiredProvider("kubernetes", self.config),
+            ]
         else:
-            return []
+            return resources
 
     def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
         if self.config.provider == schema.ProviderEnum.local:

src/_nebari/stages/infrastructure/template/local/main.tf

@@ -1,20 +1,3 @@
-terraform {
-  required_providers {
-    kind = {
-      source  = "registry.terraform.io/tehcyx/kind"
-      version = "0.4.0"
-    }
-    docker = {
-      source  = "kreuzwerker/docker"
-      version = "2.16.0"
-    }
-    kubectl = {
-      source  = "gavinbunney/kubectl"
-      version = ">= 1.7.0"
-    }
-  }
-}
-
 provider "kind" {
 
 }

src/_nebari/stages/kubernetes_ingress/__init__.py

@@ -11,6 +11,7 @@
 from _nebari.stages.tf_objects import (
     NebariHelmProvider,
     NebariKubernetesProvider,
+    NebariOpentofuRequiredProvider,
     NebariTerraformState,
 )
 from nebari import schema
@@ -168,9 +169,12 @@ class KubernetesIngressStage(NebariTerraformStage):
 
     def tf_objects(self) -> List[Dict]:
         return [
-            NebariTerraformState(self.name, self.config),
+            *super().tf_objects(),
+            NebariOpentofuRequiredProvider("kubernetes", self.config),
             NebariKubernetesProvider(self.config),
+            NebariOpentofuRequiredProvider("helm", self.config),
             NebariHelmProvider(self.config),
+            NebariTerraformState(self.name, self.config),
         ]
 
     def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):

src/_nebari/stages/kubernetes_initialize/__init__.py

@@ -7,6 +7,7 @@
 from _nebari.stages.tf_objects import (
     NebariHelmProvider,
     NebariKubernetesProvider,
+    NebariOpentofuRequiredProvider,
     NebariTerraformState,
 )
 from nebari import schema
@@ -63,10 +64,14 @@ class KubernetesInitializeStage(NebariTerraformStage):
     output_schema = OutputSchema
 
     def tf_objects(self) -> List[Dict]:
+        resources = super().tf_objects()
         return [
-            NebariTerraformState(self.name, self.config),
+            *resources,
+            NebariOpentofuRequiredProvider("kubernetes", self.config),
             NebariKubernetesProvider(self.config),
+            NebariOpentofuRequiredProvider("helm", self.config),
             NebariHelmProvider(self.config),
+            NebariTerraformState(self.name, self.config),
         ]
 
     def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):

src/_nebari/stages/kubernetes_keycloak/__init__.py

@@ -14,6 +14,7 @@
 from _nebari.stages.tf_objects import (
     NebariHelmProvider,
     NebariKubernetesProvider,
+    NebariOpentofuRequiredProvider,
     NebariTerraformState,
 )
 from _nebari.utils import modified_environ
@@ -218,9 +219,13 @@ class KubernetesKeycloakStage(NebariTerraformStage):
 
     def tf_objects(self) -> List[Dict]:
         return [
-            NebariTerraformState(self.name, self.config),
+            *super().tf_objects(),
+            NebariOpentofuRequiredProvider("kubernetes", self.config),
             NebariKubernetesProvider(self.config),
+            NebariOpentofuRequiredProvider("helm", self.config),
             NebariHelmProvider(self.config),
+            NebariOpentofuRequiredProvider("keycloak", self.config),
+            NebariTerraformState(self.name, self.config),
         ]
 
     def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):

src/_nebari/stages/kubernetes_keycloak_configuration/__init__.py

@@ -4,7 +4,10 @@
 
 from _nebari.stages.base import NebariTerraformStage
 from _nebari.stages.kubernetes_keycloak import Authentication
-from _nebari.stages.tf_objects import NebariTerraformState
+from _nebari.stages.tf_objects import (
+    NebariOpentofuRequiredProvider,
+    NebariTerraformState,
+)
 from nebari import schema
 from nebari.hookspecs import NebariStage, hookimpl
 
@@ -26,7 +29,11 @@ class KubernetesKeycloakConfigurationStage(NebariTerraformStage):
 
     def tf_objects(self) -> List[Dict]:
         return [
+            *super().tf_objects(),
             NebariTerraformState(self.name, self.config),
+            NebariOpentofuRequiredProvider("helm", self.config),
+            NebariOpentofuRequiredProvider("kubernetes", self.config),
+            NebariOpentofuRequiredProvider("keycloak", self.config),
         ]
 
     def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):