diff --git a/Gopkg.lock b/Gopkg.lock index 895bfa1c05..60acdd8341 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -255,6 +255,7 @@ packages = [ "addrs", "configs/configschema", + "dag", "helper/didyoumean", "lang", "lang/blocktoattr", @@ -624,6 +625,7 @@ "github.com/hashicorp/hcl2/hcl/hclsyntax", "github.com/hashicorp/hcl2/hclparse", "github.com/hashicorp/hcl2/hclwrite", + "github.com/hashicorp/terraform/dag", "github.com/hashicorp/terraform/lang", "github.com/mattn/go-zglob", "github.com/mitchellh/mapstructure", diff --git a/README.md b/README.md index 7d902b1edf..d2f8051d2a 100644 --- a/README.md +++ b/README.md @@ -1365,8 +1365,12 @@ at all. This section contains detailed documentation for the following aspects of Terragrunt: +1. [Values](#values) + 1. [Local Values](#local-values) + 1. [Global Values](#global-values) + 1. [Include Values](#include-values) + 1. [Value Caveats](#value-caveats) 1. [Inputs](#inputs) -1. [Locals](#locals) 1. [AWS credentials](#aws-credentials) 1. [AWS IAM policies](#aws-iam-policies) 1. [Built-in Functions](#built-in-functions) @@ -1383,101 +1387,398 @@ This section contains detailed documentation for the following aspects of Terrag 1. [License](#license) -### Inputs +### Values -You can set values for your module's input parameters by specifying an `inputs` block in `terragrunt.hcl`: +`terragrunt` provides the following "values" (or "variables") to help you keep your configurations DRY when using expressions +in multiple config locations: `locals`, `globals`, and `include`. Each type of value is detailed below, and every value +can be used in any expression with a few [caveats](#value-caveats). + + +#### Local Values + +Local values let you bind a name to an expression, so you can reuse that expression within a single `terragrunt` config. +For example, suppose that you need to use the AWS region in multiple inputs. You can bind the name `aws_region` +using locals: + +``` +locals { + aws_region = "us-east-1" +} -```hcl inputs = { - instance_type = "t2.micro" - instance_count = 10 + aws_region = local.aws_region + s3_endpoint = "com.amazonaws.${local.aws_region}.s3" +} +``` - tags = { - Name = "example-app" +You can use any valid terragrunt expression in the `locals` configuration. The `locals` block also supports referencing +other values: + +``` +locals { + x = 2 + y = 40 + answer = local.x + local.y +} +``` + +#### Global Values + +Since `locals` are only available inside the configuration they're defined in, `terragrunt` also supports `globals`. +`globals` are available across both configurations when `include` is used. When the same global is defined in both configurations, +the value of the expression in the "child" configuration will be used as the value of the global in both configurations. + +Global values are mainly useful for: +* [Passing a common value to multiple child configs](#passing-a-common-value) +* [Affecting the behavior of a parent config from the child config](#affecting-the-behavior-of-a-parent) +* [Set a default value in the parent config that can be optionally overridden from the child config.](#default-values) + +##### Passing a common value + +Parent config: +``` +globals { + source_prefix = "/some/path" +} +``` + +Child config: +``` +terraform { + source = "${global.source_prefix}/some/module" +} +``` + +In this example, the value of the child's `terraform.source` would evaluate to `/some/path/some/module` + +##### Affecting the behavior of a parent + +Parent config: +``` +globals { + region = null +} + +remote_state { + backend = "s3" + config = { + bucket = "bucket-name + region = global.region + key = "key" } } ``` -Whenever you run a Terragrunt command, Terragrunt will set any inputs you pass in as environment variables. For example, -with the `terragrunt.hcl` file above, running `terragrunt apply` is roughly equivalent to: +Child 1 config: +``` +globals { + region = "us-east-1" +} +``` +Child 2 config: +``` +globals { + region = "us-west-2" +} ``` -$ terragrunt apply -# Roughly equivalent to: +In this example, the parent's `remote_state` block would change based on which child configuration it was included from. -TF_VAR_instance_type="t2.micro" \ -TF_VAR_instance_count=10 \ -TF_VAR_tags='{"Name":"example-app"}' \ -terraform apply +##### Default values + +Parent config: ``` +globals { + bucket = "my-bucket" +} -Note that Terragrunt will respect any `TF_VAR_xxx` variables you've manually set in your environment, ensuring that -anything in `inputs` will NOT be override anything you've already set in your environment. +remote_state { + backend = "s3" + config = { + bucket = global.bucket + region = "us-east-1" + key = "key" + } +} +``` +Child 1 config: +``` +globals { + # This block is optional, but included here to show that it's empty. +} +``` -### Locals +Child 2 config: +``` +globals { + bucket = "my-other-bucket" +} +``` -You can use locals to bind a name to an expression, so you can reuse that expression without having to repeat it multiple times (keeping your Terragrunt configuration DRY). -config. For example, suppose that you need to use the AWS region in multiple inputs. You can bind the name `aws_region` -using locals: +In this example, the bucket name in the `remote_state` has a default value, and child configs can optionally override it +to use a different bucket. + +#### Include Values + +Once `terragrunt` has evaluated the `include` block (if any) in the given config, the following values will be available +for use in any expression: +* [`include.parent`](#includeparent) +* [`include.child`](#includechild) +* [`include.relative`](#includerelative) +* [`include.relative_reverse`](#includerelative_reverse) + +##### `include.parent` + +This value is a direct replacement for the `get_parent_terragrunt_dir()` function. + +`include.parent` returns the absolute directory where the Terragrunt parent configuration file (by default +`terragrunt.hcl`) lives. This is useful when you need to use relative paths with [remote Terraform +configurations](#remote-terraform-configurations) and you want those paths relative to your parent Terragrunt +configuration file and not relative to the temporary directory where Terragrunt downloads the code. + +This value is very similar to [`include.child](#includechild) except it returns the root instead of the +leaf of your terragrunt configuration folder. ``` -locals { - aws_region = "us-east-1" +/terraform-code +├── terragrunt.hcl +├── common.tfvars +├── app1 +│ └── terragrunt.hcl +├── tests +│ ├── app2 +│ | └── terragrunt.hcl +│ └── app3 +│ └── terragrunt.hcl +``` + +```hcl +terraform { + extra_arguments "common_vars" { + commands = [ + "apply", + "plan", + "import", + "push", + "refresh" + ] + + arguments = [ + "-var-file=${include.parent}/common.tfvars" + ] + } } +``` -inputs = { - aws_region = local.aws_region - s3_endpoint = "com.amazonaws.${local.aws_region}.s3" +The common.tfvars located in the terraform root folder will be included by all applications, whatever their relative location to the root. + +##### `include.child` + +This value is a direct replacement for the `get_terragrunt_dir()` function. + +`include.child` returns the directory where the Terragrunt configuration file (by default `terragrunt.hcl`) lives. +This is useful when you need to use relative paths with [remote Terraform +configurations](#remote-terraform-configurations) and you want those paths relative to your Terragrunt configuration +file and not relative to the temporary directory where Terragrunt downloads the code. + +For example, imagine you have the following file structure: + +``` +/terraform-code +├── common.tfvars +├── frontend-app +│ └── terragrunt.hcl +``` + +Inside of `/terraform-code/frontend-app/terragrunt.hcl` you might try to write code that looks like this: + +```hcl +terraform { + source = "git::git@github.com:foo/modules.git//frontend-app?ref=v0.0.3" + + extra_arguments "custom_vars" { + commands = [ + "apply", + "plan", + "import", + "push", + "refresh" + ] + + arguments = [ + "-var-file=../common.tfvars" # Note: This relative path will NOT work correctly! + ] + } } ``` -You can use any valid terragrunt expression in the `locals` configuration. The `locals` block also supports referencing other `locals`: +Note how the `source` parameter is set, so Terragrunt will download the `frontend-app` code from the `modules` repo +into a temporary folder and run `terraform` in that temporary folder. Note also that there is an `extra_arguments` +block that is trying to allow the `frontend-app` to read some shared variables from a `common.tfvars` file. +Unfortunately, the relative path (`../common.tfvars`) won't work, as it will be relative to the temporary folder! +Moreover, you can't use an absolute path, or the code won't work on any of your teammates' computers. + +To make the relative path work, you need to use `include.child` to combine the path with the folder where +the `terragrunt.hcl` file lives: +```hcl +terraform { + source = "git::git@github.com:foo/modules.git//frontend-app?ref=v0.0.3" + + extra_arguments "custom_vars" { + commands = [ + "apply", + "plan", + "import", + "push", + "refresh" + ] + + # With the include.child value, you can use relative paths! + arguments = [ + "-var-file=${include.child}/../common.tfvars" + ] + } +} ``` -locals { - x = 2 - y = 40 - answer = local.x + local.y + +For the example above, this path will resolve to `/terraform-code/frontend-app/../common.tfvars`, which is exactly +what you want. + + +##### `include.relative` + +This value is a direct replacement for the `path_relative_to_include()` function. + +`include.relative` returns the relative path between the included `terragrunt.hcl` file (parent) and the `terragrunt.hcl` +that included it (child). For example, consider the following folder structure: + +``` +├── terragrunt.hcl +└── prod +    └── mysql +    └── terragrunt.hcl +└── stage +    └── mysql +    └── terragrunt.hcl +``` + +Imagine `prod/mysql/terragrunt.hcl` and `stage/mysql/terragrunt.hcl` include all settings from the root +`terragrunt.hcl` file: + +```hcl +include { + path = find_in_parent_folders() } ``` -##### Including globally defined locals +The root `terragrunt.hcl` can use `include.relative` in its `remote_state` configuration to ensure +each child stores its remote state at a different `key`: + +```hcl +remote_state { + backend = "s3" + config = { + bucket = "my-terraform-bucket" + region = "us-east-1" + key = "${include.relative}/terraform.tfstate" + } +} +``` + +The resulting `key` will be `prod/mysql/terraform.tfstate` for the prod `mysql` module and +`stage/mysql/terraform.tfstate` for the stage `mysql` module. -Currently you can only reference `locals` defined in the same config file. `terragrunt` does not automatically include -`locals` defined in the parent config of an `include` block into the current context. If you wish to reuse variables -globally, consider using `yaml` or `json` files that are included and merged using the `terraform` built in functions -available to `terragrunt`. +##### `include.relative_reverse` + +This value is a direct replacement for the `path_relative_from_include()` function. + +`include.relative_reverse` returns the reverse of `include.relative`, meaning the relative path between the child config +and the parent config. For example, where `include.relative` would return `some/subdirectory`, `include.relative_reverse` +would return `../..`. + +#### Value Caveats + +Any value can be used in any other value's expression, with the following caveats: + +1. **No dependency loops:** If including a value in another value's expression would cause this value or another value +to become dependent on itself, an error will be thrown. Example of a dependency loop: + ``` + locals { + one = global.four + two = local.one + } + globals { + three = local.two + four = global.three + } + + # Error! global.four depends on global.three, which depends on local.two, which depends on local.one, which depends on global.four! + # Therefore, global.four cannot be evaluated unless global.four is evaluated. This is a dependency loop. + ``` +2. **Globals cannot be dependencies of an `include` block:** Since the `include` block in a child config must be evaluated +before globals can be resolved, attempting to use a global in the `include` block will result in an error. Example: + ``` + locals { + include_path = "${global.prefix}/terragrunt.hcl}" + } + include { + path = local.include_path + } + + # Error! local.include_path depends on global.prefix! + # Since global.prefix can't be resolved, an error will be thrown. + ``` + +##### Including values from other projects + +Currently you can only reference `locals` or `globals` defined in the parent or child config. If you wish to reuse variables +between multiple parent configs or projects, consider using `yaml` or `json` files that are included and merged using the +`terraform` built in functions available to `terragrunt`. For example, suppose you had the following directory tree: ``` . -├── terragrunt.hcl -├── mysql -│   └── terragrunt.hcl -└── vpc - └── terragrunt.hcl +├── project1 +│   ├── mysql +│   │   └── terragrunt.hcl +│   ├── terragrunt.hcl +│   └── vpc +│   └── terragrunt.hcl +└── project2 + ├── mysql + │   └── terragrunt.hcl + ├── terragrunt.hcl + └── vpc + └── terragrunt.hcl ``` -Instead of adding the `locals` block to the parent `terragrunt.hcl` file, you can define a file `common_vars.yaml` -that contains the global variables you wish to pull in: +Here, you can define a file `common_vars.yaml` that contains the global variables you wish to pull in: ``` . -├── terragrunt.hcl ├── common_vars.yaml -├── mysql -│   └── terragrunt.hcl -└── vpc - └── terragrunt.hcl +├── project1 +│   ├── mysql +│   │   └── terragrunt.hcl +│   ├── terragrunt.hcl +│   └── vpc +│   └── terragrunt.hcl +└── project2 + ├── mysql + │   └── terragrunt.hcl + ├── terragrunt.hcl + └── vpc + └── terragrunt.hcl ``` -You can then include them into the `locals` block of the child terragrunt config using `yamldecode` and `file`: +You can then include them into the `locals` (or `globals`) block of a terragrunt config using `yamldecode` and `file`: ``` -# child terragrunt.hcl +# project1/terragrunt.hcl locals { common_vars = yamldecode(file("${get_terragrunt_dir()}/${find_in_parent_folders("common_vars.yaml")}")), region = "us-east-1" @@ -1485,9 +1786,40 @@ locals { ``` This configuration will load in the `common_vars.yaml` file and bind it to the attribute `common_vars` so that it is available -in the current context. Note that because `locals` is a block, there currently is a way to merge the map into the top +in the current context. Note that because `locals` is a block, there currently is no way to merge the map into the top level. +### Inputs + +You can set values for your module's input parameters by specifying an `inputs` block in `terragrunt.hcl`: + +```hcl +inputs = { + instance_type = "t2.micro" + instance_count = 10 + + tags = { + Name = "example-app" + } +} +``` + +Whenever you run a Terragrunt command, Terragrunt will set any inputs you pass in as environment variables. For example, +with the `terragrunt.hcl` file above, running `terragrunt apply` is roughly equivalent to: + +``` +$ terragrunt apply + +# Roughly equivalent to: + +TF_VAR_instance_type="t2.micro" \ +TF_VAR_instance_count=10 \ +TF_VAR_tags='{"Name":"example-app"}' \ +terraform apply +``` + +Note that Terragrunt will respect any `TF_VAR_xxx` variables you've manually set in your environment, ensuring that +anything in `inputs` will NOT be override anything you've already set in your environment. ### AWS credentials @@ -1675,44 +2007,6 @@ include { #### path_relative_to_include -`path_relative_to_include()` returns the relative path between the current `terragrunt.hcl` file and the `path` -specified in its `include` block. For example, consider the following folder structure: - -``` -├── terragrunt.hcl -└── prod -    └── mysql -    └── terragrunt.hcl -└── stage -    └── mysql -    └── terragrunt.hcl -``` - -Imagine `prod/mysql/terragrunt.hcl` and `stage/mysql/terragrunt.hcl` include all settings from the root -`terragrunt.hcl` file: - -```hcl -include { - path = find_in_parent_folders() -} -``` - -The root `terragrunt.hcl` can use the `path_relative_to_include()` in its `remote_state` configuration to ensure -each child stores its remote state at a different `key`: - -```hcl -remote_state { - backend = "s3" - config = { - bucket = "my-terraform-bucket" - region = "us-east-1" - key = "${path_relative_to_include()}/terraform.tfstate" - } -} -``` - -The resulting `key` will be `prod/mysql/terraform.tfstate` for the prod `mysql` module and -`stage/mysql/terraform.tfstate` for the stage `mysql` module. #### path_relative_from_include @@ -1803,118 +2097,9 @@ as the environment variable `TF_VAR_foo` and to read that value in using this `g #### get_terragrunt_dir -`get_terragrunt_dir()` returns the directory where the Terragrunt configuration file (by default `terragrunt.hcl`) lives. -This is useful when you need to use relative paths with [remote Terraform -configurations](#remote-terraform-configurations) and you want those paths relative to your Terragrunt configuration -file and not relative to the temporary directory where Terragrunt downloads the code. - -For example, imagine you have the following file structure: - -``` -/terraform-code -├── common.tfvars -├── frontend-app -│ └── terragrunt.hcl -``` - -Inside of `/terraform-code/frontend-app/terragrunt.hcl` you might try to write code that looks like this: - -```hcl -terraform { - source = "git::git@github.com:foo/modules.git//frontend-app?ref=v0.0.3" - - extra_arguments "custom_vars" { - commands = [ - "apply", - "plan", - "import", - "push", - "refresh" - ] - - arguments = [ - "-var-file=../common.tfvars" # Note: This relative path will NOT work correctly! - ] - } -} -``` - -Note how the `source` parameter is set, so Terragrunt will download the `frontend-app` code from the `modules` repo -into a temporary folder and run `terraform` in that temporary folder. Note also that there is an `extra_arguments` -block that is trying to allow the `frontend-app` to read some shared variables from a `common.tfvars` file. -Unfortunately, the relative path (`../common.tfvars`) won't work, as it will be relative to the temporary folder! -Moreover, you can't use an absolute path, or the code won't work on any of your teammates' computers. - -To make the relative path work, you need to use `get_terragrunt_dir()` to combine the path with the folder where -the `terragrunt.hcl` file lives: - -```hcl -terraform { - source = "git::git@github.com:foo/modules.git//frontend-app?ref=v0.0.3" - - extra_arguments "custom_vars" { - commands = [ - "apply", - "plan", - "import", - "push", - "refresh" - ] - - # With the get_terragrunt_dir() function, you can use relative paths! - arguments = [ - "-var-file=${get_terragrunt_dir()}/../common.tfvars" - ] - } -} -``` - -For the example above, this path will resolve to `/terraform-code/frontend-app/../common.tfvars`, which is exactly -what you want. - #### get_parent_terragrunt_dir -`get_parent_terragrunt_dir()` returns the absolute directory where the Terragrunt parent configuration file (by default -`terragrunt.hcl`) lives. This is useful when you need to use relative paths with [remote Terraform -configurations](#remote-terraform-configurations) and you want those paths relative to your parent Terragrunt -configuration file and not relative to the temporary directory where Terragrunt downloads the code. - -This function is very similar to [get_terragrunt_dir()](#get_terragrunt_dir) except it returns the root instead of the -leaf of your terragrunt configuration folder. - -``` -/terraform-code -├── terragrunt.hcl -├── common.tfvars -├── app1 -│ └── terragrunt.hcl -├── tests -│ ├── app2 -│ | └── terragrunt.hcl -│ └── app3 -│ └── terragrunt.hcl -``` - -```hcl -terraform { - extra_arguments "common_vars" { - commands = [ - "apply", - "plan", - "import", - "push", - "refresh" - ] - - arguments = [ - "-var-file=${get_parent_terragrunt_dir()}/common.tfvars" - ] - } -} -``` - -The common.tfvars located in the terraform root folder will be included by all applications, whatever their relative location to the root. #### get_terraform_commands_that_need_vars diff --git a/config/config_graph.go b/config/config_graph.go new file mode 100644 index 0000000000..db938a7423 --- /dev/null +++ b/config/config_graph.go @@ -0,0 +1,566 @@ +package config + +import ( + "fmt" + "github.com/gruntwork-io/terragrunt/errors" + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/util" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hclparse" + "github.com/hashicorp/terraform/dag" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/gocty" + "path/filepath" +) + +const local = "local" +const global = "global" +const include = "include" + +type rootVertex struct { +} + +type variableVertex struct { + Evaluator *configEvaluator + Type string + Name string + Expr hcl.Expression + Evaluated bool +} + +// basicEdge is a basic implementation of Edge that has the source and +// target vertex. +type basicEdge struct { + S, T dag.Vertex +} + +func (e *basicEdge) Hashcode() interface{} { + return fmt.Sprintf("%p-%p", e.S, e.T) +} + +func (e *basicEdge) Source() dag.Vertex { + return e.S +} + +func (e *basicEdge) Target() dag.Vertex { + return e.T +} + +type evaluatorGlobals struct { + options *options.TerragruntOptions + parser *hclparse.Parser + graph dag.AcyclicGraph + root rootVertex + vertices map[string]variableVertex + values map[string]cty.Value +} + +type configEvaluator struct { + globals evaluatorGlobals + + configPath string + configFile *hcl.File + + localVertices map[string]variableVertex + localValues map[string]cty.Value + includeVertex *variableVertex + includePath *string + includeValue *map[string]cty.Value +} + +type EvaluationResult struct { + ConfigFile *hcl.File + Variables map[string]cty.Value +} + +func newConfigEvaluator(configPath string, globals evaluatorGlobals, includeValue *map[string]cty.Value) *configEvaluator { + eval := configEvaluator{} + eval.globals = globals + eval.configPath = configPath + + eval.localVertices = map[string]variableVertex{} + eval.localValues = map[string]cty.Value{} + + eval.includeVertex = nil + eval.includeValue = includeValue + + return &eval +} + +// Evaluation Steps: +// 1. Parse child HCL, extract locals, globals, and include +// 2. Add vertices for child locals, globals, and include +// 3. Add edges for child variables based on interpolations used +// a. When encountering globals that aren't defined in this config, create a vertex for them with an empty expression +// 4. Verify DAG and reduce graph +// a. Verify no globals are used in path to include (if exists) +// 5. Evaluate everything except globals +// 6. If include exists, find parent HCL, parse, and extract locals and globals +// 7. Add vertices for parent locals +// 8. Add vertices for parent globals that don't already exist, or add expressions to empty globals +// 9. Verify and reduce graph +// a. Verify that there are no globals that are empty. +// 10. Evaluate everything, skipping things that were evaluated in (5) +func ParseConfigVariables(filename string, terragruntOptions *options.TerragruntOptions) (*EvaluationResult, *EvaluationResult, error) { + globals := evaluatorGlobals{ + options: terragruntOptions, + parser: hclparse.NewParser(), + graph: dag.AcyclicGraph{}, + root: rootVertex{}, + vertices: map[string]variableVertex{}, + values: map[string]cty.Value{}, + } + + // Add root of graph + globals.graph.Add(globals.root) + + child := *newConfigEvaluator(filename, globals, nil) + var childResult *EvaluationResult = nil + + // 1, 2, 3, 4 + err := child.parseConfig() + if err != nil { + return nil, nil, err + } + + // 5 + diags := globals.evaluateVariables(false) + if diags != nil { + return nil, nil, diags + } + + var parent *configEvaluator = nil + var parentResult *EvaluationResult = nil + if child.includePath != nil { + // 6, 7, 8, 9 + parent = newConfigEvaluator(*child.includePath, globals, child.includeValue) + err = (*parent).parseConfig() + if err != nil { + return nil, nil, err + } + } + + // 10 + diags = globals.evaluateVariables(true) + if diags != nil { + return nil, nil, diags + } + + childResult, err = child.toResult() + if err != nil { + return nil, nil, err + } + if parent != nil { + parentResult, err = parent.toResult() + if err != nil { + return nil, nil, err + } + } + + return childResult, parentResult, nil +} + +func (eval *configEvaluator) parseConfig() error { + configString, err := util.ReadFileAsString(eval.configPath) + if err != nil { + return err + } + + configFile, err := parseHcl(eval.globals.parser, configString, eval.configPath) + if err != nil { + return err + } + + eval.configFile = configFile + + localsBlock, globalsBlock, includeBlock, diags := eval.getBlocks(configFile) + if diags != nil && diags.HasErrors() { + return diags + } + + var addedVertices []variableVertex + err = eval.addVertices(local, localsBlock, func(vertex variableVertex) error { + eval.localVertices[vertex.Name] = vertex + addedVertices = append(addedVertices, vertex) + return nil + }) + if err != nil { + return err + } + if includeBlock != nil { + err = eval.addVertices(include, includeBlock, func(vertex variableVertex) error { + // TODO: validate include name and ensure only setting this once + eval.includeVertex = &vertex + addedVertices = append(addedVertices, vertex) + return nil + }) + } + if err != nil { + return err + } + err = eval.addVertices(global, globalsBlock, func(vertex variableVertex) error { + eval.globals.vertices[vertex.Name] = vertex + addedVertices = append(addedVertices, vertex) + return nil + }) + if err != nil { + return err + } + + // TODO validate include + + err = eval.addAllEdges(eval.localVertices) + if err != nil { + return err + } + if eval.includeVertex != nil { + err = eval.addEdges(*eval.includeVertex) + if err != nil { + return err + } + } + err = eval.addAllEdges(eval.globals.vertices) + if err != nil { + return err + } + + // TODO: validate that includes only depend on locals + + err = eval.globals.graph.Validate() + if err != nil { + return err + } + + eval.globals.graph.TransitiveReduction() + + return nil +} + +func (eval *configEvaluator) evaluateVariable(vertex variableVertex, diags hcl.Diagnostics, evaluateGlobals bool) bool { + if vertex.Type == global && !evaluateGlobals { + return false + } + + if vertex.Evaluated { + return true + } + + valuesCty, err := eval.convertValuesToVariables() + if err != nil { + // TODO: diags.Extend(??) + return false + } + + ctx := hcl.EvalContext{ + Variables: valuesCty, + } + + value, currentDiags := vertex.Expr.Value(&ctx) + if currentDiags != nil && currentDiags.HasErrors() { + _ = diags.Extend(currentDiags) + return false + } + + vertex.Evaluated = true + + switch vertex.Type { + case global: + eval.globals.values[vertex.Name] = value + + case local: + eval.localValues[vertex.Name] = value + case include: + includePath, includeValue, err := eval.evaluateInclude(value) + if err != nil { + // TODO: diags.Extend(??) + return false + } + + eval.includePath = &includePath + eval.includeValue = &includeValue + default: + // TODO: diags.Extend(??) + return false + } + + return true +} + +func (eval *configEvaluator) evaluateInclude(value cty.Value) (string, map[string]cty.Value, error) { + // TODO: validate this is a string? + includePath := value.AsString() + configPath := eval.configPath + + if includePath == "" { + return "", nil, IncludedConfigMissingPath(configPath) + } + + childConfigPathAbs, err := filepath.Abs(configPath) + if err != nil { + return "", nil, err + } + + if !filepath.IsAbs(includePath) { + includePath = util.JoinPath(filepath.Dir(childConfigPathAbs), includePath) + } + + relative, err := util.GetPathRelativeTo(filepath.Dir(configPath), filepath.Dir(includePath)) + if err != nil { + return "", nil, err + } + + includeValue := map[string]cty.Value{ + "parent": cty.StringVal(filepath.ToSlash(filepath.Dir(includePath))), + "relative": cty.StringVal(relative), + } + + return includePath, includeValue, nil +} + +func (eval *configEvaluator) getBlocks(file *hcl.File) (hcl.Body, hcl.Body, hcl.Body, hcl.Diagnostics) { + const locals = "locals" + const globals = "globals" + + blocksSchema := &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + {Type: locals}, + {Type: globals}, + {Type: include}, + }, + } + + parsedBlocks, _, diags := file.Body.PartialContent(blocksSchema) + if diags != nil && diags.HasErrors() { + return nil, nil, nil, diags + } + blocksByType := map[string][]*hcl.Block{} + + for _, block := range parsedBlocks.Blocks { + if block.Type == locals || block.Type == globals || block.Type == include { + blocks := blocksByType[block.Type] + if blocks == nil { + blocks = []*hcl.Block{} + } + + blocksByType[block.Type] = append(blocks, block) + } + } + + // TODO: validate blocks + + + if _, exists := blocksByType[include]; exists { + return blocksByType[locals][0].Body, blocksByType[globals][0].Body, blocksByType[include][0].Body, diags + } else { + return blocksByType[locals][0].Body, blocksByType[globals][0].Body, nil, diags + } +} + +func (eval *configEvaluator) addVertices(vertexType string, block hcl.Body, consumer func(vertex variableVertex) error) error { + attrs, diags := block.JustAttributes() + if diags != nil && diags.HasErrors() { + return diags + } + + for name, attr := range attrs { + var vertex *variableVertex = nil + + if vertexType == global { + globalVertex, exists := eval.globals.vertices[name] + if exists && globalVertex.Expr == nil { + // This was referenced by a child but not overridden there + vertex = &globalVertex + globalVertex.Evaluator = eval + globalVertex.Expr = attr.Expr + } + } + + if vertex == nil { + vertex = &variableVertex{ + Evaluator: eval, + Type: vertexType, + Name: name, + Expr: attr.Expr, + Evaluated: false, + } + } + + eval.globals.graph.Add(*vertex) + err := consumer(*vertex) + if err != nil { + return err + } + } + + return nil +} + +func (eval *configEvaluator) addAllEdges(vertices map[string]variableVertex) error { + for _, vertex := range vertices { + err := eval.addEdges(vertex) + if err != nil { + return err + } + } + + return nil +} + +func (eval *configEvaluator) addEdges(target variableVertex) error { + if target.Expr == nil { + return nil + } + + variables := target.Expr.Variables() + + if variables == nil || len(variables) <= 0 { + eval.globals.graph.Connect(&basicEdge{ + S: eval.globals.root, + T: target, + }) + return nil + } + + for _, variable := range variables { + sourceType, sourceName, err := getVariableRootAndName(variable) + if err != nil { + return err + } + + switch sourceType { + case global: + source, exists := eval.globals.vertices[sourceName] + if !exists { + // Could come from parent context, add empty node for now. + source = variableVertex{ + Evaluator: nil, + Type: global, + Name: sourceName, + Expr: nil, + Evaluated: false, + } + } + eval.globals.graph.Connect(&basicEdge{ + S: source, + T: target, + }) + case local: + source, exists := eval.localVertices[sourceName] + if !exists { + // TODO: error + return nil + } + + eval.globals.graph.Connect(&basicEdge{ + S: source, + T: target, + }) + case include: + // TODO validate options + eval.globals.graph.Connect(&basicEdge{ + S: eval.includeVertex, + T: target, + }) + default: + // TODO: error + return nil + } + } + + return nil +} + +func getVariableRootAndName(variable hcl.Traversal) (string, string, error) { + // TODO: validation + sourceType := variable.RootName() + sourceName := variable[1].(hcl.TraverseAttr).Name + return sourceType, sourceName, nil +} + +func (eval *configEvaluator) convertValuesToVariables() (map[string]cty.Value, error) { + values := map[string]map[string]cty.Value{ + local: eval.localValues, + global: eval.globals.values, + } + + if eval.includeValue != nil { + values[include] = *eval.includeValue + } + + variables := map[string]cty.Value{} + for k, v := range values { + variable, err := gocty.ToCtyValue(v, generateTypeFromMap(v)) + if err != nil { + return nil, errors.WithStackTrace(err) + } + + variables[k] = variable + } + + return variables, nil +} + +func (eval *configEvaluator) toResult() (*EvaluationResult, error) { + variables, err := eval.convertValuesToVariables() + if err != nil { + return nil, err + } + + return &EvaluationResult{ + ConfigFile: eval.configFile, + Variables: variables, + }, nil +} + +func (globals *evaluatorGlobals) evaluateVariables(evaluateGlobals bool) hcl.Diagnostics { + diags := hcl.Diagnostics{} + + walkBreadthFirst(globals.graph, globals.root, func(v dag.Vertex) (shouldContinue bool) { + if _, isRoot := v.(rootVertex); isRoot { + return true + } + + vertex, ok := v.(variableVertex) + if !ok { + // TODO: diags.Extend(??) + return false + } + + return vertex.Evaluator.evaluateVariable(vertex, diags, evaluateGlobals) + }) + + if diags.HasErrors() { + return diags + } + + return nil +} + +func walkBreadthFirst(g dag.AcyclicGraph, root dag.Vertex, cb func(vertex dag.Vertex) (shouldContinue bool)) { + visited := map[dag.Vertex]struct{}{} + queue := []dag.Vertex{root} + + for len(queue) > 0 { + v := queue[0] + queue = queue[1:] // pop + + if _, contained := visited[v]; !contained { + visited[v] = struct{}{} + shouldContinue := cb(v) + + if shouldContinue { + for _, child := range g.DownEdges(v).List() { + queue = append(queue, child) + } + } + } + } +} + +func generateTypeFromMap(value map[string]cty.Value) cty.Type { + typeMap := map[string]cty.Type{} + for k, v := range value { + typeMap[k] = v.Type() + } + return cty.Object(typeMap) +} diff --git a/config/config_graph_test.go b/config/config_graph_test.go new file mode 100644 index 0000000000..24b290bb92 --- /dev/null +++ b/config/config_graph_test.go @@ -0,0 +1,20 @@ +package config + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGraphCreation(t *testing.T) { + filename := "../test/fixture-config-graph/one/two/three/" + DefaultTerragruntConfigPath + + child, parent, err := ParseConfigVariables(filename, terragruntOptionsForTest(t, filename)) + if err != nil { + t.Error(err) + return + } + + assert.Equal(t,"test-us-east-1", child.Variables["local"].AsValueMap()["full-name"].AsString()) + assert.Equal(t,"../../../terragrunt.hcl-one/two/three", child.Variables["global"].AsValueMap()["source-postfix"].AsString()) + assert.Equal(t,"../../../terragrunt.hcl-one/two/three", parent.Variables["global"].AsValueMap()["source-postfix"].AsString()) +} diff --git a/test/fixture-config-graph/one/two/three/terragrunt.hcl b/test/fixture-config-graph/one/two/three/terragrunt.hcl new file mode 100644 index 0000000000..7e0ba19c85 --- /dev/null +++ b/test/fixture-config-graph/one/two/three/terragrunt.hcl @@ -0,0 +1,17 @@ +locals { + full-name = "${local.name}-${local.region}" + name = "test" + region = "us-east-1" + parent = "${local.parent-dir}/terragrunt.hcl" + parent-dir = "../../.." +} +globals { + region = local.region + source-postfix = "${local.parent}-${include.relative}" +} +include { + path = "${local.parent}" +} +input = { + region = global.region +} diff --git a/test/fixture-config-graph/terragrunt.hcl b/test/fixture-config-graph/terragrunt.hcl new file mode 100644 index 0000000000..ebaebcdd47 --- /dev/null +++ b/test/fixture-config-graph/terragrunt.hcl @@ -0,0 +1,10 @@ +locals { + source-prefix = "src-" +} +globals { + region = "us-west-2" + source-postfix = null +} +terraform { + source = "${local.source-prefix}${global.source-postfix}" +}