feat: evaluator v2

Signed-off-by: Tiago Natel <[email protected]>

Author: i4ki

cmd/terramate/cli/cli.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime/pprof"
 	"strings"
 	"time"
 
@@ -34,6 +35,7 @@ import (
 	"github.com/terramate-io/terramate/hcl/fmt"
 	"github.com/terramate-io/terramate/hcl/info"
 	"github.com/terramate-io/terramate/modvendor/download"
+	"github.com/terramate-io/terramate/runtime"
 	"github.com/terramate-io/terramate/versions"
 
 	"github.com/terramate-io/terramate/stack/trigger"
@@ -109,6 +111,7 @@ type cliSpec struct {
 	LogDestination string   `optional:"true" default:"stderr" enum:"stderr,stdout" help:"Destination of log messages"`
 	Quiet          bool     `optional:"false" help:"Disable output"`
 	Verbose        int      `short:"v" optional:"true" default:"0" type:"counter" help:"Increase verboseness of output"`
+	CPUProfiling   bool     `optional:"true" default:"false" help:"Create a CPU profile file when running"`
 
 	DisableCheckGitUntracked   bool `optional:"true" default:"false" help:"Disable git check for untracked files"`
 	DisableCheckGitUncommitted bool `optional:"true" default:"false" help:"Disable git check for uncommitted files"`
@@ -324,6 +327,19 @@ func newCLI(version string, args []string, stdin io.Reader, stdout, stderr io.Wr
 		fatal(err, "parsing cli args %v", args)
 	}
 
+	if parsedArgs.CPUProfiling {
+		stdfmt.Println("Creating CPU profile...")
+		f, err := os.Create("terramate.prof")
+		if err != nil {
+			fatal(err, "can't create profile output file")
+		}
+		err = pprof.StartCPUProfile(f)
+		if err != nil {
+			fatal(err, "error when starting CPU profiling")
+		}
+
+	}
+
 	configureLogging(parsedArgs.LogLevel, parsedArgs.LogFmt,
 		parsedArgs.LogDestination, stdout, stderr)
 	// If we don't re-create the logger after configuring we get some
@@ -474,6 +490,8 @@ func newCLI(version string, args []string, stdin io.Reader, stdout, stderr io.Wr
 		log.Fatal().Msg("flag --changed provided but no git repository found")
 	}
 
+	globalsResolver := globals.NewResolver(&prj.root)
+	prj.globals = globalsResolver
 	uimode := HumanMode
 	if val := os.Getenv("CI"); envVarIsSet(val) {
 		uimode = AutomationMode
@@ -516,6 +534,13 @@ func (c *cli) run() {
 
 	logger.Debug().Msg("Handle command.")
 
+	// We start the CPU Profiling during the flags parsing, but can't defer
+	// the stop there, as the CLI parsing returns far before the program is
+	// done running. Therefore we schedule it here.
+	if c.parsedArgs.CPUProfiling {
+		defer pprof.StopCPUProfile()
+	}
+
 	switch c.ctx.Command() {
 	case "fmt":
 		c.format()
@@ -812,7 +837,7 @@ func (c *cli) gencodeWithVendor() (generate.Report, download.Report) {
 
 	log.Debug().Msg("generating code")
 
-	report := generate.Do(c.cfg(), c.vendorDir(), vendorRequestEvents)
+	report := generate.Do(c.cfg(), c.globals(), c.vendorDir(), vendorRequestEvents)
 
 	log.Debug().Msg("code generation finished, waiting for vendor requests to be handled")
 
@@ -1496,7 +1521,7 @@ func (c *cli) generateDebug() {
 		selectedStacks[stack.Dir()] = struct{}{}
 	}
 
-	results, err := generate.Load(c.cfg(), c.vendorDir())
+	results, err := generate.Load(c.cfg(), c.globals(), c.vendorDir())
 	if err != nil {
 		fatal(err, "generate debug: loading generated code")
 	}
@@ -1536,16 +1561,25 @@ func (c *cli) printStacksGlobals() {
 
 	for _, stackEntry := range c.filterStacks(report.Stacks) {
 		stack := stackEntry.Stack
-		report := globals.ForStack(c.cfg(), stack)
-		if err := report.AsError(); err != nil {
+		tree := stackEntry.Stack.Tree()
+		evalctx := eval.New(
+			stack.Dir,
+			runtime.NewResolver(c.cfg(), stack),
+			c.globals(),
+		)
+		evalctx.SetFunctions(stdlib.Functions(evalctx, tree.HostDir()))
+
+		expr, _ := ast.ParseExpression(`global`, `<print-globals>`)
+		globals, err := evalctx.Eval(expr)
+		if err != nil {
 			logger := log.With().
 				Stringer("stack", stack.Dir).
 				Logger()
 
 			errlog.Fatal(logger, err, "listing stacks globals: loading stack")
 		}
 
-		globalsStrRepr := report.Globals.String()
+		globalsStrRepr := fmt.FormatAttributes(globals.AsValueMap())
 		if globalsStrRepr == "" {
 			continue
 		}
@@ -1676,6 +1710,18 @@ func (c *cli) partialEval() {
 	}
 }
 
+func (c *cli) detectEvalContext(overrideGlobals map[string]string) *eval.Context {
+	var st *config.Stack
+	if config.IsStack(c.cfg(), c.wd()) {
+		var err error
+		st, err = config.LoadStack(c.cfg(), prj.PrjAbsPath(c.rootdir(), c.wd()))
+		if err != nil {
+			fatal(err, "setup eval context: loading stack config")
+		}
+	}
+	return c.setupEvalContext(st, overrideGlobals)
+}
+
 func (c *cli) evalRunArgs(st *config.Stack, cmd []string) []string {
 	ctx := c.setupEvalContext(st, map[string]string{})
 	var newargs []string
@@ -1749,62 +1795,42 @@ func (c *cli) outputEvalResult(val cty.Value, asJSON bool) {
 	c.output.MsgStdOut(string(data))
 }
 
-func (c *cli) detectEvalContext(overrideGlobals map[string]string) *eval.Context {
-	var st *config.Stack
-	if config.IsStack(c.cfg(), c.wd()) {
-		var err error
-		st, err = config.LoadStack(c.cfg(), prj.PrjAbsPath(c.rootdir(), c.wd()))
-		if err != nil {
-			fatal(err, "setup eval context: loading stack config")
-		}
-	}
-	return c.setupEvalContext(st, overrideGlobals)
-}
-
 func (c *cli) setupEvalContext(st *config.Stack, overrideGlobals map[string]string) *eval.Context {
-	runtime := c.cfg().Runtime()
-
-	var tdir string
+	var pdir prj.Path
 	if st != nil {
-		tdir = st.HostDir(c.cfg())
-		runtime.Merge(st.RuntimeValues(c.cfg()))
+		pdir = st.Dir
 	} else {
-		tdir = c.wd()
-	}
-
-	ctx := eval.NewContext(stdlib.NoFS(tdir))
-	ctx.SetNamespace("terramate", runtime)
-
-	wdPath := prj.PrjAbsPath(c.rootdir(), tdir)
-	tree, ok := c.cfg().Lookup(wdPath)
-	if !ok {
-		fatal(errors.E("configuration at %s not found", wdPath))
-	}
-	exprs, err := globals.LoadExprs(tree)
-	if err != nil {
-		fatal(err, "loading globals expressions")
+		pdir = prj.PrjAbsPath(c.rootdir(), c.wd())
 	}
 
+	var overrideStmts eval.Stmts
 	for name, exprStr := range overrideGlobals {
 		expr, err := ast.ParseExpression(exprStr, "<cmdline>")
 		if err != nil {
 			fatal(errors.E(err, "--global %s=%s is an invalid expresssion", name, exprStr))
 		}
 		parts := strings.Split(name, ".")
-		length := len(parts)
-		globalPath := globals.NewGlobalAttrPath(parts[0:length-1], parts[length-1])
-		exprs.SetOverride(
-			wdPath,
-			globalPath,
-			expr,
-			info.NewRange(c.rootdir(), hhcl.Range{
-				Filename: "<eval argument>",
+		ref := eval.NewRef("global", parts...)
+		overrideStmts = append(overrideStmts, eval.Stmt{
+			Origin: ref,
+			LHS:    ref,
+			Info: eval.NewInfo(pdir, info.NewRange(c.rootdir(), hhcl.Range{
 				Start:    hhcl.InitialPos,
 				End:      hhcl.InitialPos,
-			}),
-		)
+				Filename: `<cmdline>`,
+			})),
+			RHS: eval.NewExprRHS(expr),
+		})
 	}
-	_ = exprs.Eval(ctx)
+
+	tree, _ := c.cfg().Lookup(pdir)
+	ctx := eval.New(
+		tree.Dir(),
+		runtime.NewResolver(c.cfg(), st),
+		globals.NewResolver(c.cfg(), overrideStmts...),
+	)
+
+	ctx.SetFunctions(stdlib.NoFS(ctx, c.wd()))
 	return ctx
 }
 
@@ -1821,7 +1847,7 @@ func (c *cli) checkOutdatedGeneratedCode() {
 		return
 	}
 
-	outdatedFiles, err := generate.DetectOutdated(c.cfg(), c.vendorDir())
+	outdatedFiles, err := generate.DetectOutdated(c.cfg(), c.globals(), c.vendorDir())
 	if err != nil {
 		fatal(err, "failed to check outdated code on project")
 	}
@@ -1860,11 +1886,12 @@ func (c *cli) gitSafeguardRemoteEnabled() bool {
 	return true
 }
 
-func (c *cli) wd() string           { return c.prj.wd }
-func (c *cli) rootdir() string      { return c.prj.rootdir }
-func (c *cli) cfg() *config.Root    { return &c.prj.root }
-func (c *cli) rootNode() hcl.Config { return c.prj.root.Tree().Node }
-func (c *cli) cred() credential     { return c.cloud.client.Credential.(credential) }
+func (c *cli) wd() string                 { return c.prj.wd }
+func (c *cli) rootdir() string            { return c.prj.rootdir }
+func (c *cli) cfg() *config.Root          { return &c.prj.root }
+func (c *cli) globals() *globals.Resolver { return c.prj.globals }
+func (c *cli) rootNode() hcl.Config       { return c.prj.root.Tree().Node }
+func (c *cli) cred() credential           { return c.cloud.client.Credential.(credential) }
 
 func (c *cli) friendlyFmtDir(dir string) (string, bool) {
 	return prj.FriendlyFmtDir(c.rootdir(), c.wd(), dir)

cmd/terramate/cli/project.go

@@ -11,6 +11,7 @@ import (
 	"github.com/terramate-io/terramate/config"
 	"github.com/terramate-io/terramate/errors"
 	"github.com/terramate-io/terramate/git"
+	"github.com/terramate-io/terramate/globals"
 	"github.com/terramate-io/terramate/hcl"
 	"github.com/terramate-io/terramate/stack"
 )
@@ -21,6 +22,7 @@ type project struct {
 	isRepo         bool
 	root           config.Root
 	baseRef        string
+	globals        *globals.Resolver
 	normalizedRepo string
 
 	git struct {

config/assert_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/terramate-io/terramate/errors"
 	"github.com/terramate-io/terramate/hcl"
 	"github.com/terramate-io/terramate/hcl/eval"
+	"github.com/terramate-io/terramate/project"
 	"github.com/terramate-io/terramate/stdlib"
 	"github.com/terramate-io/terramate/test"
 	"github.com/zclconf/go-cty/cty"
@@ -175,13 +176,15 @@ func TestAssertConfigEval(t *testing.T) {
 		tcase := tcase
 		t.Run(tcase.name, func(t *testing.T) {
 			t.Parallel()
-			hclctx := eval.NewContext(stdlib.Functions(test.TempDir(t)))
+			evalctx := eval.New(project.RootPath)
+			funcs := stdlib.Functions(evalctx, t.TempDir())
+			evalctx.SetFunctions(funcs)
 
 			for k, v := range tcase.namespaces {
-				hclctx.SetNamespace(k, v.asCtyMap())
+				evalctx.SetNamespace(k, v.asCtyMap())
 			}
 
-			got, err := config.EvalAssert(hclctx, tcase.assert)
+			got, err := config.EvalAssert(evalctx, tcase.assert)
 			assert.IsError(t, err, tcase.wantErr)
 			if !equalAsserts(tcase.want, got) {
 				t.Fatalf("got %#v != want %#v", got, tcase.want)

config/config.go

@@ -38,6 +38,8 @@ const (
 type Root struct {
 	tree Tree
 
+	lookupCache map[string]*Tree
+
 	runtime project.Runtime
 }
 
@@ -104,7 +106,8 @@ func TryLoadConfig(fromdir string) (tree *Root, configpath string, found bool, e
 // NewRoot creates a new [Root] tree for the cfg tree.
 func NewRoot(tree *Tree) *Root {
 	r := &Root{
-		tree: *tree,
+		tree:        *tree,
+		lookupCache: make(map[string]*Tree),
 	}
 	r.initRuntime()
 	return r
@@ -127,7 +130,13 @@ func (root *Root) HostDir() string { return root.tree.RootDir() }
 
 // Lookup a node from the root using a filesystem query path.
 func (root *Root) Lookup(path project.Path) (*Tree, bool) {
-	return root.tree.lookup(path)
+	tree, ok := root.lookupCache[path.String()]
+	if ok {
+		return tree, tree != nil
+	}
+	tree, ok = root.tree.lookup(path)
+	root.lookupCache[path.String()] = tree
+	return tree, ok
 }
 
 // StacksByPaths returns the stacks from the provided relative paths.

config/script_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/terramate-io/terramate/hcl/ast"
 	"github.com/terramate-io/terramate/hcl/eval"
 	"github.com/terramate-io/terramate/hcl/info"
+	"github.com/terramate-io/terramate/project"
 	"github.com/terramate-io/terramate/stdlib"
 	"github.com/terramate-io/terramate/test"
 
@@ -298,7 +299,9 @@ func TestScriptEval(t *testing.T) {
 		tcase := tcase
 		t.Run(tcase.name, func(t *testing.T) {
 			t.Parallel()
-			hclctx := eval.NewContext(stdlib.Functions(test.TempDir(t)))
+			scopedir := test.TempDir(t)
+			hclctx := eval.New(project.NewPath("/"))
+			hclctx.SetFunctions(stdlib.Functions(hclctx, scopedir))
 			hclctx.SetNamespace("global", tcase.globals)
 
 			got, err := config.EvalScript(hclctx, tcase.script)