Performance enhancement of GitTopLevelDir in shell pkg

cache/cache.go

@@ -0,0 +1,49 @@
+package cache
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "sync"
+
+ "github.com/gruntwork-io/terragrunt/options"
+)
+
+type CacheKey interface {
+ string
+}
+
+type CacheValue interface {
+ string | options.IAMRoleOptions
+}
+
+type GenericCache[cacheValue CacheValue] struct {
+ Cache map[string]cacheValue
+ Mutex *sync.Mutex
+}
+
+// NewGenericCache - create new generic cache
+func NewGenericCache[cacheValue CacheValue]() *GenericCache[cacheValue] {
+ return &GenericCache[cacheValue]{
+ Cache: map[string]cacheValue{},
+ Mutex: &sync.Mutex{},
+ }
+}
+
+// Get - get cached value, sha256 hash is used as key to have fixed length keys and avoid duplicates
+func (cache *GenericCache[CacheValue]) Get(key string) (CacheValue, bool) {
+ cache.Mutex.Lock()
+ defer cache.Mutex.Unlock()
+ keyHash := sha256.Sum256([]byte(key))
+ cacheKey := fmt.Sprintf("%x", keyHash)
+ value, found := cache.Cache[cacheKey]
+ return value, found
+}
+
+// Put - put value in cache, sha256 hash is used as key to have fixed length keys and avoid duplicates
+func (cache *GenericCache[cacheValue]) Put(key string, value cacheValue) {
+ cache.Mutex.Lock()
+ defer cache.Mutex.Unlock()
+ keyHash := sha256.Sum256([]byte(key))
+ cacheKey := fmt.Sprintf("%x", keyHash)
+ cache.Cache[cacheKey] = value
+}

cache/cache_test.go

@@ -0,0 +1,56 @@
+package cache
+
+import (
+ "testing"
+
+ "github.com/gruntwork-io/terragrunt/options"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestStringCacheCreation(t *testing.T) {
+ t.Parallel()
+
+ cache := NewGenericCache[string]()
+
+ assert.NotNil(t, cache.Mutex)
+ assert.NotNil(t, cache.Cache)
+
+ assert.Equal(t, 0, len(cache.Cache))
+}
+
+func TestStringCacheOperation(t *testing.T) {
+ t.Parallel()
+
+ cache := NewGenericCache[string]()
+
+ value, found := cache.Get("potato")
+
+ assert.False(t, found)
+ assert.Empty(t, value)
+
+ cache.Put("potato", "carrot")
+ value, found = cache.Get("potato")
+
+ assert.True(t, found)
+ assert.NotEmpty(t, value)
+ assert.Equal(t, "carrot", value)
+}
+
+func TestIAMRoleOptionsCacheOperation(t *testing.T) {
+ t.Parallel()
+
+ cache := NewGenericCache[options.IAMRoleOptions]()
+
+ value, found := cache.Get("option1")
+
+ assert.False(t, found)
+ assert.Equal(t, options.IAMRoleOptions{}, value)
+
+ iamRoleOption := &options.IAMRoleOptions{RoleARN: "random"}
+
+ cache.Put("option1", *iamRoleOption)
+ value, found = cache.Get("option1")
+
+ assert.True(t, found)
+ assert.Equal(t, *iamRoleOption, value)
+}

config/cache.go

@@ -1,79 +1,10 @@
 package config
 
 import (
- "crypto/sha256"
  "fmt"
  "sync"
-
- "github.com/gruntwork-io/terragrunt/options"
 )
 
-// StringCache - structure to store cached values
-type StringCache struct {
- Cache map[string]string
- Mutex *sync.Mutex
-}
-
-// NewStringCache - create new string cache
-func NewStringCache() *StringCache {
- return &StringCache{
- Cache: map[string]string{},
- Mutex: &sync.Mutex{},
- }
-}
-
-// Get - get cached value, sha256 hash is used as key to have fixed length keys and avoid duplicates
-func (cache *StringCache) Get(key string) (string, bool) {
- cache.Mutex.Lock()
- defer cache.Mutex.Unlock()
- keyHash := sha256.Sum256([]byte(key))
- cacheKey := fmt.Sprintf("%x", keyHash)
- value, found := cache.Cache[cacheKey]
- return value, found
-}
-
-// Put - put value in cache, sha256 hash is used as key to have fixed length keys and avoid duplicates
-func (cache *StringCache) Put(key string, value string) {
- cache.Mutex.Lock()
- defer cache.Mutex.Unlock()
- keyHash := sha256.Sum256([]byte(key))
- cacheKey := fmt.Sprintf("%x", keyHash)
- cache.Cache[cacheKey] = value
-}
-
-// IAMRoleOptionsCache - cache for IAMRole options
-type IAMRoleOptionsCache struct {
- Cache map[string]options.IAMRoleOptions
- Mutex *sync.Mutex
-}
-
-// NewIAMRoleOptionsCache - create new cache for IAM roles
-func NewIAMRoleOptionsCache() *IAMRoleOptionsCache {
- return &IAMRoleOptionsCache{
- Cache: map[string]options.IAMRoleOptions{},
- Mutex: &sync.Mutex{},
- }
-}
-
-// Get - get cached value, sha256 hash is used as key to have fixed length keys and avoid duplicates
-func (cache *IAMRoleOptionsCache) Get(key string) (options.IAMRoleOptions, bool) {
- cache.Mutex.Lock()
- defer cache.Mutex.Unlock()
- keyHash := sha256.Sum256([]byte(key))
- cacheKey := fmt.Sprintf("%x", keyHash)
- value, found := cache.Cache[cacheKey]
- return value, found
-}
-
-// Put - put value in cache, sha256 hash is used as key to have fixed length keys and avoid duplicates
-func (cache *IAMRoleOptionsCache) Put(key string, value options.IAMRoleOptions) {
- cache.Mutex.Lock()
- defer cache.Mutex.Unlock()
- keyHash := sha256.Sum256([]byte(key))
- cacheKey := fmt.Sprintf("%x", keyHash)
- cache.Cache[cacheKey] = value
-}
-
 // TerragruntConfigCache - structure to store cached values
 type TerragruntConfigCache struct {
  Cache map[string]TerragruntConfig

config/cache_test.go

@@ -6,35 +6,6 @@ import (
  "github.com/stretchr/testify/assert"
 )
 
-func TestStringCacheCreation(t *testing.T) {
- t.Parallel()
-
- cache := NewStringCache()
-
- assert.NotNil(t, cache.Mutex)
- assert.NotNil(t, cache.Cache)
-
- assert.Equal(t, 0, len(cache.Cache))
-}
-
-func TestStringCacheOperation(t *testing.T) {
- t.Parallel()
-
- cache := NewStringCache()
-
- value, found := cache.Get("potato")
-
- assert.False(t, found)
- assert.Empty(t, value)
-
- cache.Put("potato", "carrot")
- value, found = cache.Get("potato")
-
- assert.True(t, found)
- assert.NotEmpty(t, value)
- assert.Equal(t, "carrot", value)
-}
-
 func TestTerragruntConfigCacheCreation(t *testing.T) {
  t.Parallel()
 

config/config.go

@@ -22,6 +22,7 @@ import (
 
  "github.com/gruntwork-io/go-commons/errors"
  "github.com/gruntwork-io/go-commons/files"
+ "github.com/gruntwork-io/terragrunt/cache"
  "github.com/gruntwork-io/terragrunt/codegen"
  "github.com/gruntwork-io/terragrunt/options"
  "github.com/gruntwork-io/terragrunt/remote"
@@ -769,7 +770,7 @@ func ParseConfigString(
 }
 
 // iamRoleCache - store for cached values of IAM roles
-var iamRoleCache = NewIAMRoleOptionsCache()
+var iamRoleCache = cache.NewGenericCache[options.IAMRoleOptions]()
 
 // setIAMRole - extract IAM role details from Terragrunt flags block
 func setIAMRole(configString string, terragruntOptions *options.TerragruntOptions, includeFromChild *IncludeConfig, filename string) error {

config/config_helpers.go

@@ -19,6 +19,7 @@ import (
 
  "github.com/gruntwork-io/go-commons/errors"
  "github.com/gruntwork-io/terragrunt/aws_helper"
+ "github.com/gruntwork-io/terragrunt/cache"
  "github.com/gruntwork-io/terragrunt/options"
  "github.com/gruntwork-io/terragrunt/shell"
  "github.com/gruntwork-io/terragrunt/util"
@@ -285,7 +286,7 @@ func parseGetEnvParameters(parameters []string) (EnvVar, error) {
 
 // runCommandCache - cache of evaluated `run_cmd` invocations
 // see: https://github.com/gruntwork-io/terragrunt/issues/1427
-var runCommandCache = NewStringCache()
+var runCommandCache = cache.NewGenericCache[string]()
 
 // runCommand is a helper function that runs a command and returns the stdout as the interporation
 // for each `run_cmd` in locals section, function is called twice
@@ -664,7 +665,7 @@ func getModulePathFromSourceUrl(sourceUrl string) (string, error) {
 //
 // The cache keys are the canonical paths to the encrypted files, and the values are the
 // plain-text result of the decrypt operation.
-var sopsCache = NewStringCache()
+var sopsCache = cache.NewGenericCache[string]()
 
 // decrypts and returns sops encrypted utf-8 yaml or json data as a string
 func sopsDecryptFile(params []string, trackInclude *TrackInclude, terragruntOptions *options.TerragruntOptions) (string, error) {

shell/run_shell_cmd.go

@@ -14,6 +14,7 @@ import (
 
  "github.com/gruntwork-io/go-commons/collections"
  "github.com/gruntwork-io/go-commons/errors"
+ "github.com/gruntwork-io/terragrunt/cache"
  "github.com/gruntwork-io/terragrunt/options"
  "github.com/gruntwork-io/terragrunt/terraform"
  "github.com/gruntwork-io/terragrunt/util"
@@ -268,8 +269,14 @@ type CmdOutput struct {
  Stderr string
 }
 
+var gitTopLevelDirs = cache.NewGenericCache[string]()
+
 // GitTopLevelDir - fetch git repository path from passed directory
 func GitTopLevelDir(terragruntOptions *options.TerragruntOptions, path string) (string, error) {
+ if gitTopLevelDir, found := gitTopLevelDirs.Get(path); found {
+ return gitTopLevelDir, nil
+ }
+
  stdout := bytes.Buffer{}
  stderr := bytes.Buffer{}
  opts, err := options.NewTerragruntOptionsWithConfigPath(path)
@@ -284,7 +291,9 @@ func GitTopLevelDir(terragruntOptions *options.TerragruntOptions, path string) (
  if err != nil {
  return "", err
  }
- return strings.TrimSpace(cmd.Stdout), nil
+ cmdOutput := strings.TrimSpace(cmd.Stdout)
+ gitTopLevelDirs.Put(path, cmdOutput)
+ return cmdOutput, nil
 }
 
 // ProcessExecutionError - error returned when a command fails, contains StdOut and StdErr

shell/run_shell_cmd_test.go

@@ -53,3 +53,36 @@ func TestRunShellOutputToStderrAndStdout(t *testing.T) {
  assert.True(t, strings.Contains(stderr.String(), "Terraform"), "Output directed to stderr")
  assert.True(t, len(stdout.String()) == 0, "No output to stdout")
 }
+
+func TestGitLevelTopDirCaching(t *testing.T) {
+ t.Parallel()
+
+ terragruntOptions, err := options.NewTerragruntOptionsForTest("")
+ assert.Nil(t, err, "Unexpected error creating NewTerragruntOptionsForTest: %v", err)
+
+ path := "./"
+
+ cmd, err := RunShellCommandWithOutput(terragruntOptions, path, true, false, "git", "rev-parse", "--show-toplevel")
+ assert.NoError(t, err)
+ expectedResult := strings.TrimSpace(cmd.Stdout)
+
+ actualResult, err := GitTopLevelDir(terragruntOptions, path)
+ assert.NoError(t, err, "Unexpected error executing GitTopLevelDir: %v", err)
+ assert.Equal(t, expectedResult, actualResult)
+
+ cachedResult, found := gitTopLevelDirs.Get(path)
+ assert.True(t, found)
+ assert.Equal(t, expectedResult, cachedResult)
+
+ delete(gitTopLevelDirs.Cache, path)
+}
+
+func BenchmarkPerformanceOfGitTopLevelDir(b *testing.B) {
+ for i := 0; i <= b.N; i++ {
+ terragruntOptions, err := options.NewTerragruntOptionsForTest("")
+ assert.Nil(b, err, "Unexpected error creating NewTerragruntOptionsForTest: %v", err)
+
+ _, err = GitTopLevelDir(terragruntOptions, "")
+ assert.Nil(b, err, "Unexpected error running GitTopLevelDir: %v", err)
+ }
+}