diff --git a/.circleci/config.yml b/.circleci/config.yml index cbf554042..f4ccfc8b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -161,6 +161,11 @@ jobs: export GOPATH=~/go/bin && export PATH=$PATH:$GOPATH pre-commit install pre-commit run --all-files + - run: + name: generate mocks + command: | + go install github.com/vektra/mockery/v2@v2.36.0 + go generate ./... - run: name: run lint command: | diff --git a/.gitignore b/.gitignore index 3c0f84693..6a7a2314b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ vendor .terraform.lock.hcl terragrunt .DS_Store +mocks/ diff --git a/cli/commands/flags.go b/cli/commands/flags.go index bfd057235..2624777b5 100644 --- a/cli/commands/flags.go +++ b/cli/commands/flags.go @@ -50,20 +50,18 @@ const ( TerragruntJsonOutDirFlagName = "terragrunt-json-out-dir" // Terragrunt Provider Cache flags/envs - TerragruntProviderCacheFlagName = "terragrunt-provider-cache" - TerragruntProviderCacheEnvVarName = "TERRAGRUNT_PROVIDER_CACHE" - TerragruntProviderCacheDirFlagName = "terragrunt-provider-cache-dir" - TerragruntProviderCacheDirEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_DIR" - TerragruntProviderCacheDisablePartialLockFileFlagName = "terragrunt-provider-cache-disable-partial-lock-file" - TerragruntProviderCacheDisablePartialLockFileEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_DISABLE_PARTIAL_LOCK_FILE" - TerragruntProviderCacheHostnameFlagName = "terragrunt-provider-cache-hostname" - TerragruntProviderCacheHostnameEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_HOSTNAME" - TerragruntProviderCachePortFlagName = "terragrunt-provider-cache-port" - TerragruntProviderCachePortEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_PORT" - TerragruntProviderCacheTokenFlagName = "terragrunt-provider-cache-token" - TerragruntProviderCacheTokenEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_TOKEN" - TerragruntProviderCacheRegistryNamesFlagName = "terragrunt-provider-cache-registry-names" - TerragruntProviderCacheRegistryNamesEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_REGISTRY_NAMES" + TerragruntProviderCacheFlagName = "terragrunt-provider-cache" + TerragruntProviderCacheEnvVarName = "TERRAGRUNT_PROVIDER_CACHE" + TerragruntProviderCacheDirFlagName = "terragrunt-provider-cache-dir" + TerragruntProviderCacheDirEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_DIR" + TerragruntProviderCacheHostnameFlagName = "terragrunt-provider-cache-hostname" + TerragruntProviderCacheHostnameEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_HOSTNAME" + TerragruntProviderCachePortFlagName = "terragrunt-provider-cache-port" + TerragruntProviderCachePortEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_PORT" + TerragruntProviderCacheTokenFlagName = "terragrunt-provider-cache-token" + TerragruntProviderCacheTokenEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_TOKEN" + TerragruntProviderCacheRegistryNamesFlagName = "terragrunt-provider-cache-registry-names" + TerragruntProviderCacheRegistryNamesEnvVarName = "TERRAGRUNT_PROVIDER_CACHE_REGISTRY_NAMES" HelpFlagName = "help" VersionFlagName = "version" @@ -287,12 +285,6 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags { EnvVar: TerragruntProviderCacheDirEnvVarName, Usage: "The path to the Terragrunt provider cache directory. By default, 'terragrunt/providers' folder in the user cache directory.", }, - &cli.BoolFlag{ - Name: TerragruntProviderCacheDisablePartialLockFileFlagName, - Destination: &opts.ProviderCacheDisablePartialLockFile, - EnvVar: TerragruntProviderCacheDisablePartialLockFileEnvVarName, - Usage: "Don't use 'plugin_cache_may_break_dependency_lock_file' with Terragrunt provider caching. Provider downloads for modules without lock files will be much slower.", - }, &cli.GenericFlag[string]{ Name: TerragruntProviderCacheTokenFlagName, Destination: &opts.ProviderCacheToken, diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 91cc49130..9f10d53ce 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -21,6 +21,7 @@ import ( "github.com/gruntwork-io/terragrunt/terraform/cache/controllers" "github.com/gruntwork-io/terragrunt/terraform/cache/handlers" "github.com/gruntwork-io/terragrunt/terraform/cliconfig" + "github.com/gruntwork-io/terragrunt/terraform/getproviders" "github.com/gruntwork-io/terragrunt/util" "golang.org/x/exp/maps" ) @@ -31,7 +32,7 @@ const ( ) var ( - // HTTPStatusCacheProviderReg is regular expression to determine the success result of the command `terraform lock providers -platform=cache provider`. + // HTTPStatusCacheProviderReg is regular expression to determine the success result of the command `terraform init`. // The reg matches if the text contains "423 Locked", for example: // // - registry.terraform.io/hashicorp/template: could not query provider registry for registry.terraform.io/hashicorp/template: 423 Locked. @@ -68,14 +69,6 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e return nil, errors.WithStackTrace(err) } - if opts.ProviderCacheArchiveDir == "" { - opts.ProviderCacheArchiveDir = filepath.Join(cacheDir, "archives") - } - - if opts.ProviderCacheArchiveDir, err = filepath.Abs(opts.ProviderCacheArchiveDir); err != nil { - return nil, errors.WithStackTrace(err) - } - if opts.ProviderCacheToken == "" { opts.ProviderCacheToken = uuid.New().String() } @@ -95,7 +88,6 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e cache.WithToken(opts.ProviderCacheToken), cache.WithUserProviderDir(userProviderDir), cache.WithProviderCacheDir(opts.ProviderCacheDir), - cache.WithProviderArchiveDir(opts.ProviderCacheArchiveDir), ) return &ProviderCache{Server: cache}, nil @@ -111,10 +103,9 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti } var ( - cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename) - terraformLockFilename = filepath.Join(opts.WorkingDir, terraform.TerraformLockFile) - cacheRequestID = uuid.New().String() - env = providerCacheEnvironment(opts, cliConfigFilename) + cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename) + cacheRequestID = uuid.New().String() + env = providerCacheEnvironment(opts, cliConfigFilename) ) // Create terraform cli config file that enables provider caching and does not use provider cache dir @@ -130,29 +121,16 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti return nil, err } - cache.Provider.WaitForCacheReady(cacheRequestID) + caches := cache.Provider.WaitForCacheReady(cacheRequestID) + if err := getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches); err != nil { + return nil, err + } // Create terraform cli config file that uses provider cache dir if err := cache.createLocalCLIConfig(opts, cliConfigFilename, ""); err != nil { return nil, err } - if opts.ProviderCacheDisablePartialLockFile && !util.FileExists(terraformLockFilename) { - log.Infof("Getting terraform modules for %s", opts.WorkingDir) - if err := runTerraformCommand(ctx, opts, []string{terraform.CommandNameGet}, env); err != nil { - return nil, err - } - - log.Infof("Generating Terraform lock file for %s", opts.WorkingDir) - // Create complete terraform lock files. By default this feature is disabled, since it's not superfast. - // Instead we use Terraform `TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE` feature, that creates hashes from the local cache. - // And since the Terraform developers warn that this feature will be removed soon, it's good to have a workaround. - if err := runTerraformCommand(ctx, opts, []string{terraform.CommandNameProviders, terraform.CommandNameLock}, env); err != nil { - return nil, err - } - - } - cloneOpts := opts.Clone(opts.TerragruntConfigPath) cloneOpts.WorkingDir = opts.WorkingDir maps.Copy(cloneOpts.Env, env) @@ -252,6 +230,7 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a if err := shell.RunTerraformCommand(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 { return err } + return nil } @@ -266,12 +245,6 @@ func providerCacheEnvironment(opts *options.TerragruntOptions, cliConfigFile str envs[envName] = opts.ProviderCacheToken } - if !opts.ProviderCacheDisablePartialLockFile { - // By using `TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE` we force terraform to generate `.terraform.lock.hcl` only based on cached files, otherwise it downloads three files (provider zip archive, SHA256SUMS, sig) from the original registry to calculate hashes. - // https://developer.hashicorp.com/terraform/cli/config/config-file#allowing-the-provider-plugin-cache-to-break-the-dependency-lock-file - envs[terraform.EnvNameTFPluginCacheMayBreakDependencyLockFile] = "1" - } - // By using `TF_CLI_CONFIG_FILE` we force terraform to use our auto-generated cli configuration file. // https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_cli_config_file envs[terraform.EnvNameTFCLIConfigFile] = cliConfigFile diff --git a/docs/_docs/02_features/provider-cache.md b/docs/_docs/02_features/provider-cache.md index 395cf6a1b..7d4aad7d1 100644 --- a/docs/_docs/02_features/provider-cache.md +++ b/docs/_docs/02_features/provider-cache.md @@ -81,9 +81,8 @@ terragrunt apply * Configure Terraform instances to use the Terragrunt Provider Cache server as a remote registry: * Create local CLI config file `.terraformrc` for each module that concatenates the user configuration from the Terraform [CLI config file](https://developer.hashicorp.com/terraform/cli/config/config-file) with additional sections: * [provider-installation](https://developer.hashicorp.com/terraform/cli/config/config-file#provider-installation) forces Terraform to look for for the required providers in the cache directory and create symbolic links to them, if not found, then request them from the remote registry. - * [host](https://github.com/hashicorp/terraform/issues/28309) forces Terraform to [forward](#how-forwarding-terraform-requests-through-the-terragrunt-Provider-cache-works) all provider requests through the Terragrunt Provider Cache server. + * [host](https://github.com/hashicorp/terraform/issues/28309) forces Terraform to [forward](#how-forwarding-terraform-requests-through-the-terragrunt-Provider-cache-works) all provider requests through the Terragrunt Provider Cache server. The address link contains [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier) and is unique for each module, used by Terragrunt Provider Cache server to associate modules with the requested providers. * Set environment variables: - * [TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE](https://developer.hashicorp.com/terraform/cli/config/config-file#allowing-the-provider-plugin-cache-to-break-the-dependency-lock-file) allows to generate `.terraform.lock.hcl` files based only on provider hashes from the cache directory. * [TF_CLI_CONFIG_FILE](https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_plugin_cache_dir) sets to use just created local CLI config `.terragrunt-cache/.terraformrc` * [TF_TOKEN_*](https://developer.hashicorp.com/terraform/cli/config/config-file#environment-variable-credentials) sets per-remote-registry tokens for authentication to Terragrunt Provider Cache server. * Any time Terragrunt is going to run `init`: @@ -91,7 +90,7 @@ terragrunt apply * The Terragrunt Provider Cache server will download the provider from the remote registry, unpack and store it into the cache directory or [create a symlink](#reusing-providers-from-the-user-plugins-directory) if the required provider exists in the user plugins directory. Note that the Terragrunt Provider Cache server will ensure that each unique provider is only ever downloaded and stored on disk once, handling concurrency (from multiple Terraform and Terragrunt instances) correctly. Along with the provider, the cache server downloads hashes and signatures of the providers to check that the files are not corrupted. * The Terragrunt Provider Cache server returns the HTTP status _429 Locked_ to Terraform. This is because we do _not_ want Terraform to actually download any providers as a result of calling `terraform init`; we only use that command to request the Terragrunt Provider Cache Server to start caching providers. * At this point, all providers are downloaded and cached, so finally, we run `terragrunt init` a second time, which will find all the providers it needs in the cache, and it'll create symlinks to them nearly instantly, with no additional downloading. - * Note that if a Terraform module doesn't have a lock file, Terraform does _not_ use the cache, so it would end up downloading all the providers from scratch. To work around this, by default, we use the `plugin_cache_may_break_dependency_lock_file` feature, which, for modules without lock files, will allow Terraform to automatically generate a partial lock file if it finds the providers it needs in the cache, with no additional downloading. This ensures that no additional downloads are necessary, but at the cost of partial lock files. If you wish to disable this feature, set the [`terragrunt-provider-cache-disable-partial-lock-file`](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-provider-cache-disable-partial-lock-file) flag, and for modules without a lock file, Terragrunt will call `terraform providers lock`, which will give you a complete lock file, at the cost of more downloading and bandwidth. + * Note that if a Terraform module doesn't have a lock file, Terraform does _not_ use the cache, so it would end up downloading all the providers from scratch. To work around this, we generate `.terraform.lock.hcl` based on the request made by `terrafrom init` to the Terragrunt Provider Cache server. Since `terraform init` only requestes the providers that need to be added/updated, we can keep track of them using the Terragrunt Provider Cache server and update the Terraform lock file with the appropriate hashes without having to parse `tf` configs. #### Reusing providers from the user plugins directory diff --git a/go.mod b/go.mod index 67c6b24e8..79316a4fa 100644 --- a/go.mod +++ b/go.mod @@ -69,6 +69,7 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/pkg/errors v0.9.1 github.com/posener/complete v1.2.3 + github.com/rogpeppe/go-internal v1.11.0 github.com/urfave/cli/v2 v2.26.0 go.opentelemetry.io/otel v1.23.1 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.23.1 @@ -215,6 +216,7 @@ require ( github.com/sourcegraph/jsonrpc2 v0.2.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.3.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/terraform-linters/tflint-plugin-sdk v0.17.0 // indirect github.com/terraform-linters/tflint-ruleset-terraform v0.4.0 // indirect github.com/urfave/cli v1.22.14 // indirect diff --git a/go.sum b/go.sum index 569986cfc..7d39ccc9e 100644 --- a/go.sum +++ b/go.sum @@ -1010,6 +1010,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/options/options.go b/options/options.go index 127722b37..565d0773f 100644 --- a/options/options.go +++ b/options/options.go @@ -280,12 +280,6 @@ type TerragruntOptions struct { // The path to store unpacked providers. The file structure is the same as terraform plugin cache dir. ProviderCacheDir string - // The path to store archive providers that are retrieved from the source registry and cached to reduce traffic. - ProviderCacheArchiveDir string - - // Don't use 'plugin_cache_may_break_dependency_lock_file' with Terragrunt provider caching. - ProviderCacheDisablePartialLockFile bool - // The Token for authentication to the Terragrunt Provider Cache server. ProviderCacheToken string @@ -454,73 +448,71 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) *TerragruntOpt // during xxx-all commands (e.g., apply-all, plan-all). See https://github.com/gruntwork-io/terragrunt/issues/367 // for more info. return &TerragruntOptions{ - TerragruntConfigPath: terragruntConfigPath, - OriginalTerragruntConfigPath: opts.OriginalTerragruntConfigPath, - TerraformPath: opts.TerraformPath, - OriginalTerraformCommand: opts.OriginalTerraformCommand, - TerraformCommand: opts.TerraformCommand, - TerraformVersion: opts.TerraformVersion, - TerragruntVersion: opts.TerragruntVersion, - AutoInit: opts.AutoInit, - RunAllAutoApprove: opts.RunAllAutoApprove, - NonInteractive: opts.NonInteractive, - TerraformCliArgs: util.CloneStringList(opts.TerraformCliArgs), - WorkingDir: workingDir, - Logger: util.CreateLogEntryWithWriter(opts.ErrWriter, workingDir, opts.LogLevel, opts.Logger.Logger.Hooks), - LogLevel: opts.LogLevel, - ValidateStrict: opts.ValidateStrict, - Env: util.CloneStringMap(opts.Env), - Source: opts.Source, - SourceMap: opts.SourceMap, - SourceUpdate: opts.SourceUpdate, - DownloadDir: opts.DownloadDir, - Debug: opts.Debug, - OriginalIAMRoleOptions: opts.OriginalIAMRoleOptions, - IAMRoleOptions: opts.IAMRoleOptions, - IgnoreDependencyErrors: opts.IgnoreDependencyErrors, - IgnoreDependencyOrder: opts.IgnoreDependencyOrder, - IgnoreExternalDependencies: opts.IgnoreExternalDependencies, - IncludeExternalDependencies: opts.IncludeExternalDependencies, - Writer: opts.Writer, - ErrWriter: opts.ErrWriter, - MaxFoldersToCheck: opts.MaxFoldersToCheck, - AutoRetry: opts.AutoRetry, - RetryMaxAttempts: opts.RetryMaxAttempts, - RetrySleepIntervalSec: opts.RetrySleepIntervalSec, - RetryableErrors: util.CloneStringList(opts.RetryableErrors), - ExcludeDirs: opts.ExcludeDirs, - IncludeDirs: opts.IncludeDirs, - ModulesThatInclude: opts.ModulesThatInclude, - Parallelism: opts.Parallelism, - StrictInclude: opts.StrictInclude, - RunTerragrunt: opts.RunTerragrunt, - AwsProviderPatchOverrides: opts.AwsProviderPatchOverrides, - HclFile: opts.HclFile, - JSONOut: opts.JSONOut, - Check: opts.Check, - CheckDependentModules: opts.CheckDependentModules, - FetchDependencyOutputFromState: opts.FetchDependencyOutputFromState, - UsePartialParseConfigCache: opts.UsePartialParseConfigCache, - OutputPrefix: opts.OutputPrefix, - IncludeModulePrefix: opts.IncludeModulePrefix, - FailIfBucketCreationRequired: opts.FailIfBucketCreationRequired, - DisableBucketUpdate: opts.DisableBucketUpdate, - TerraformImplementation: opts.TerraformImplementation, - JsonLogFormat: opts.JsonLogFormat, - TerraformLogsToJson: opts.TerraformLogsToJson, - GraphRoot: opts.GraphRoot, - ScaffoldVars: opts.ScaffoldVars, - ScaffoldVarFiles: opts.ScaffoldVarFiles, - JsonDisableDependentModules: opts.JsonDisableDependentModules, - ProviderCache: opts.ProviderCache, - ProviderCacheToken: opts.ProviderCacheToken, - ProviderCacheDir: opts.ProviderCacheDir, - ProviderCacheArchiveDir: opts.ProviderCacheArchiveDir, - ProviderCacheDisablePartialLockFile: opts.ProviderCacheDisablePartialLockFile, - ProviderCacheRegistryNames: opts.ProviderCacheRegistryNames, - DisableLogColors: opts.DisableLogColors, - OutputFolder: opts.OutputFolder, - JsonOutputFolder: opts.JsonOutputFolder, + TerragruntConfigPath: terragruntConfigPath, + OriginalTerragruntConfigPath: opts.OriginalTerragruntConfigPath, + TerraformPath: opts.TerraformPath, + OriginalTerraformCommand: opts.OriginalTerraformCommand, + TerraformCommand: opts.TerraformCommand, + TerraformVersion: opts.TerraformVersion, + TerragruntVersion: opts.TerragruntVersion, + AutoInit: opts.AutoInit, + RunAllAutoApprove: opts.RunAllAutoApprove, + NonInteractive: opts.NonInteractive, + TerraformCliArgs: util.CloneStringList(opts.TerraformCliArgs), + WorkingDir: workingDir, + Logger: util.CreateLogEntryWithWriter(opts.ErrWriter, workingDir, opts.LogLevel, opts.Logger.Logger.Hooks), + LogLevel: opts.LogLevel, + ValidateStrict: opts.ValidateStrict, + Env: util.CloneStringMap(opts.Env), + Source: opts.Source, + SourceMap: opts.SourceMap, + SourceUpdate: opts.SourceUpdate, + DownloadDir: opts.DownloadDir, + Debug: opts.Debug, + OriginalIAMRoleOptions: opts.OriginalIAMRoleOptions, + IAMRoleOptions: opts.IAMRoleOptions, + IgnoreDependencyErrors: opts.IgnoreDependencyErrors, + IgnoreDependencyOrder: opts.IgnoreDependencyOrder, + IgnoreExternalDependencies: opts.IgnoreExternalDependencies, + IncludeExternalDependencies: opts.IncludeExternalDependencies, + Writer: opts.Writer, + ErrWriter: opts.ErrWriter, + MaxFoldersToCheck: opts.MaxFoldersToCheck, + AutoRetry: opts.AutoRetry, + RetryMaxAttempts: opts.RetryMaxAttempts, + RetrySleepIntervalSec: opts.RetrySleepIntervalSec, + RetryableErrors: util.CloneStringList(opts.RetryableErrors), + ExcludeDirs: opts.ExcludeDirs, + IncludeDirs: opts.IncludeDirs, + ModulesThatInclude: opts.ModulesThatInclude, + Parallelism: opts.Parallelism, + StrictInclude: opts.StrictInclude, + RunTerragrunt: opts.RunTerragrunt, + AwsProviderPatchOverrides: opts.AwsProviderPatchOverrides, + HclFile: opts.HclFile, + JSONOut: opts.JSONOut, + Check: opts.Check, + CheckDependentModules: opts.CheckDependentModules, + FetchDependencyOutputFromState: opts.FetchDependencyOutputFromState, + UsePartialParseConfigCache: opts.UsePartialParseConfigCache, + OutputPrefix: opts.OutputPrefix, + IncludeModulePrefix: opts.IncludeModulePrefix, + FailIfBucketCreationRequired: opts.FailIfBucketCreationRequired, + DisableBucketUpdate: opts.DisableBucketUpdate, + TerraformImplementation: opts.TerraformImplementation, + JsonLogFormat: opts.JsonLogFormat, + TerraformLogsToJson: opts.TerraformLogsToJson, + GraphRoot: opts.GraphRoot, + ScaffoldVars: opts.ScaffoldVars, + ScaffoldVarFiles: opts.ScaffoldVarFiles, + JsonDisableDependentModules: opts.JsonDisableDependentModules, + ProviderCache: opts.ProviderCache, + ProviderCacheToken: opts.ProviderCacheToken, + ProviderCacheDir: opts.ProviderCacheDir, + ProviderCacheRegistryNames: opts.ProviderCacheRegistryNames, + DisableLogColors: opts.DisableLogColors, + OutputFolder: opts.OutputFolder, + JsonOutputFolder: opts.JsonOutputFolder, } } diff --git a/terraform/cache/config.go b/terraform/cache/config.go index fc7a92a49..dddffabf3 100644 --- a/terraform/cache/config.go +++ b/terraform/cache/config.go @@ -45,13 +45,6 @@ func WithProviderCacheDir(cacheDir string) Option { } } -func WithProviderArchiveDir(archiveDir string) Option { - return func(cfg Config) Config { - cfg.providerArchiveDir = archiveDir - return cfg - } -} - func WithUserProviderDir(userProviderDir string) Option { return func(cfg Config) Config { cfg.userProviderDir = userProviderDir @@ -65,9 +58,8 @@ type Config struct { token string shutdownTimeout time.Duration - userProviderDir string - providerCacheDir string - providerArchiveDir string + userProviderDir string + providerCacheDir string } func NewConfig(opts ...Option) *Config { diff --git a/terraform/cache/controllers/downloader.go b/terraform/cache/controllers/downloader.go index d5e7b0eeb..9e32febe1 100644 --- a/terraform/cache/controllers/downloader.go +++ b/terraform/cache/controllers/downloader.go @@ -51,10 +51,10 @@ func (controller *DownloaderController) downloadProviderAction(ctx echo.Context) Host: remoteHost, Path: "/" + remotePath, } - provider := &models.Provider{DownloadURL: downloadURL.String()} + provider := models.NewProviderFromDownloadURL(downloadURL.String()) if cache := controller.ProviderService.GetProviderCache(provider); cache != nil { - if filename := cache.ArchivePath(); filename != "" { + if filename := cache.Filename(); filename != "" { log.Debugf("Using cached provider %s", cache.Provider) return ctx.File(filename) } diff --git a/terraform/cache/controllers/provider.go b/terraform/cache/controllers/provider.go index 56062db74..e82473729 100644 --- a/terraform/cache/controllers/provider.go +++ b/terraform/cache/controllers/provider.go @@ -12,6 +12,7 @@ import ( "github.com/gruntwork-io/terragrunt/terraform/cache/models" "github.com/gruntwork-io/terragrunt/terraform/cache/router" "github.com/gruntwork-io/terragrunt/terraform/cache/services" + "github.com/gruntwork-io/terragrunt/terraform/getproviders" "github.com/labstack/echo/v4" ) @@ -129,9 +130,12 @@ func (controller *ProviderController) findPlatformsAction(ctx echo.Context) erro var body map[string]json.RawMessage if cacheRequestID != "" { - if err := handlers.DecodeJSONBody(resp, provider); err != nil { + var responseBody = new(getproviders.Package) + + if err := handlers.DecodeJSONBody(resp, responseBody); err != nil { return err } + provider.Package = responseBody controller.ProviderService.CacheProvider(ctx.Request().Context(), cacheRequestID, provider) return ctx.NoContent(HTTPStatusCacheProvider) diff --git a/terraform/cache/models/provider.go b/terraform/cache/models/provider.go index 35ed12109..a42b25fa1 100644 --- a/terraform/cache/models/provider.go +++ b/terraform/cache/models/provider.go @@ -5,31 +5,27 @@ import ( "net/url" "path" - "github.com/gruntwork-io/terragrunt/terraform/provider" + "github.com/gruntwork-io/terragrunt/terraform/getproviders" ) -type SigningKeyList struct { - GPGPublicKeys []*provider.SigningKey `json:"gpg_public_keys"` -} - // Provider represents the details of the Terraform provider. type Provider struct { + *getproviders.Package + RegistryName string Namespace string Name string Version string OS string Arch string +} - Protocols []string `json:"protocols"` - Filename string `json:"filename"` - - DownloadURL string `json:"download_url"` - SHA256SumsURL string `json:"shasums_url"` - SHA256SumsSignatureURL string `json:"shasums_signature_url"` - - SHA256Sum string `json:"shasum"` - SigningKeys SigningKeyList `json:"signing_keys"` +func NewProviderFromDownloadURL(downloadURL string) *Provider { + return &Provider{ + Package: &getproviders.Package{ + DownloadURL: downloadURL, + }, + } } // VersionURL returns the URL used to query the all Versions for a single provider. @@ -53,15 +49,15 @@ func (provider *Provider) PlatformURL() *url.URL { } func (provider *Provider) String() string { - return fmt.Sprintf("%s/%s v%s", provider.Namespace, provider.Name, provider.Version) + return fmt.Sprintf("%s/%s/%s v%s", provider.RegistryName, provider.Namespace, provider.Name, provider.Version) } func (provider *Provider) Platform() string { return fmt.Sprintf("%s_%s", provider.OS, provider.Arch) } -func (provider *Provider) Path() string { - return path.Join(provider.RegistryName, provider.Namespace, provider.Name, provider.Version) +func (provider *Provider) Address() string { + return path.Join(provider.RegistryName, provider.Namespace, provider.Name) } // Match returns true if all defined provider properties are matched. diff --git a/terraform/cache/server.go b/terraform/cache/server.go index 0df675640..c06c48e48 100644 --- a/terraform/cache/server.go +++ b/terraform/cache/server.go @@ -29,7 +29,7 @@ type Server struct { func NewServer(opts ...Option) *Server { cfg := NewConfig(opts...) - providerService := services.NewProviderService(cfg.providerCacheDir, cfg.providerArchiveDir, cfg.userProviderDir) + providerService := services.NewProviderService(cfg.providerCacheDir, cfg.userProviderDir) authorization := &handlers.Authorization{ Token: cfg.token, diff --git a/terraform/cache/server_test.go b/terraform/cache/server_test.go index e02512e7c..9f6acda9e 100644 --- a/terraform/cache/server_test.go +++ b/terraform/cache/server_test.go @@ -40,13 +40,10 @@ func TestServer(t *testing.T) { providerCacheDir, err := os.MkdirTemp("", "*") require.NoError(t, err) - providerArchiveDir, err := os.MkdirTemp("", "*") - require.NoError(t, err) - pluginCacheDir, err := os.MkdirTemp("", "*") require.NoError(t, err) - opts := []Option{WithToken(token), WithProviderArchiveDir(providerArchiveDir), WithProviderCacheDir(providerCacheDir), WithUserProviderDir(pluginCacheDir)} + opts := []Option{WithToken(token), WithProviderCacheDir(providerCacheDir), WithUserProviderDir(pluginCacheDir)} testCases := []struct { opts []Option diff --git a/terraform/cache/services/provider.go b/terraform/cache/services/provider.go index 414202892..06cecae4a 100644 --- a/terraform/cache/services/provider.go +++ b/terraform/cache/services/provider.go @@ -16,7 +16,7 @@ import ( "github.com/gruntwork-io/go-commons/errors" "github.com/gruntwork-io/terragrunt/pkg/log" "github.com/gruntwork-io/terragrunt/terraform/cache/models" - "github.com/gruntwork-io/terragrunt/terraform/provider" + "github.com/gruntwork-io/terragrunt/terraform/getproviders" "github.com/gruntwork-io/terragrunt/util" "github.com/hashicorp/go-getter/v2" "github.com/hashicorp/go-multierror" @@ -65,49 +65,94 @@ type ProviderCache struct { *models.Provider requestIDs []string - started chan struct{} - archiveCached bool - ready bool + started chan struct{} + documentSHA256Sums []byte + signature []byte + archiveCached bool + ready bool userProviderDir string - providerDir string + packageDir string lockfilePath string archivePath string } -func (cache *ProviderCache) AuthenticatePackage(ctx context.Context) (*provider.PackageAuthenticationResult, error) { +func (cache *ProviderCache) DocumentSHA256Sums(ctx context.Context) ([]byte, error) { + if cache.documentSHA256Sums != nil || cache.SHA256SumsURL == "" { + return cache.documentSHA256Sums, nil + } + + var documentSHA256Sums = new(bytes.Buffer) + + if err := util.Fetch(ctx, cache.SHA256SumsURL, documentSHA256Sums); err != nil { + return nil, fmt.Errorf("failed to retrieve authentication checksums for provider %q: %w", cache.Provider, err) + } + + cache.documentSHA256Sums = documentSHA256Sums.Bytes() + return cache.documentSHA256Sums, nil +} + +func (cache *ProviderCache) Signature(ctx context.Context) ([]byte, error) { + if cache.signature != nil || cache.SHA256SumsSignatureURL == "" { + return cache.signature, nil + } + + var signature = new(bytes.Buffer) + + if err := util.Fetch(ctx, cache.SHA256SumsSignatureURL, signature); err != nil { + return nil, fmt.Errorf("failed to retrieve authentication signature for provider %q: %w", cache.Provider, err) + } + + cache.signature = signature.Bytes() + return cache.signature, nil +} + +func (cache *ProviderCache) Version() string { + return cache.Provider.Version +} + +func (cache *ProviderCache) Address() string { + return cache.Provider.Address() +} + +func (cache *ProviderCache) PackageDir() string { + return cache.packageDir +} + +func (cache *ProviderCache) AuthenticatePackage(ctx context.Context) (*getproviders.PackageAuthenticationResult, error) { var ( - checksum [sha256.Size]byte - document = new(bytes.Buffer) - signature = new(bytes.Buffer) + checksum [sha256.Size]byte + documentSHA256Sums []byte + signature []byte + err error ) - if _, err := hex.Decode(checksum[:], []byte(cache.SHA256Sum)); err != nil { - return nil, errors.Errorf("registry response includes invalid SHA256 hash %q for provider %q: %w", cache.SHA256Sum, cache.Provider, err) + if documentSHA256Sums, err = cache.DocumentSHA256Sums(ctx); err != nil { + return nil, err } - if err := util.Fetch(ctx, cache.SHA256SumsURL, document); err != nil { - return nil, fmt.Errorf("failed to retrieve authentication checksums for provider %q: %w", cache.Provider, err) + if signature, err = cache.Signature(ctx); err != nil { + return nil, err } - if err := util.Fetch(ctx, cache.SHA256SumsSignatureURL, signature); err != nil { - return nil, fmt.Errorf("failed to retrieve cryptographic signature for provider %q: %w", cache.Provider, err) + if _, err := hex.Decode(checksum[:], []byte(cache.SHA256Sum)); err != nil { + return nil, errors.Errorf("registry response includes invalid SHA256 hash %q for provider %q: %w", cache.SHA256Sum, cache.Provider, err) } - keys := make([]provider.SigningKey, len(cache.SigningKeys.GPGPublicKeys)) + keys := make([]getproviders.SigningKey, len(cache.SigningKeys.GPGPublicKeys)) for i, key := range cache.SigningKeys.GPGPublicKeys { keys[i] = *key } - providerPackage := provider.PackageAuthenticationAll( - provider.NewMatchingChecksumAuthentication(document.Bytes(), cache.Filename, checksum), - provider.NewArchiveChecksumAuthentication(checksum), - provider.NewSignatureAuthentication(document.Bytes(), signature.Bytes(), keys), + providerPackage := getproviders.PackageAuthenticationAll( + getproviders.NewMatchingChecksumAuthentication(documentSHA256Sums, cache.Package.Filename, checksum), + getproviders.NewArchiveChecksumAuthentication(checksum), + getproviders.NewSignatureAuthentication(documentSHA256Sums, signature, keys), ) return providerPackage.Authenticate(cache.archivePath) } -func (cache *ProviderCache) ArchivePath() string { +func (cache *ProviderCache) Filename() string { if util.FileExists(cache.archivePath) { return cache.archivePath } @@ -122,17 +167,17 @@ func (cache *ProviderCache) addRequestID(requestID string) { // 1. Checks if the required provider exists in the user plugins directory, located at %APPDATA%\terraform.d\plugins on Windows and ~/.terraform.d/plugins on other systems. If so, creates a symlink to this folder. (Some providers are not available for darwin_arm64, in this case we can use https://github.com/kreuzwerker/m1-terraform-provider-helper which compiles and saves providers to the user plugins directory) // 2. Downloads the provider from the original registry, unpacks and saves it into the cache directory. func (cache *ProviderCache) warmUp(ctx context.Context) error { - if util.FileExists(cache.providerDir) { + if util.FileExists(cache.packageDir) { return nil } - if err := os.MkdirAll(filepath.Dir(cache.providerDir), os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Dir(cache.packageDir), os.ModePerm); err != nil { return errors.WithStackTrace(err) } if util.FileExists(cache.userProviderDir) { - log.Debugf("Create symlink file %s to %s", cache.providerDir, cache.userProviderDir) - if err := os.Symlink(cache.userProviderDir, cache.providerDir); err != nil { + log.Debugf("Create symlink file %s to %s", cache.packageDir, cache.userProviderDir) + if err := os.Symlink(cache.userProviderDir, cache.packageDir); err != nil { return errors.WithStackTrace(err) } log.Infof("Cached %s from user plugins directory", cache.Provider) @@ -152,7 +197,7 @@ func (cache *ProviderCache) warmUp(ctx context.Context) error { log.Debugf("Unpack provider archive %s", cache.archivePath) - if err := unzip.Decompress(cache.providerDir, cache.archivePath, true, unzipFileMode); err != nil { + if err := unzip.Decompress(cache.packageDir, cache.archivePath, true, unzipFileMode); err != nil { return errors.WithStackTrace(err) } @@ -195,9 +240,6 @@ type ProviderService struct { // The path to store unpacked providers. The file structure is the same as terraform plugin cache dir. baseCacheDir string - // The path to store archive providers that are retrieved from the source registry and cached to reduce traffic. - baseArchiveDir string - // the user plugins directory, by default: %APPDATA%\terraform.d\plugins on Windows, ~/.terraform.d/plugins on other systems. baseUserCacheDir string @@ -208,22 +250,26 @@ type ProviderService struct { cacheReadyMu sync.RWMutex } -func NewProviderService(baseCacheDir, baseArchiveDir, baseUserCacheDir string) *ProviderService { +func NewProviderService(baseCacheDir, baseUserCacheDir string) *ProviderService { return &ProviderService{ baseCacheDir: baseCacheDir, baseUserCacheDir: baseUserCacheDir, - baseArchiveDir: baseArchiveDir, providerCacheWarmUpCh: make(chan *ProviderCache), } } // WaitForCacheReady returns cached providers that were requested by `terraform init` from the cache server, with an URL containing the given `requestID` value. // The function returns the value only when all cache requests have been processed. -func (service *ProviderService) WaitForCacheReady(requestID string) ProviderCaches { +func (service *ProviderService) WaitForCacheReady(requestID string) getproviders.Providers { service.cacheReadyMu.Lock() defer service.cacheReadyMu.Unlock() - return service.providerCaches.FindByRequestID(requestID) + var providers getproviders.Providers + + for _, provider := range service.providerCaches.FindByRequestID(requestID) { + providers = append(providers, provider) + } + return providers } // CacheProvider starts caching the given provider using non-blocking approach. @@ -242,10 +288,10 @@ func (service *ProviderService) CacheProvider(ctx context.Context, requestID str Provider: provider, started: make(chan struct{}, 1), - userProviderDir: filepath.Join(service.baseUserCacheDir, provider.Path(), provider.Platform()), - providerDir: filepath.Join(service.baseCacheDir, provider.Path(), provider.Platform()), - lockfilePath: filepath.Join(service.baseArchiveDir, fmt.Sprintf("%s.lock", packageName)), - archivePath: filepath.Join(service.baseArchiveDir, fmt.Sprintf("%s%s", packageName, path.Ext(provider.Filename))), + userProviderDir: filepath.Join(service.baseUserCacheDir, provider.Address(), provider.Version, provider.Platform()), + packageDir: filepath.Join(service.baseCacheDir, provider.Address(), provider.Version, provider.Platform()), + lockfilePath: filepath.Join(service.baseCacheDir, fmt.Sprintf("%s.lock", packageName)), + archivePath: filepath.Join(service.baseCacheDir, fmt.Sprintf("%s%s", packageName, path.Ext(provider.Filename))), } select { @@ -279,25 +325,10 @@ func (service *ProviderService) RunCacheWorker(ctx context.Context) error { } log.Debugf("Provider cache dir %q", service.baseCacheDir) - if service.baseArchiveDir == "" { - return errors.Errorf("provider archive directory not specified") - } - log.Debugf("Provider archive dir %q", service.baseArchiveDir) - - if service.baseCacheDir == service.baseArchiveDir { - // We can only store uncompressed provider files in `baseCacheDir` because tofu considers any files there as providers. - // https://github.com/opentofu/opentofu/blob/bdab86962fdd0a2106a744d7f8f1d3d3e7bc893e/internal/getproviders/filesystem_search.go#L27 - return errors.Errorf("the same directory is used for both unarchived and archived provider files") - } - if err := os.MkdirAll(service.baseCacheDir, os.ModePerm); err != nil { return errors.WithStackTrace(err) } - if err := os.MkdirAll(service.baseArchiveDir, os.ModePerm); err != nil { - return errors.WithStackTrace(err) - } - errGroup, ctx := errgroup.WithContext(ctx) for { select { @@ -316,7 +347,7 @@ func (service *ProviderService) RunCacheWorker(ctx context.Context) error { defer lockfile.Unlock() //nolint:errcheck if err := cache.warmUp(ctx); err != nil { - os.Remove(cache.providerDir) //nolint:errcheck + os.Remove(cache.packageDir) //nolint:errcheck os.Remove(cache.archivePath) //nolint:errcheck return err } diff --git a/terraform/getproviders/hash.go b/terraform/getproviders/hash.go new file mode 100644 index 000000000..46b1121ba --- /dev/null +++ b/terraform/getproviders/hash.go @@ -0,0 +1,113 @@ +package getproviders + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/gruntwork-io/go-commons/errors" + "github.com/rogpeppe/go-internal/dirhash" +) + +// Hash is a specially-formatted string representing a checksum of a package or the contents of the package. +type Hash string + +func (hash Hash) String() string { + return string(hash) +} + +// HashScheme is an enumeration of schemes. +type HashScheme string + +const ( + // HashScheme1 is the scheme identifier for the first hash scheme. + HashScheme1 HashScheme = HashScheme("h1:") + + // HashSchemeZip is the scheme identifier for the legacy hash scheme that applies to distribution archives (.zip files) rather than package contents. + HashSchemeZip HashScheme = HashScheme("zh:") +) + +// New creates a new Hash value with the receiver as its scheme and the given raw string as its value. +func (scheme HashScheme) New(value string) Hash { + return Hash(string(scheme) + value) +} + +// PackageHashLegacyZipSHA implements the old provider package hashing scheme of taking a SHA256 hash of the containing .zip archive itself, rather than of the contents of the archive. +func PackageHashLegacyZipSHA(path string) (Hash, error) { + archivePath, err := filepath.EvalSymlinks(path) + if err != nil { + return "", errors.WithStackTrace(err) + } + + file, err := os.Open(archivePath) + if err != nil { + return "", errors.WithStackTrace(err) + } + defer file.Close() + + hash := sha256.New() + if _, err = io.Copy(hash, file); err != nil { + return "", errors.WithStackTrace(err) + } + + gotHash := hash.Sum(nil) + return HashSchemeZip.New(fmt.Sprintf("%x", gotHash)), nil +} + +// HashLegacyZipSHAFromSHA is a convenience method to produce the schemed-string hash format from an already-calculated hash of a provider .zip archive. +func HashLegacyZipSHAFromSHA(sum [sha256.Size]byte) Hash { + return HashSchemeZip.New(fmt.Sprintf("%x", sum[:])) +} + +// PackageHashV1 computes a hash of the contents of the package at the given location using hash algorithm 1. The resulting Hash is guaranteed to have the scheme HashScheme1. +func PackageHashV1(path string) (Hash, error) { + // We'll first dereference a possible symlink at our PackageDir location, as would be created if this package were linked in from another cache. + packageDir, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + + if fileInfo, err := os.Stat(packageDir); err != nil { + return "", errors.WithStackTrace(err) + } else if !fileInfo.IsDir() { + return "", errors.Errorf("packageDir is not a directory %q", packageDir) + } + + s, err := dirhash.HashDir(packageDir, "", dirhash.Hash1) + return Hash(s), err +} + +func DocumentHashes(doc []byte) []Hash { + var hashes []Hash + + sc := bufio.NewScanner(bytes.NewReader(doc)) + for sc.Scan() { + parts := bytes.Fields(sc.Bytes()) + columns := 2 + if len(parts) != columns { + // Doesn't look like a valid sums file line, so we'll assume this whole thing isn't a checksums file. + continue + } + + // If this is a checksums file then the first part should be a hex-encoded SHA256 hash, so it should be 64 characters long and contain only hex digits. + hashStr := parts[0] + hashLen := 64 + if len(hashStr) != hashLen { + return nil // doesn't look like a checksums file + } + + var gotSHA256Sum [sha256.Size]byte + if _, err := hex.Decode(gotSHA256Sum[:], hashStr); err != nil { + return nil // doesn't look like a checksums file + } + + hashes = append(hashes, HashLegacyZipSHAFromSHA(gotSHA256Sum)) + } + + return hashes +} diff --git a/terraform/provider/hash_test.go b/terraform/getproviders/hash_test.go similarity index 97% rename from terraform/provider/hash_test.go rename to terraform/getproviders/hash_test.go index 808f5b30d..c4958af66 100644 --- a/terraform/provider/hash_test.go +++ b/terraform/getproviders/hash_test.go @@ -1,4 +1,4 @@ -package provider +package getproviders import ( "fmt" diff --git a/terraform/getproviders/hclwrite.go b/terraform/getproviders/hclwrite.go new file mode 100644 index 000000000..545e75ac7 --- /dev/null +++ b/terraform/getproviders/hclwrite.go @@ -0,0 +1,43 @@ +package getproviders + +import ( + "strings" + + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" +) + +// getAttributeValueAsString returns a value of Attribute as string. There is no way to get value as string directly, so we parses tokens of Attribute and build string representation. +func getAttributeValueAsUnquotedString(attr *hclwrite.Attribute) string { + // find TokenEqual + expr := attr.Expr() + exprTokens := expr.BuildTokens(nil) + + // TokenIdent records SpaceBefore, but we should ignore it here. + quotedValue := strings.TrimSpace(string(exprTokens.Bytes())) + + // unquote + value := strings.Trim(quotedValue, "\"") + + return value +} + +// tokensForListPerLine builds a hclwrite.Tokens for a given hashes, but breaks the line for each element. +func tokensForListPerLine(hashes []Hash) hclwrite.Tokens { + // The original TokensForValue implementation does not break line by line for hashes, so we build a token sequence by ourselves. + tokens := append(hclwrite.Tokens{}, + &hclwrite.Token{Type: hclsyntax.TokenOBrack, Bytes: []byte{'['}}, + &hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}}) + + for _, hash := range hashes { + ts := hclwrite.TokensForValue(cty.StringVal(hash.String())) + tokens = append(tokens, ts...) + tokens = append(tokens, + &hclwrite.Token{Type: hclsyntax.TokenComma, Bytes: []byte{','}}, + &hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}}) + } + + tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCBrack, Bytes: []byte{']'}}) + return tokens +} diff --git a/terraform/getproviders/lock.go b/terraform/getproviders/lock.go new file mode 100644 index 000000000..c4d52ca4d --- /dev/null +++ b/terraform/getproviders/lock.go @@ -0,0 +1,111 @@ +package getproviders + +import ( + "context" + "os" + "path/filepath" + "sort" + + "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/terragrunt/pkg/log" + "github.com/gruntwork-io/terragrunt/terraform" + "github.com/gruntwork-io/terragrunt/util" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/zclconf/go-cty/cty" + "golang.org/x/exp/slices" +) + +// UpdateLockfile updates the dependency lock file. If `.terraform.lock.hcl` does not exist, it will be created, otherwise it will be updated. +func UpdateLockfile(ctx context.Context, workingDir string, providers Providers) error { + var ( + filename = filepath.Join(workingDir, terraform.TerraformLockFile) + file = hclwrite.NewFile() + ) + + if util.FileExists(filename) { + content, err := os.ReadFile(filename) + if err != nil { + return errors.WithStackTrace(err) + } + + var diags hcl.Diagnostics + file, diags = hclwrite.ParseConfig(content, filename, hcl.Pos{Line: 1, Column: 1}) + if diags.HasErrors() { + return errors.WithStackTrace(diags) + } + } + + if err := updateLockfile(ctx, file, providers); err != nil { + return err + } + + if err := os.WriteFile(filename, file.Bytes(), 0644); err != nil { + return errors.WithStackTrace(err) + } + return nil +} + +func updateLockfile(ctx context.Context, file *hclwrite.File, providers Providers) error { + sort.Slice(providers, func(i, j int) bool { + return providers[i].Address() < providers[j].Address() + }) + + for _, provider := range providers { + providerBlock := file.Body().FirstMatchingBlock("provider", []string{provider.Address()}) + if providerBlock != nil { + // update the existing provider block + err := updateProviderBlock(ctx, providerBlock, provider) + if err != nil { + return err + } + } else { + // create a new provider block + file.Body().AppendNewline() + providerBlock = file.Body().AppendNewBlock("provider", []string{provider.Address()}) + + err := updateProviderBlock(ctx, providerBlock, provider) + if err != nil { + return err + } + } + } + + return nil +} + +// updateProviderBlock updates the provider block in the dependency lock file. +func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, provider Provider) error { + versionAttr := providerBlock.Body().GetAttribute("version") + if versionAttr != nil { + // a version attribute found + versionVal := getAttributeValueAsUnquotedString(versionAttr) + log.Debugf("Check provider version in lock file: address = %s, lock = %s, config = %s", provider.Address(), versionVal, provider.Version()) + if versionVal == provider.Version() { + // Avoid unnecessary recalculations if no version change + return nil + } + } + + providerBlock.Body().SetAttributeValue("version", cty.StringVal(provider.Version())) + + // Constraints can contain multiple constraint expressions, including comparison operators, but in the Terragrunt Provider Cache use case, we assume that the required_providers are pinned to a specific version to detect the required version without terraform init, so we can simply specify the constraints attribute as the same as the version. This may differ from what terraform generates, but we expect that it doesn't matter in practice. + providerBlock.Body().SetAttributeValue("constraints", cty.StringVal(provider.Version())) + + documentSHA256Sums, err := provider.DocumentSHA256Sums(ctx) + if err != nil { + return err + } + + h1Hash, err := PackageHashV1(provider.PackageDir()) + if err != nil { + return err + } + zipHashes := DocumentHashes(documentSHA256Sums) + + hashes := append(zipHashes, h1Hash) + slices.Sort(hashes) + + providerBlock.Body().SetAttributeRaw("hashes", tokensForListPerLine(hashes)) + return nil +} diff --git a/terraform/getproviders/lock_test.go b/terraform/getproviders/lock_test.go new file mode 100644 index 000000000..c88efe5f0 --- /dev/null +++ b/terraform/getproviders/lock_test.go @@ -0,0 +1,161 @@ +package getproviders + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/gruntwork-io/terragrunt/terraform/getproviders/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func mockProviderUpdateLock(t *testing.T, address, version string) Provider { + packageDir, err := os.MkdirTemp("", "") + require.NoError(t, err) + file, err := os.Create(filepath.Join(packageDir, fmt.Sprintf("terraform-provider-v%s", version))) + require.NoError(t, err) + _, err = file.WriteString(fmt.Sprintf("mock-provider-content-%s-%s", address, version)) + require.NoError(t, err) + err = file.Close() + require.NoError(t, err) + + var document string + + for i := 0; i < 2; i++ { + packageName := fmt.Sprintf("%s-%s-%d", address, version, i) + hasher := sha256.New() + _, err := hasher.Write([]byte(packageName)) + require.NoError(t, err) + sha := hex.EncodeToString(hasher.Sum(nil)) + document += fmt.Sprintf("%s %s\n", sha, packageName) + } + + provider := new(mocks.Provider) + provider.On("Address").Return(address) + provider.On("Version").Return(version) + provider.On("PackageDir").Return(packageDir) + provider.On("DocumentSHA256Sums", mock.Anything).Return([]byte(document), nil) + + return provider +} + +func TestUpdateLockfile(t *testing.T) { + t.Parallel() + + testCases := []struct { + providers Providers + initialLockfile string + expectedLockfile string + }{ + { + Providers{ + mockProviderUpdateLock(t, "registry.terraform.io/hashicorp/aws", "5.37.0"), + }, + ``, + ` +provider "registry.terraform.io/hashicorp/aws" { + version = "5.37.0" + constraints = "5.37.0" + hashes = [ + "h1:SHOEBOHEif46z7Bb86YZ5evCtAeK5A4gtHdT8RU5OhA=", + "zh:7c810fb11d8b3ded0cb554a27c27a9d002cc644a7a57c29cae01eeea890f0398", + "zh:a3366f6b57b0f4b8bf8a5fecf42a834652709a97dd6db1b499c4ab186e33a41f", + ] +} +`, + }, + { + Providers{ + mockProviderUpdateLock(t, "registry.terraform.io/hashicorp/aws", "5.36.0"), + mockProviderUpdateLock(t, "registry.terraform.io/hashicorp/template", "2.2.0"), + }, + ` +provider "registry.terraform.io/hashicorp/aws" { + version = "5.37.0" + constraints = "5.37.0" + hashes = [ + "h1:SHOEBOHEif46z7Bb86YZ5evCtAeK5A4gtHdT8RU5OhA=", + "zh:7c810fb11d8b3ded0cb554a27c27a9d002cc644a7a57c29cae01eeea890f0398", + "zh:a3366f6b57b0f4b8bf8a5fecf42a834652709a97dd6db1b499c4ab186e33a41f", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.101.0" + constraints = "3.101.0" + hashes = [ + "h1:Jrkhx+qKaf63sIV/WvE8sPR53QuC16pvTrBjxFVMPYM=", + "zh:38b02bce5cbe83f938a71716bbf9e8b07fed8b2c6b83c19b5e708eda7dee0f1d", + "zh:3ed094366ab35c4fcd632471a7e45a84ca6c72b00477cdf1276e541a0171b369", + ] +} +`, + ` +provider "registry.terraform.io/hashicorp/aws" { + version = "5.36.0" + constraints = "5.36.0" + hashes = [ + "h1:RpTjHdEAYqidB9hFPs68dIhkeIE1c/ZH9fEBdddf0Ik=", + "zh:8721239b83a06212fb2f474d2acddfa2659a224ef66c77e28e1efe2290a30fa7", + "zh:ed83a9620eab99e091b9f786f20f03fddb50cba030839fe0529bd518bfd67f8d", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.101.0" + constraints = "3.101.0" + hashes = [ + "h1:Jrkhx+qKaf63sIV/WvE8sPR53QuC16pvTrBjxFVMPYM=", + "zh:38b02bce5cbe83f938a71716bbf9e8b07fed8b2c6b83c19b5e708eda7dee0f1d", + "zh:3ed094366ab35c4fcd632471a7e45a84ca6c72b00477cdf1276e541a0171b369", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + constraints = "2.2.0" + hashes = [ + "h1:kvJsWhTmFya0WW8jAfY40fDtYhWQ6mOwPQC2ncDNjZs=", + "zh:02d170f0a0f453155686baf35c10b5a7a230ef20ca49f6e26de1c2691ac70a59", + "zh:d88ec10849d5a1d9d1db458847bbc62049f0282a2139e5176d645b75a0346992", + ] +} +`, + }, + } + + for i, testCase := range testCases { + testCase := testCase + + t.Run(fmt.Sprintf("testCase-%d", i), func(t *testing.T) { + t.Parallel() + + workingDir, err := os.MkdirTemp("", "") + require.NoError(t, err) + lockfilePath := filepath.Join(workingDir, ".terraform.lock.hcl") + + if testCase.initialLockfile != "" { + file, err := os.Create(lockfilePath) + require.NoError(t, err) + _, err = file.WriteString(testCase.initialLockfile) + require.NoError(t, err) + err = file.Close() + require.NoError(t, err) + } + + err = UpdateLockfile(context.Background(), workingDir, testCase.providers) + require.NoError(t, err) + + actualLockfile, err := os.ReadFile(lockfilePath) + require.NoError(t, err) + + assert.Equal(t, testCase.expectedLockfile, string(actualLockfile)) + }) + } +} diff --git a/terraform/getproviders/package.go b/terraform/getproviders/package.go new file mode 100644 index 000000000..0e9643aee --- /dev/null +++ b/terraform/getproviders/package.go @@ -0,0 +1,70 @@ +package getproviders + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/gruntwork-io/go-commons/errors" + "github.com/gruntwork-io/terragrunt/util" +) + +// SigningKey represents a key used to sign packages from a registry, along with an optional trust signature from the registry operator. These are both in ASCII armored OpenPGP format. +type SigningKey struct { + ASCIIArmor string `json:"ascii_armor"` + TrustSignature string `json:"trust_signature"` +} + +type SigningKeyList struct { + GPGPublicKeys []*SigningKey `json:"gpg_public_keys"` +} + +// Package represents the details of the Terraform provider. +type Package struct { + Protocols []string `json:"protocols"` + Filename string `json:"filename"` + OS string `json:"os"` + Arch string `json:"arch"` + + DownloadURL string `json:"download_url"` + SHA256SumsURL string `json:"shasums_url"` + SHA256SumsSignatureURL string `json:"shasums_signature_url"` + + SHA256Sum string `json:"shasum"` + SigningKeys SigningKeyList `json:"signing_keys"` +} + +func (provider *Package) Checksum() ([sha256.Size]byte, error) { + var checksum [sha256.Size]byte + + if _, err := hex.Decode(checksum[:], []byte(provider.SHA256Sum)); err != nil { + return checksum, errors.Errorf("registry response includes invalid SHA256 hash %q: %w", provider.SHA256Sum, err) + } + return checksum, nil +} + +func (provider *Package) FetchSignature(ctx context.Context) ([]byte, error) { + var signature = new(bytes.Buffer) + + if err := util.Fetch(ctx, provider.SHA256SumsSignatureURL, signature); err != nil { + return nil, fmt.Errorf("failed to retrieve authentication checksums: %w", err) + } + + return signature.Bytes(), nil +} + +func (provider *Package) FetchSHA256Sums(ctx context.Context) ([]byte, error) { + var document = new(bytes.Buffer) + + if err := util.Fetch(ctx, provider.SHA256SumsURL, document); err != nil { + return nil, fmt.Errorf("failed to retrieve authentication checksums: %w", err) + } + + return document.Bytes(), nil +} + +func (provider *Package) FetchArchive(ctx context.Context, saveTo string) error { + return util.FetchToFile(ctx, provider.DownloadURL, saveTo) +} diff --git a/terraform/provider/package_authentication.go b/terraform/getproviders/package_authentication.go similarity index 90% rename from terraform/provider/package_authentication.go rename to terraform/getproviders/package_authentication.go index 14e54d9d9..ee0e9e49b 100644 --- a/terraform/provider/package_authentication.go +++ b/terraform/getproviders/package_authentication.go @@ -1,7 +1,6 @@ -package provider +package getproviders import ( - "bufio" "bytes" "crypto/sha256" "encoding/hex" @@ -60,12 +59,6 @@ func (result PackageAuthenticationResult) ThirdPartySigned() bool { return result == partnerProvider || result == communityProvider } -// SigningKey represents a key used to sign packages from a registry, along with an optional trust signature from the registry operator. These are both in ASCII armored OpenPGP format. -type SigningKey struct { - ASCIIArmor string `json:"ascii_armor"` - TrustSignature string `json:"trust_signature"` -} - // PackageAuthentication implementation is responsible for authenticating that a package is what its distributor intended to distribute and that it has not been tampered with. type PackageAuthentication interface { // Authenticate takes the path of a package and returns a PackageAuthenticationResult, or an error if the authentication checks fail. @@ -265,32 +258,7 @@ func (auth signatureAuthentication) checkDetachedSignature(keyring openpgp.KeyRi } func (auth signatureAuthentication) AcceptableHashes() []Hash { - var hashes []Hash - - sc := bufio.NewScanner(bytes.NewReader(auth.Document)) - for sc.Scan() { - parts := bytes.Fields(sc.Bytes()) - if len(parts) != 0 && len(parts) < 2 { - // Doesn't look like a valid sums file line, so we'll assume this whole thing isn't a checksums file. - return nil - } - - // If this is a checksums file then the first part should be a hex-encoded SHA256 hash, so it should be 64 characters long and contain only hex digits. - hashStr := parts[0] - hashLen := 64 - if len(hashStr) != hashLen { - return nil // doesn't look like a checksums file - } - - var gotSHA256Sum [sha256.Size]byte - if _, err := hex.Decode(gotSHA256Sum[:], hashStr); err != nil { - return nil // doesn't look like a checksums file - } - - hashes = append(hashes, HashLegacyZipSHAFromSHA(gotSHA256Sum)) - } - - return hashes + return DocumentHashes(auth.Document) } // findSigningKey attempts to verify the signature using each of the keys returned by the registry. If a valid signature is found, it returns the signing key. diff --git a/terraform/provider/package_authentication_test.go b/terraform/getproviders/package_authentication_test.go similarity index 99% rename from terraform/provider/package_authentication_test.go rename to terraform/getproviders/package_authentication_test.go index b33d4e6ff..eb3c506c4 100644 --- a/terraform/provider/package_authentication_test.go +++ b/terraform/getproviders/package_authentication_test.go @@ -1,4 +1,4 @@ -package provider +package getproviders import ( "crypto/sha256" diff --git a/terraform/getproviders/provider.go b/terraform/getproviders/provider.go new file mode 100644 index 000000000..4e4fdde1e --- /dev/null +++ b/terraform/getproviders/provider.go @@ -0,0 +1,23 @@ +//go:generate mockery --name Provider + +package getproviders + +import ( + "context" +) + +type Providers []Provider + +type Provider interface { + // Address returns a source address of the provider. e.g.: registry.terraform.io/hashicorp/aws + Address() string + + // Version returns a version of the provider. e.g.: 5.36.0 + Version() string + + // DocumentSHA256Sums returns a document with providers hashes for different platforms. + DocumentSHA256Sums(ctx context.Context) ([]byte, error) + + // PackageDir returns a directory with the unpacked provider. + PackageDir() string +} diff --git a/terraform/provider/public_keys.go b/terraform/getproviders/public_keys.go similarity index 99% rename from terraform/provider/public_keys.go rename to terraform/getproviders/public_keys.go index 404f31bde..742656454 100644 --- a/terraform/provider/public_keys.go +++ b/terraform/getproviders/public_keys.go @@ -1,4 +1,4 @@ -package provider +package getproviders // HashicorpPublicKey is the HashiCorp public key, also available at // https://www.hashicorp.com/security diff --git a/terraform/provider/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip b/terraform/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip similarity index 100% rename from terraform/provider/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip rename to terraform/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip diff --git a/terraform/provider/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid.zip b/terraform/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid.zip similarity index 100% rename from terraform/provider/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid.zip rename to terraform/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid.zip diff --git a/terraform/provider/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/extra-data.txt b/terraform/getproviders/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/extra-data.txt similarity index 100% rename from terraform/provider/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/extra-data.txt rename to terraform/getproviders/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/extra-data.txt diff --git a/terraform/provider/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/terraform-provider-happycloud b/terraform/getproviders/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/terraform-provider-happycloud similarity index 100% rename from terraform/provider/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/terraform-provider-happycloud rename to terraform/getproviders/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/terraform-provider-happycloud diff --git a/terraform/provider/hash.go b/terraform/provider/hash.go deleted file mode 100644 index b4ea3c269..000000000 --- a/terraform/provider/hash.go +++ /dev/null @@ -1,58 +0,0 @@ -package provider - -import ( - "crypto/sha256" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/gruntwork-io/go-commons/errors" -) - -// Hash is a specially-formatted string representing a checksum of a package or the contents of the package. -type Hash string - -func (hash Hash) String() string { - return string(hash) -} - -// HashScheme is an enumeration of schemes. -type HashScheme string - -const ( - // HashSchemeZip is the scheme identifier for the legacy hash scheme that applies to distribution archives (.zip files) rather than package contents. - HashSchemeZip HashScheme = HashScheme("zh:") -) - -// New creates a new Hash value with the receiver as its scheme and the given raw string as its value. -func (scheme HashScheme) New(value string) Hash { - return Hash(string(scheme) + value) -} - -// PackageHashLegacyZipSHA implements the old provider package hashing scheme of taking a SHA256 hash of the containing .zip archive itself, rather than of the contents of the archive. -func PackageHashLegacyZipSHA(path string) (Hash, error) { - archivePath, err := filepath.EvalSymlinks(path) - if err != nil { - return "", errors.WithStackTrace(err) - } - - file, err := os.Open(archivePath) - if err != nil { - return "", errors.WithStackTrace(err) - } - defer file.Close() - - hash := sha256.New() - if _, err = io.Copy(hash, file); err != nil { - return "", errors.WithStackTrace(err) - } - - gotHash := hash.Sum(nil) - return HashSchemeZip.New(fmt.Sprintf("%x", gotHash)), nil -} - -// HashLegacyZipSHAFromSHA is a convenience method to produce the schemed-string hash format from an already-calculated hash of a provider .zip archive. -func HashLegacyZipSHAFromSHA(sum [sha256.Size]byte) Hash { - return HashSchemeZip.New(fmt.Sprintf("%x", sum[:])) -} diff --git a/test/integration_test.go b/test/integration_test.go index ad1257318..ef15a13fc 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -33,6 +33,8 @@ import ( terraws "github.com/gruntwork-io/terratest/modules/aws" "github.com/gruntwork-io/terratest/modules/git" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclwrite" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/api/iterator" @@ -242,7 +244,7 @@ func TestTerragruntProviderCache(t *testing.T) { subDir = filepath.Join(rootPath, subDir) entries, err := os.ReadDir(subDir) - assert.NoError(t, err) + require.NoError(t, err) for _, entry := range entries { if !entry.IsDir() { @@ -250,6 +252,15 @@ func TestTerragruntProviderCache(t *testing.T) { } actualApps++ + appPath := filepath.Join(subDir, entry.Name()) + + lockfilePath := filepath.Join(appPath, ".terraform.lock.hcl") + lockfileContent, err := os.ReadFile(lockfilePath) + require.NoError(t, err) + + lockfile, diags := hclwrite.ParseConfig(lockfileContent, lockfilePath, hcl.Pos{Line: 1, Column: 1}) + require.False(t, diags.HasErrors()) + for _, provider := range providers { var ( actualProviderSymlinks int @@ -257,7 +268,10 @@ func TestTerragruntProviderCache(t *testing.T) { provider = path.Join(registryName, provider) ) - providerPath := filepath.Join(subDir, entry.Name(), ".terraform/providers", provider) + providerBlock := lockfile.Body().FirstMatchingBlock("provider", []string{filepath.Dir(provider)}) + assert.NotNil(t, providerBlock) + + providerPath := filepath.Join(appPath, ".terraform/providers", provider) assert.True(t, util.FileExists(providerPath)) entries, err := os.ReadDir(providerPath) @@ -7079,7 +7093,7 @@ func wrappedBinary() string { } return TERRAFORM_BINARY } - return value + return filepath.Base(value) } func isTerraform() bool {