diff --git a/.golangci.yml b/.golangci.yml index e4222ff797..8b1c10a1d1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -67,6 +67,7 @@ linters: - "!**/pkg/auth/types/aws_credentials.go" - "!**/pkg/auth/types/github_oidc_credentials.go" - "!**/internal/aws_utils/**" + - "!**/pkg/provisioner/backend/**" - "$test" deny: # AWS: Identity and auth-related SDKs diff --git a/CLAUDE.md b/CLAUDE.md index fece936d9a..bfd0daf6ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -538,7 +538,15 @@ All cmds/flags need Docusaurus docs in `website/docs/cli/commands/`. Use `
` **Common mistakes:** Using command name vs. filename, not checking slug frontmatter, guessing URLs. ### Documentation Requirements (MANDATORY) -Use `
` for arguments/flags. Follow Docusaurus conventions: frontmatter, purpose note, screengrab, usage/examples/arguments/flags sections. File location: `website/docs/cli/commands//.mdx` +CLI command docs MUST include: +1. **Frontmatter** - title, sidebar_label, sidebar_class_name, id, description +2. **Intro component** - `import Intro from '@site/src/components/Intro'` then `Brief description` +3. **Screengrab** - `import Screengrab from '@site/src/components/Screengrab'` then `` +4. **Usage section** - Shell code block with command syntax +5. **Arguments/Flags** - Use `
` for each argument/flag with `
` description +6. **Examples section** - Practical usage examples + +File location: `website/docs/cli/commands//.mdx` ### Website Build (MANDATORY) ALWAYS build after doc changes: `cd website && npm run build`. Verify: no broken links, missing images, MDX component rendering. diff --git a/cmd/root.go b/cmd/root.go index 3b1c656dca..10b42802d1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -52,6 +52,7 @@ import ( "github.com/cloudposse/atmos/cmd/internal" _ "github.com/cloudposse/atmos/cmd/list" _ "github.com/cloudposse/atmos/cmd/profile" + "github.com/cloudposse/atmos/cmd/terraform/backend" themeCmd "github.com/cloudposse/atmos/cmd/theme" "github.com/cloudposse/atmos/cmd/version" _ "github.com/cloudposse/atmos/cmd/workflow" @@ -1223,6 +1224,7 @@ func Execute() error { version.SetAtmosConfig(&atmosConfig) devcontainer.SetAtmosConfig(&atmosConfig) themeCmd.SetAtmosConfig(&atmosConfig) + backend.SetAtmosConfig(&atmosConfig) if initErr != nil { // Handle config initialization errors based on command context. diff --git a/cmd/terraform/backend/backend.go b/cmd/terraform/backend/backend.go new file mode 100644 index 0000000000..0cc2502fea --- /dev/null +++ b/cmd/terraform/backend/backend.go @@ -0,0 +1,37 @@ +package backend + +import ( + "github.com/spf13/cobra" + + "github.com/cloudposse/atmos/pkg/schema" +) + +// AtmosConfigPtr will be set by SetAtmosConfig before command execution. +var atmosConfigPtr *schema.AtmosConfiguration + +// SetAtmosConfig sets the Atmos configuration for the backend command. +// This is called from root.go after atmosConfig is initialized. +func SetAtmosConfig(config *schema.AtmosConfiguration) { + atmosConfigPtr = config +} + +// backendCmd represents the backend command. +var backendCmd = &cobra.Command{ + Use: "backend", + Short: "Manage Terraform state backends", + Long: `Create, list, describe, update, and delete Terraform state backends.`, +} + +func init() { + // Add CRUD subcommands + backendCmd.AddCommand(createCmd) + backendCmd.AddCommand(listCmd) + backendCmd.AddCommand(describeCmd) + backendCmd.AddCommand(updateCmd) + backendCmd.AddCommand(deleteCmd) +} + +// GetBackendCommand returns the backend command for parent registration. +func GetBackendCommand() *cobra.Command { + return backendCmd +} diff --git a/cmd/terraform/backend/backend_commands_test.go b/cmd/terraform/backend/backend_commands_test.go new file mode 100644 index 0000000000..fedfaccd40 --- /dev/null +++ b/cmd/terraform/backend/backend_commands_test.go @@ -0,0 +1,612 @@ +package backend + +import ( + "errors" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/flags" + "github.com/cloudposse/atmos/pkg/schema" +) + +// setupTestWithMocks creates gomock controller, mock dependencies, and cleanup. +func setupTestWithMocks(t *testing.T) (*MockConfigInitializer, *MockProvisioner) { + t.Helper() + + ctrl := gomock.NewController(t) + + mockConfigInit := NewMockConfigInitializer(ctrl) + mockProv := NewMockProvisioner(ctrl) + + // Inject mocks. + SetConfigInitializer(mockConfigInit) + SetProvisioner(mockProv) + + // Register cleanup. + t.Cleanup(func() { + ResetDependencies() + ctrl.Finish() + }) + + return mockConfigInit, mockProv +} + +// setupViperForTest resets Viper and sets test values. +func setupViperForTest(t *testing.T, values map[string]any) { + t.Helper() + + // Save current state. + oldViper := viper.GetViper() + oldKeys := make(map[string]any) + for _, key := range oldViper.AllKeys() { + oldKeys[key] = oldViper.Get(key) + } + + // Reset and set new values. + viper.Reset() + for k, v := range values { + viper.Set(k, v) + } + + // Register cleanup to restore. + t.Cleanup(func() { + viper.Reset() + for key, val := range oldKeys { + viper.Set(key, val) + } + }) +} + +// TestExecuteProvisionCommand tests the shared provision command implementation. +func TestExecuteProvisionCommand(t *testing.T) { + tests := []struct { + name string + args []string + viperValues map[string]any + setupMocks func(*MockConfigInitializer, *MockProvisioner) + expectError bool + expectedError error + }{ + { + name: "successful provision", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(&schema.AtmosConfiguration{}, nil, nil) + mp.EXPECT(). + CreateBackend(gomock.Any()). + Return(nil) + }, + expectError: false, + }, + { + name: "missing stack flag", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "", + "identity": "", + }, + setupMocks: func(*MockConfigInitializer, *MockProvisioner) {}, + expectError: true, + expectedError: errUtils.ErrRequiredFlagNotProvided, + }, + { + name: "config init failure", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(nil, nil, errors.New("config init failed")) + }, + expectError: true, + }, + { + name: "provision failure", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(&schema.AtmosConfiguration{}, nil, nil) + mp.EXPECT(). + CreateBackend(gomock.Any()). + Return(errors.New("provision failed")) + }, + expectError: true, + }, + { + name: "with auth context", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "prod", + "identity": "aws-prod", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "prod", "aws-prod"). + Return(&schema.AtmosConfiguration{}, &schema.AuthContext{AWS: &schema.AWSAuthContext{}}, nil) + mp.EXPECT(). + CreateBackend(gomock.Any()). + Return(nil) + }, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockConfigInit, mockProv := setupTestWithMocks(t) + setupViperForTest(t, tt.viperValues) + tt.setupMocks(mockConfigInit, mockProv) + + cmd := &cobra.Command{Use: "test"} + parser := flags.NewStandardParser( + flags.WithStackFlag(), + flags.WithIdentityFlag(), + ) + parser.RegisterFlags(cmd) + require.NoError(t, parser.BindToViper(viper.GetViper())) + + err := ExecuteProvisionCommand(cmd, tt.args, parser, "test.RunE") + + if tt.expectError { + assert.Error(t, err) + if tt.expectedError != nil { + assert.ErrorIs(t, err, tt.expectedError) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestDeleteCmd_RunE tests the delete command RunE function. +func TestDeleteCmd_RunE(t *testing.T) { + tests := []struct { + name string + args []string + viperValues map[string]any + setupMocks func(*MockConfigInitializer, *MockProvisioner) + expectError bool + expectedError error + }{ + { + name: "successful delete with force", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "force": true, + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(&schema.AtmosConfiguration{}, nil, nil) + mp.EXPECT(). + DeleteBackend(gomock.Any()). + Return(nil) + }, + expectError: false, + }, + { + name: "delete without force flag", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "force": false, + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(&schema.AtmosConfiguration{}, nil, nil) + mp.EXPECT(). + DeleteBackend(gomock.Any()). + Return(nil) + }, + expectError: false, + }, + { + name: "missing stack flag", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "", + "identity": "", + "force": true, + }, + setupMocks: func(*MockConfigInitializer, *MockProvisioner) {}, + expectError: true, + expectedError: errUtils.ErrRequiredFlagNotProvided, + }, + { + name: "config init failure", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "force": true, + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(nil, nil, errors.New("config init failed")) + }, + expectError: true, + }, + { + name: "delete backend failure", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "force": true, + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(&schema.AtmosConfiguration{}, nil, nil) + mp.EXPECT(). + DeleteBackend(gomock.Any()). + Return(errors.New("delete failed")) + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockConfigInit, mockProv := setupTestWithMocks(t) + setupViperForTest(t, tt.viperValues) + tt.setupMocks(mockConfigInit, mockProv) + + err := deleteCmd.RunE(deleteCmd, tt.args) + + if tt.expectError { + assert.Error(t, err) + if tt.expectedError != nil { + assert.ErrorIs(t, err, tt.expectedError) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestDescribeCmd_RunE tests the describe command RunE function. +func TestDescribeCmd_RunE(t *testing.T) { + tests := []struct { + name string + args []string + viperValues map[string]any + setupMocks func(*MockConfigInitializer, *MockProvisioner) + expectError bool + expectedError error + }{ + { + name: "successful describe with yaml format", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "yaml", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + atmosConfig := &schema.AtmosConfiguration{} + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(atmosConfig, nil, nil) + mp.EXPECT(). + DescribeBackend(atmosConfig, "vpc", map[string]string{"format": "yaml"}). + Return(nil) + }, + expectError: false, + }, + { + name: "successful describe with json format", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "json", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + atmosConfig := &schema.AtmosConfiguration{} + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(atmosConfig, nil, nil) + mp.EXPECT(). + DescribeBackend(atmosConfig, "vpc", map[string]string{"format": "json"}). + Return(nil) + }, + expectError: false, + }, + { + name: "missing stack flag", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "", + "identity": "", + "format": "yaml", + }, + setupMocks: func(*MockConfigInitializer, *MockProvisioner) {}, + expectError: true, + expectedError: errUtils.ErrRequiredFlagNotProvided, + }, + { + name: "config init failure", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "yaml", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(nil, nil, errors.New("config init failed")) + }, + expectError: true, + }, + { + name: "describe backend failure", + args: []string{"vpc"}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "yaml", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + atmosConfig := &schema.AtmosConfiguration{} + mci.EXPECT(). + InitConfigAndAuth("vpc", "dev", ""). + Return(atmosConfig, nil, nil) + mp.EXPECT(). + DescribeBackend(atmosConfig, "vpc", map[string]string{"format": "yaml"}). + Return(errors.New("describe failed")) + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockConfigInit, mockProv := setupTestWithMocks(t) + setupViperForTest(t, tt.viperValues) + tt.setupMocks(mockConfigInit, mockProv) + + err := describeCmd.RunE(describeCmd, tt.args) + + if tt.expectError { + assert.Error(t, err) + if tt.expectedError != nil { + assert.ErrorIs(t, err, tt.expectedError) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestListCmd_RunE tests the list command RunE function. +func TestListCmd_RunE(t *testing.T) { + tests := []struct { + name string + args []string + viperValues map[string]any + setupMocks func(*MockConfigInitializer, *MockProvisioner) + expectError bool + expectedError error + }{ + { + name: "successful list with table format", + args: []string{}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "table", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + atmosConfig := &schema.AtmosConfiguration{} + mci.EXPECT(). + InitConfigAndAuth("", "dev", ""). + Return(atmosConfig, nil, nil) + mp.EXPECT(). + ListBackends(atmosConfig, map[string]string{"format": "table"}). + Return(nil) + }, + expectError: false, + }, + { + name: "successful list with json format", + args: []string{}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "json", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + atmosConfig := &schema.AtmosConfiguration{} + mci.EXPECT(). + InitConfigAndAuth("", "dev", ""). + Return(atmosConfig, nil, nil) + mp.EXPECT(). + ListBackends(atmosConfig, map[string]string{"format": "json"}). + Return(nil) + }, + expectError: false, + }, + { + name: "missing stack flag", + args: []string{}, + viperValues: map[string]any{ + "stack": "", + "identity": "", + "format": "table", + }, + setupMocks: func(*MockConfigInitializer, *MockProvisioner) {}, + expectError: true, + expectedError: errUtils.ErrRequiredFlagNotProvided, + }, + { + name: "config init failure", + args: []string{}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "table", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + mci.EXPECT(). + InitConfigAndAuth("", "dev", ""). + Return(nil, nil, errors.New("config init failed")) + }, + expectError: true, + }, + { + name: "list backends failure", + args: []string{}, + viperValues: map[string]any{ + "stack": "dev", + "identity": "", + "format": "table", + }, + setupMocks: func(mci *MockConfigInitializer, mp *MockProvisioner) { + atmosConfig := &schema.AtmosConfiguration{} + mci.EXPECT(). + InitConfigAndAuth("", "dev", ""). + Return(atmosConfig, nil, nil) + mp.EXPECT(). + ListBackends(atmosConfig, map[string]string{"format": "table"}). + Return(errors.New("list failed")) + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockConfigInit, mockProv := setupTestWithMocks(t) + setupViperForTest(t, tt.viperValues) + tt.setupMocks(mockConfigInit, mockProv) + + err := listCmd.RunE(listCmd, tt.args) + + if tt.expectError { + assert.Error(t, err) + if tt.expectedError != nil { + assert.ErrorIs(t, err, tt.expectedError) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestSetConfigInitializer tests the SetConfigInitializer function. +func TestSetConfigInitializer(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mock := NewMockConfigInitializer(ctrl) + SetConfigInitializer(mock) + + t.Cleanup(func() { + ResetDependencies() + }) + + // Verify the mock was set. + assert.Equal(t, mock, configInit) +} + +// TestSetProvisioner tests the SetProvisioner function. +func TestSetProvisioner(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mock := NewMockProvisioner(ctrl) + SetProvisioner(mock) + + t.Cleanup(func() { + ResetDependencies() + }) + + // Verify the mock was set. + assert.Equal(t, mock, prov) +} + +// TestResetDependencies tests the ResetDependencies function. +func TestResetDependencies(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Set mocks. + mockConfigInit := NewMockConfigInitializer(ctrl) + mockProv := NewMockProvisioner(ctrl) + SetConfigInitializer(mockConfigInit) + SetProvisioner(mockProv) + + // Reset. + ResetDependencies() + + // Verify defaults are restored (type assertion). + _, isDefaultConfigInit := configInit.(*defaultConfigInitializer) + assert.True(t, isDefaultConfigInit, "configInit should be reset to defaultConfigInitializer") + + _, isDefaultProv := prov.(*defaultProvisioner) + assert.True(t, isDefaultProv, "prov should be reset to defaultProvisioner") +} + +// TestCreateDescribeComponentFunc_ReturnsNonNil tests that CreateDescribeComponentFunc returns a non-nil function. +func TestCreateDescribeComponentFunc_ReturnsNonNil(t *testing.T) { + // Test that the function is created correctly with nil authManager. + describeFunc := CreateDescribeComponentFunc(nil) + assert.NotNil(t, describeFunc, "describeFunc should not be nil") + + // The function is created but we can't easily test execution + // without real config - the important thing is it doesn't panic. +} + +// TestParseCommonFlags_Success tests successful parsing in ParseCommonFlags. +func TestParseCommonFlags_Success(t *testing.T) { + // Test successful parsing with all flags. + setupViperForTest(t, map[string]any{ + "stack": "test-stack", + "identity": "test-identity", + }) + + cmd := &cobra.Command{Use: "test"} + parser := flags.NewStandardParser( + flags.WithStackFlag(), + flags.WithIdentityFlag(), + ) + parser.RegisterFlags(cmd) + require.NoError(t, parser.BindToViper(viper.GetViper())) + + opts, err := ParseCommonFlags(cmd, parser) + assert.NoError(t, err) + assert.NotNil(t, opts) + assert.Equal(t, "test-stack", opts.Stack) + assert.Equal(t, "test-identity", opts.Identity) +} diff --git a/cmd/terraform/backend/backend_create.go b/cmd/terraform/backend/backend_create.go new file mode 100644 index 0000000000..7fde1ff43a --- /dev/null +++ b/cmd/terraform/backend/backend_create.go @@ -0,0 +1,36 @@ +package backend + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cloudposse/atmos/pkg/flags" +) + +var createParser *flags.StandardParser + +var createCmd = &cobra.Command{ + Use: "", + Short: "Provision backend infrastructure", + Long: `Create or update S3 backend with secure defaults (versioning, encryption, public access blocking). This operation is idempotent.`, + Example: ` atmos terraform backend create vpc --stack dev`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ExecuteProvisionCommand(cmd, args, createParser, "backend.create.RunE") + }, +} + +func init() { + createCmd.DisableFlagParsing = false + + createParser = flags.NewStandardParser( + flags.WithStackFlag(), + flags.WithIdentityFlag(), + ) + + createParser.RegisterFlags(createCmd) + + if err := createParser.BindToViper(viper.GetViper()); err != nil { + panic(err) + } +} diff --git a/cmd/terraform/backend/backend_create_test.go b/cmd/terraform/backend/backend_create_test.go new file mode 100644 index 0000000000..6392334b92 --- /dev/null +++ b/cmd/terraform/backend/backend_create_test.go @@ -0,0 +1,31 @@ +package backend + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCreateCmd_Structure(t *testing.T) { + testCommandStructure(t, commandTestParams{ + cmd: createCmd, + parser: createParser, + expectedUse: "", + expectedShort: "Provision backend infrastructure", + requiredFlags: []string{}, + }) +} + +func TestCreateCmd_Init(t *testing.T) { + // Verify init() ran successfully by checking parser and flags are set up. + assert.NotNil(t, createParser, "createParser should be initialized") + assert.NotNil(t, createCmd, "createCmd should be initialized") + assert.False(t, createCmd.DisableFlagParsing, "DisableFlagParsing should be false") + + // Verify flags are registered. + stackFlag := createCmd.Flags().Lookup("stack") + assert.NotNil(t, stackFlag, "stack flag should be registered") + + identityFlag := createCmd.Flags().Lookup("identity") + assert.NotNil(t, identityFlag, "identity flag should be registered") +} diff --git a/cmd/terraform/backend/backend_delete.go b/cmd/terraform/backend/backend_delete.go new file mode 100644 index 0000000000..2d256c14b8 --- /dev/null +++ b/cmd/terraform/backend/backend_delete.go @@ -0,0 +1,74 @@ +package backend + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cloudposse/atmos/pkg/flags" + "github.com/cloudposse/atmos/pkg/perf" +) + +var deleteParser *flags.StandardParser + +var deleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Delete backend infrastructure", + Long: `Permanently delete backend infrastructure. + +Requires the --force flag for safety. The backend must be empty +(no state files) before it can be deleted.`, + Example: ` atmos terraform backend delete vpc --stack dev --force`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + defer perf.Track(atmosConfigPtr, "backend.delete.RunE")() + + component := args[0] + + v := viper.GetViper() + opts, err := ParseCommonFlags(cmd, deleteParser) + if err != nil { + return err + } + + force := v.GetBool("force") + + // Initialize config and auth using injected dependency. + atmosConfig, authContext, err := configInit.InitConfigAndAuth(component, opts.Stack, opts.Identity) + if err != nil { + return err + } + + // Create describe component callback. + describeFunc := CreateDescribeComponentFunc(nil) // Auth already handled in InitConfigAndAuth + + // Execute delete command using injected provisioner. + return prov.DeleteBackend(&DeleteBackendParams{ + AtmosConfig: atmosConfig, + Component: component, + Stack: opts.Stack, + Force: force, + DescribeFunc: describeFunc, + AuthContext: authContext, + }) + }, +} + +func init() { + deleteCmd.DisableFlagParsing = false + + // Create parser with functional options. + deleteParser = flags.NewStandardParser( + flags.WithStackFlag(), + flags.WithIdentityFlag(), + flags.WithBoolFlag("force", "", false, "Force deletion without confirmation"), + flags.WithEnvVars("force", "ATMOS_FORCE"), + ) + + // Register flags with the command. + deleteParser.RegisterFlags(deleteCmd) + + // Bind flags to Viper. + if err := deleteParser.BindToViper(viper.GetViper()); err != nil { + panic(err) + } +} diff --git a/cmd/terraform/backend/backend_delete_test.go b/cmd/terraform/backend/backend_delete_test.go new file mode 100644 index 0000000000..d3a35db747 --- /dev/null +++ b/cmd/terraform/backend/backend_delete_test.go @@ -0,0 +1,76 @@ +package backend + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeleteCmd_Structure(t *testing.T) { + testCommandStructure(t, commandTestParams{ + cmd: deleteCmd, + parser: deleteParser, + expectedUse: "delete ", + expectedShort: "Delete backend infrastructure", + requiredFlags: []string{"force"}, + }) + + t.Run("force flag is boolean", func(t *testing.T) { + forceFlag := deleteCmd.Flags().Lookup("force") + assert.NotNil(t, forceFlag, "force flag should be registered") + assert.Equal(t, "bool", forceFlag.Value.Type()) + }) +} + +func TestDeleteCmd_FlagDefaults(t *testing.T) { + tests := []struct { + name string + flagName string + expectedType string + hasDefault bool + }{ + { + name: "force flag is boolean", + flagName: "force", + expectedType: "bool", + hasDefault: true, + }, + { + name: "stack flag is string", + flagName: "stack", + expectedType: "string", + hasDefault: true, + }, + { + name: "identity flag is string", + flagName: "identity", + expectedType: "string", + hasDefault: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + flag := deleteCmd.Flags().Lookup(tt.flagName) + assert.NotNil(t, flag, "flag %s should exist", tt.flagName) + assert.Equal(t, tt.expectedType, flag.Value.Type()) + }) + } +} + +func TestDeleteCmd_Init(t *testing.T) { + // Verify init() ran successfully by checking parser and flags are set up. + assert.NotNil(t, deleteParser, "deleteParser should be initialized") + assert.NotNil(t, deleteCmd, "deleteCmd should be initialized") + assert.False(t, deleteCmd.DisableFlagParsing, "DisableFlagParsing should be false") + + // Verify flags are registered. + stackFlag := deleteCmd.Flags().Lookup("stack") + assert.NotNil(t, stackFlag, "stack flag should be registered") + + identityFlag := deleteCmd.Flags().Lookup("identity") + assert.NotNil(t, identityFlag, "identity flag should be registered") + + forceFlag := deleteCmd.Flags().Lookup("force") + assert.NotNil(t, forceFlag, "force flag should be registered") +} diff --git a/cmd/terraform/backend/backend_describe.go b/cmd/terraform/backend/backend_describe.go new file mode 100644 index 0000000000..f46d903362 --- /dev/null +++ b/cmd/terraform/backend/backend_describe.go @@ -0,0 +1,66 @@ +package backend + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cloudposse/atmos/pkg/flags" + "github.com/cloudposse/atmos/pkg/perf" +) + +var describeParser *flags.StandardParser + +var describeCmd = &cobra.Command{ + Use: "describe ", + Short: "Describe backend configuration", + Long: `Show component's backend configuration from stack. + +Returns the actual stack configuration for the backend, not a schema. +This includes backend settings, variables, and metadata from the stack manifest.`, + Example: ` atmos terraform backend describe vpc --stack dev + atmos terraform backend describe vpc --stack dev --format json`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + defer perf.Track(atmosConfigPtr, "backend.describe.RunE")() + + component := args[0] + + v := viper.GetViper() + opts, err := ParseCommonFlags(cmd, describeParser) + if err != nil { + return err + } + + format := v.GetString("format") + + // Initialize config using injected dependency. + atmosConfig, _, err := configInit.InitConfigAndAuth(component, opts.Stack, opts.Identity) + if err != nil { + return err + } + + // Execute describe command using injected provisioner. + // Pass format in a simple map since opts interface{} accepts anything. + return prov.DescribeBackend(atmosConfig, component, map[string]string{"format": format}) + }, +} + +func init() { + describeCmd.DisableFlagParsing = false + + // Create parser with functional options. + describeParser = flags.NewStandardParser( + flags.WithStackFlag(), + flags.WithIdentityFlag(), + flags.WithStringFlag("format", "f", "yaml", "Output format: yaml, json, table"), + flags.WithEnvVars("format", "ATMOS_FORMAT"), + ) + + // Register flags with the command. + describeParser.RegisterFlags(describeCmd) + + // Bind flags to Viper. + if err := describeParser.BindToViper(viper.GetViper()); err != nil { + panic(err) + } +} diff --git a/cmd/terraform/backend/backend_describe_test.go b/cmd/terraform/backend/backend_describe_test.go new file mode 100644 index 0000000000..83a6b78987 --- /dev/null +++ b/cmd/terraform/backend/backend_describe_test.go @@ -0,0 +1,85 @@ +package backend + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDescribeCmd_Structure(t *testing.T) { + testCommandStructure(t, commandTestParams{ + cmd: describeCmd, + parser: describeParser, + expectedUse: "describe ", + expectedShort: "Describe backend configuration", + requiredFlags: []string{"format"}, + }) + + t.Run("format flag is string", func(t *testing.T) { + formatFlag := describeCmd.Flags().Lookup("format") + assert.NotNil(t, formatFlag, "format flag should be registered") + assert.Equal(t, "string", formatFlag.Value.Type()) + }) +} + +func TestDescribeCmd_FlagDefaults(t *testing.T) { + tests := []struct { + name string + flagName string + expectedType string + expectedValue string + }{ + { + name: "format flag has yaml default", + flagName: "format", + expectedType: "string", + expectedValue: "yaml", + }, + { + name: "stack flag is string", + flagName: "stack", + expectedType: "string", + expectedValue: "", + }, + { + name: "identity flag is string", + flagName: "identity", + expectedType: "string", + expectedValue: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + flag := describeCmd.Flags().Lookup(tt.flagName) + assert.NotNil(t, flag, "flag %s should exist", tt.flagName) + assert.Equal(t, tt.expectedType, flag.Value.Type()) + assert.Equal(t, tt.expectedValue, flag.DefValue) + }) + } +} + +func TestDescribeCmd_Shorthand(t *testing.T) { + t.Run("format flag has shorthand", func(t *testing.T) { + flag := describeCmd.Flags().Lookup("format") + assert.NotNil(t, flag) + assert.Equal(t, "f", flag.Shorthand, "format flag should have 'f' shorthand") + }) +} + +func TestDescribeCmd_Init(t *testing.T) { + // Verify init() ran successfully by checking parser and flags are set up. + assert.NotNil(t, describeParser, "describeParser should be initialized") + assert.NotNil(t, describeCmd, "describeCmd should be initialized") + assert.False(t, describeCmd.DisableFlagParsing, "DisableFlagParsing should be false") + + // Verify flags are registered. + stackFlag := describeCmd.Flags().Lookup("stack") + assert.NotNil(t, stackFlag, "stack flag should be registered") + + identityFlag := describeCmd.Flags().Lookup("identity") + assert.NotNil(t, identityFlag, "identity flag should be registered") + + formatFlag := describeCmd.Flags().Lookup("format") + assert.NotNil(t, formatFlag, "format flag should be registered") +} diff --git a/cmd/terraform/backend/backend_helpers.go b/cmd/terraform/backend/backend_helpers.go new file mode 100644 index 0000000000..d5ee106cb1 --- /dev/null +++ b/cmd/terraform/backend/backend_helpers.go @@ -0,0 +1,260 @@ +package backend + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -source=backend_helpers.go -destination=mock_backend_helpers_test.go -package=backend + +import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + errUtils "github.com/cloudposse/atmos/errors" + e "github.com/cloudposse/atmos/internal/exec" + "github.com/cloudposse/atmos/pkg/auth" + cfg "github.com/cloudposse/atmos/pkg/config" + "github.com/cloudposse/atmos/pkg/flags" + "github.com/cloudposse/atmos/pkg/flags/global" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/provisioner" + "github.com/cloudposse/atmos/pkg/schema" +) + +// ConfigInitializer abstracts configuration and auth initialization for testability. +type ConfigInitializer interface { + InitConfigAndAuth(component, stack, identity string) (*schema.AtmosConfiguration, *schema.AuthContext, error) +} + +// CreateBackendParams contains parameters for CreateBackend operation. +type CreateBackendParams struct { + AtmosConfig *schema.AtmosConfiguration + Component string + Stack string + DescribeFunc func(string, string) (map[string]any, error) + AuthContext *schema.AuthContext +} + +// DeleteBackendParams contains parameters for DeleteBackend operation. +type DeleteBackendParams struct { + AtmosConfig *schema.AtmosConfiguration + Component string + Stack string + Force bool + DescribeFunc func(string, string) (map[string]any, error) + AuthContext *schema.AuthContext +} + +// Provisioner abstracts provisioning operations for testability. +type Provisioner interface { + CreateBackend(params *CreateBackendParams) error + DeleteBackend(params *DeleteBackendParams) error + DescribeBackend(atmosConfig *schema.AtmosConfiguration, component string, opts interface{}) error + ListBackends(atmosConfig *schema.AtmosConfiguration, opts interface{}) error +} + +// defaultConfigInitializer implements ConfigInitializer using production code. +type defaultConfigInitializer struct{} + +func (d *defaultConfigInitializer) InitConfigAndAuth(component, stack, identity string) (*schema.AtmosConfiguration, *schema.AuthContext, error) { + return InitConfigAndAuth(component, stack, identity) +} + +// defaultProvisioner implements Provisioner using production code. +type defaultProvisioner struct{} + +func (d *defaultProvisioner) CreateBackend(params *CreateBackendParams) error { + return provisioner.ProvisionWithParams(&provisioner.ProvisionParams{ + AtmosConfig: params.AtmosConfig, + ProvisionerType: "backend", + Component: params.Component, + Stack: params.Stack, + DescribeComponent: params.DescribeFunc, + AuthContext: params.AuthContext, + }) +} + +func (d *defaultProvisioner) DeleteBackend(params *DeleteBackendParams) error { + return provisioner.DeleteBackendWithParams(&provisioner.DeleteBackendParams{ + AtmosConfig: params.AtmosConfig, + Component: params.Component, + Stack: params.Stack, + Force: params.Force, + DescribeComponent: params.DescribeFunc, + AuthContext: params.AuthContext, + }) +} + +func (d *defaultProvisioner) DescribeBackend(atmosConfig *schema.AtmosConfiguration, component string, opts interface{}) error { + return provisioner.DescribeBackend(atmosConfig, component, opts) +} + +func (d *defaultProvisioner) ListBackends(atmosConfig *schema.AtmosConfiguration, opts interface{}) error { + return provisioner.ListBackends(atmosConfig, opts) +} + +// Package-level dependencies for production use. These can be overridden in tests. +var ( + configInit ConfigInitializer = &defaultConfigInitializer{} + prov Provisioner = &defaultProvisioner{} +) + +// SetConfigInitializer sets the config initializer (for testing). +// If nil is passed, resets to default implementation. +func SetConfigInitializer(ci ConfigInitializer) { + if ci == nil { + configInit = &defaultConfigInitializer{} + return + } + configInit = ci +} + +// SetProvisioner sets the provisioner (for testing). +// If nil is passed, resets to default implementation. +func SetProvisioner(p Provisioner) { + if p == nil { + prov = &defaultProvisioner{} + return + } + prov = p +} + +// ResetDependencies resets dependencies to production defaults (for test cleanup). +func ResetDependencies() { + configInit = &defaultConfigInitializer{} + prov = &defaultProvisioner{} +} + +// CommonOptions contains the standard flags shared by all backend commands. +type CommonOptions struct { + global.Flags + Stack string + Identity string +} + +// ParseCommonFlags parses common flags (stack, identity) using StandardParser with Viper precedence. +func ParseCommonFlags(cmd *cobra.Command, parser *flags.StandardParser) (*CommonOptions, error) { + v := viper.GetViper() + if err := parser.BindFlagsToViper(cmd, v); err != nil { + return nil, err + } + + opts := &CommonOptions{ + Flags: flags.ParseGlobalFlags(cmd, v), + Stack: v.GetString("stack"), + Identity: v.GetString("identity"), + } + + if opts.Stack == "" { + return nil, errUtils.ErrRequiredFlagNotProvided + } + + return opts, nil +} + +// InitConfigAndAuth initializes Atmos configuration and optional authentication. +// Returns atmosConfig, authContext, and error. +// It loads component configuration, merges component-level auth with global auth, +// and creates an AuthContext that respects component's default identity settings. +func InitConfigAndAuth(component, stack, identity string) (*schema.AtmosConfiguration, *schema.AuthContext, error) { + // Load atmos configuration. + atmosConfig, err := cfg.InitCliConfig(schema.ConfigAndStacksInfo{ + ComponentFromArg: component, + Stack: stack, + }, false) + if err != nil { + return nil, nil, errors.Join(errUtils.ErrFailedToInitConfig, err) + } + + // Load component configuration to get component-level auth settings. + componentConfig, err := e.ExecuteDescribeComponent(&e.ExecuteDescribeComponentParams{ + Component: component, + Stack: stack, + ProcessTemplates: false, + ProcessYamlFunctions: false, + Skip: nil, + AuthManager: nil, // Don't need auth to describe component + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to load component config: %w", err) + } + + // Merge component auth with global auth (component auth takes precedence). + mergedAuthConfig, err := auth.MergeComponentAuthFromConfig(&atmosConfig.Auth, componentConfig, &atmosConfig, cfg.AuthSectionName) + if err != nil { + return nil, nil, fmt.Errorf("failed to merge component auth: %w", err) + } + + // Create AuthManager with merged config (auto-selects component's default identity if present). + authManager, err := auth.CreateAndAuthenticateManager(identity, mergedAuthConfig, cfg.IdentityFlagSelectValue) + if err != nil { + return nil, nil, err + } + + // Get AuthContext from AuthManager. + var authContext *schema.AuthContext + if authManager != nil { + stackInfo := authManager.GetStackInfo() + if stackInfo != nil { + authContext = stackInfo.AuthContext + } + } + + return &atmosConfig, authContext, nil +} + +// CreateDescribeComponentFunc creates a describe component function with the given authManager. +func CreateDescribeComponentFunc(authManager auth.AuthManager) func(string, string) (map[string]any, error) { + return func(component, stack string) (map[string]any, error) { + return e.ExecuteDescribeComponent(&e.ExecuteDescribeComponentParams{ + Component: component, + Stack: stack, + ProcessTemplates: false, + ProcessYamlFunctions: false, + Skip: nil, + AuthManager: authManager, + }) + } +} + +// ExecuteProvisionCommand is the shared RunE implementation for create and update commands. +// Both operations are idempotent - they provision or update the backend to match the desired state. +func ExecuteProvisionCommand(cmd *cobra.Command, args []string, parser *flags.StandardParser, perfLabel string) error { + defer perf.Track(atmosConfigPtr, perfLabel)() + + component := args[0] + + // Parse common flags. + opts, err := ParseCommonFlags(cmd, parser) + if err != nil { + return err + } + + // Initialize config and auth using injected dependency. + atmosConfig, authContext, err := configInit.InitConfigAndAuth(component, opts.Stack, opts.Identity) + if err != nil { + return err + } + + // Create describe component callback. + // Note: We don't need to pass authContext to the describe function for backend provisioning + // since we already loaded the component config in InitConfigAndAuth. + describeFunc := func(component, stack string) (map[string]any, error) { + return e.ExecuteDescribeComponent(&e.ExecuteDescribeComponentParams{ + Component: component, + Stack: stack, + ProcessTemplates: false, + ProcessYamlFunctions: false, + Skip: nil, + AuthManager: nil, // Auth already handled. + }) + } + + // Execute provision command using injected provisioner. + return prov.CreateBackend(&CreateBackendParams{ + AtmosConfig: atmosConfig, + Component: component, + Stack: opts.Stack, + DescribeFunc: describeFunc, + AuthContext: authContext, + }) +} diff --git a/cmd/terraform/backend/backend_helpers_test.go b/cmd/terraform/backend/backend_helpers_test.go new file mode 100644 index 0000000000..943377cd5b --- /dev/null +++ b/cmd/terraform/backend/backend_helpers_test.go @@ -0,0 +1,117 @@ +package backend + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/flags" +) + +// TestParseCommonFlags tests the ParseCommonFlags function. +// Note: This test mutates the global Viper instance because ParseCommonFlags +// uses viper.GetViper() internally (standard pattern for CLI flag handling). +// The setupViperForTest helper ensures proper cleanup via t.Cleanup(). +func TestParseCommonFlags(t *testing.T) { + tests := []struct { + name string + stack string + identity string + expectError bool + expectedErr error + }{ + { + name: "valid stack and identity", + stack: "dev", + identity: "test-identity", + expectError: false, + }, + { + name: "valid stack without identity", + stack: "prod", + identity: "", + expectError: false, + }, + { + name: "missing stack", + stack: "", + identity: "test-identity", + expectError: true, + expectedErr: errUtils.ErrRequiredFlagNotProvided, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Use helper to safely set up and restore global Viper state. + setupViperForTest(t, map[string]any{ + "stack": tt.stack, + "identity": tt.identity, + }) + + // Create a test command. + cmd := &cobra.Command{ + Use: "test", + } + + // Create parser with common flags. + parser := flags.NewStandardParser( + flags.WithStringFlag("stack", "s", "", "Stack name"), + flags.WithStringFlag("identity", "i", "", "Identity"), + ) + + // Register flags with command. + parser.RegisterFlags(cmd) + + // Bind to viper. + err := parser.BindToViper(viper.GetViper()) + require.NoError(t, err) + + // Parse common flags. + opts, err := ParseCommonFlags(cmd, parser) + + if tt.expectError { + assert.Error(t, err) + if tt.expectedErr != nil { + assert.ErrorIs(t, err, tt.expectedErr) + } + assert.Nil(t, opts) + } else { + assert.NoError(t, err) + require.NotNil(t, opts) + assert.Equal(t, tt.stack, opts.Stack) + assert.Equal(t, tt.identity, opts.Identity) + } + }) + } +} + +func TestCreateDescribeComponentFunc(t *testing.T) { + t.Run("creates function with nil auth", func(t *testing.T) { + // Create the describe function with nil auth manager. + describeFunc := CreateDescribeComponentFunc(nil) + + // Verify it returns a non-nil function. + assert.NotNil(t, describeFunc) + + // Note: We cannot test the actual execution without mocking ExecuteDescribeComponent. + // This would require significant test infrastructure. + // This test verifies the function creation logic works correctly. + }) +} + +func TestCommonOptions(t *testing.T) { + t.Run("CommonOptions struct initialization", func(t *testing.T) { + opts := &CommonOptions{ + Stack: "test-stack", + Identity: "test-identity", + } + + assert.Equal(t, "test-stack", opts.Stack) + assert.Equal(t, "test-identity", opts.Identity) + }) +} diff --git a/cmd/terraform/backend/backend_list.go b/cmd/terraform/backend/backend_list.go new file mode 100644 index 0000000000..57b6d72990 --- /dev/null +++ b/cmd/terraform/backend/backend_list.go @@ -0,0 +1,59 @@ +package backend + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cloudposse/atmos/pkg/flags" + "github.com/cloudposse/atmos/pkg/perf" +) + +var listParser *flags.StandardParser + +var listCmd = &cobra.Command{ + Use: "list", + Short: "List all backends in stack", + Long: `Show all provisioned backends and their status for a given stack.`, + Example: ` atmos terraform backend list --stack dev`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + defer perf.Track(atmosConfigPtr, "backend.list.RunE")() + + v := viper.GetViper() + opts, err := ParseCommonFlags(cmd, listParser) + if err != nil { + return err + } + + format := v.GetString("format") + + // Initialize config using injected dependency (no component needed for list). + atmosConfig, _, err := configInit.InitConfigAndAuth("", opts.Stack, opts.Identity) + if err != nil { + return err + } + + // Execute list command using injected provisioner. + return prov.ListBackends(atmosConfig, map[string]string{"format": format}) + }, +} + +func init() { + listCmd.DisableFlagParsing = false + + // Create parser with functional options. + listParser = flags.NewStandardParser( + flags.WithStackFlag(), + flags.WithIdentityFlag(), + flags.WithStringFlag("format", "f", "table", "Output format: table, yaml, json"), + flags.WithEnvVars("format", "ATMOS_FORMAT"), + ) + + // Register flags with the command. + listParser.RegisterFlags(listCmd) + + // Bind flags to Viper. + if err := listParser.BindToViper(viper.GetViper()); err != nil { + panic(err) + } +} diff --git a/cmd/terraform/backend/backend_list_test.go b/cmd/terraform/backend/backend_list_test.go new file mode 100644 index 0000000000..113258660c --- /dev/null +++ b/cmd/terraform/backend/backend_list_test.go @@ -0,0 +1,110 @@ +package backend + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestListCmd_Structure(t *testing.T) { + t.Run("command is properly configured", func(t *testing.T) { + assert.NotNil(t, listCmd) + assert.Equal(t, "list", listCmd.Use) + assert.Equal(t, "List all backends in stack", listCmd.Short) + assert.NotEmpty(t, listCmd.Long) + assert.NotEmpty(t, listCmd.Example) + assert.False(t, listCmd.DisableFlagParsing) + }) + + t.Run("parser is configured with required flags", func(t *testing.T) { + assert.NotNil(t, listParser) + + // Verify format flag exists. + formatFlag := listCmd.Flags().Lookup("format") + assert.NotNil(t, formatFlag, "format flag should be registered") + assert.Equal(t, "string", formatFlag.Value.Type()) + + // Verify stack flag exists. + stackFlag := listCmd.Flags().Lookup("stack") + assert.NotNil(t, stackFlag, "stack flag should be registered") + + // Verify identity flag exists. + identityFlag := listCmd.Flags().Lookup("identity") + assert.NotNil(t, identityFlag, "identity flag should be registered") + }) + + t.Run("command accepts no arguments", func(t *testing.T) { + // The Args field should be set to cobra.NoArgs. + assert.NotNil(t, listCmd.Args) + + // Test with no args (should succeed). + err := listCmd.Args(listCmd, []string{}) + assert.NoError(t, err, "should accept no arguments") + + // Test with args (should fail). + err = listCmd.Args(listCmd, []string{"extra"}) + assert.Error(t, err, "should error with arguments") + }) +} + +func TestListCmd_FlagDefaults(t *testing.T) { + tests := []struct { + name string + flagName string + expectedType string + expectedValue string + }{ + { + name: "format flag has table default", + flagName: "format", + expectedType: "string", + expectedValue: "table", + }, + { + name: "stack flag is string", + flagName: "stack", + expectedType: "string", + expectedValue: "", + }, + { + name: "identity flag is string", + flagName: "identity", + expectedType: "string", + expectedValue: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + flag := listCmd.Flags().Lookup(tt.flagName) + assert.NotNil(t, flag, "flag %s should exist", tt.flagName) + assert.Equal(t, tt.expectedType, flag.Value.Type()) + assert.Equal(t, tt.expectedValue, flag.DefValue) + }) + } +} + +func TestListCmd_Shorthand(t *testing.T) { + t.Run("format flag has shorthand", func(t *testing.T) { + flag := listCmd.Flags().Lookup("format") + assert.NotNil(t, flag) + assert.Equal(t, "f", flag.Shorthand, "format flag should have 'f' shorthand") + }) +} + +func TestListCmd_Init(t *testing.T) { + // Verify init() ran successfully by checking parser and flags are set up. + assert.NotNil(t, listParser, "listParser should be initialized") + assert.NotNil(t, listCmd, "listCmd should be initialized") + assert.False(t, listCmd.DisableFlagParsing, "DisableFlagParsing should be false") + + // Verify flags are registered. + stackFlag := listCmd.Flags().Lookup("stack") + assert.NotNil(t, stackFlag, "stack flag should be registered") + + identityFlag := listCmd.Flags().Lookup("identity") + assert.NotNil(t, identityFlag, "identity flag should be registered") + + formatFlag := listCmd.Flags().Lookup("format") + assert.NotNil(t, formatFlag, "format flag should be registered") +} diff --git a/cmd/terraform/backend/backend_test.go b/cmd/terraform/backend/backend_test.go new file mode 100644 index 0000000000..36f568c062 --- /dev/null +++ b/cmd/terraform/backend/backend_test.go @@ -0,0 +1,52 @@ +package backend + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestSetAtmosConfig(t *testing.T) { + // Save original value. + original := atmosConfigPtr + defer func() { + atmosConfigPtr = original + }() + + config := &schema.AtmosConfiguration{ + BasePath: "/test/path", + } + SetAtmosConfig(config) + + assert.Equal(t, config, atmosConfigPtr) + assert.Equal(t, "/test/path", atmosConfigPtr.BasePath) +} + +func TestGetBackendCommand(t *testing.T) { + cmd := GetBackendCommand() + + assert.NotNil(t, cmd) + assert.Equal(t, "backend", cmd.Use) + assert.Equal(t, "Manage Terraform state backends", cmd.Short) + assert.NotEmpty(t, cmd.Long) + + // Verify subcommands are registered. + subcommands := cmd.Commands() + assert.GreaterOrEqual(t, len(subcommands), 5, "should have at least 5 subcommands") + + // Verify specific subcommands exist by checking Use field prefix. + // Note: createCmd.Use is "" but all others start with their command name. + subcommandUses := make(map[string]bool) + for _, sub := range subcommands { + subcommandUses[sub.Use] = true + } + + // Verify we have the expected commands by their Use patterns. + assert.True(t, subcommandUses[""], "should have create subcommand (Use: '')") + assert.True(t, subcommandUses["list"], "should have list subcommand") + assert.True(t, subcommandUses["describe "], "should have describe subcommand") + assert.True(t, subcommandUses["update "], "should have update subcommand") + assert.True(t, subcommandUses["delete "], "should have delete subcommand") +} diff --git a/cmd/terraform/backend/backend_test_helpers.go b/cmd/terraform/backend/backend_test_helpers.go new file mode 100644 index 0000000000..bf7f20568b --- /dev/null +++ b/cmd/terraform/backend/backend_test_helpers.go @@ -0,0 +1,68 @@ +package backend + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + + "github.com/cloudposse/atmos/pkg/flags" +) + +// commandTestParams holds parameters for testing backend command structure. +type commandTestParams struct { + cmd *cobra.Command + parser *flags.StandardParser + expectedUse string + expectedShort string + requiredFlags []string +} + +// testCommandStructure is a helper function to test common command structure patterns. +// It reduces code duplication across backend command tests. +func testCommandStructure(t *testing.T, params commandTestParams) { + t.Helper() + + t.Run("command is properly configured", func(t *testing.T) { + assert.NotNil(t, params.cmd) + assert.Equal(t, params.expectedUse, params.cmd.Use) + assert.Equal(t, params.expectedShort, params.cmd.Short) + assert.NotEmpty(t, params.cmd.Long) + assert.NotEmpty(t, params.cmd.Example) + assert.False(t, params.cmd.DisableFlagParsing) + }) + + t.Run("parser is configured with required flags", func(t *testing.T) { + assert.NotNil(t, params.parser) + + for _, flagName := range params.requiredFlags { + flag := params.cmd.Flags().Lookup(flagName) + assert.NotNil(t, flag, "%s flag should be registered", flagName) + } + + // Verify stack flag exists (common to all commands). + stackFlag := params.cmd.Flags().Lookup("stack") + assert.NotNil(t, stackFlag, "stack flag should be registered") + + // Verify identity flag exists (common to all commands). + identityFlag := params.cmd.Flags().Lookup("identity") + assert.NotNil(t, identityFlag, "identity flag should be registered") + }) + + t.Run("command requires exactly one argument", func(t *testing.T) { + // The Args field should be set to cobra.ExactArgs(1). + assert.NotNil(t, params.cmd.Args) + + // Test with no args. + err := params.cmd.Args(params.cmd, []string{}) + assert.Error(t, err, "should error with no arguments") + + // Test with one arg. + err = params.cmd.Args(params.cmd, []string{"vpc"}) + assert.NoError(t, err, "should accept exactly one argument") + + // Test with multiple args. + err = params.cmd.Args(params.cmd, []string{"vpc", "extra"}) + assert.Error(t, err, "should error with multiple arguments") + }) +} diff --git a/cmd/terraform/backend/backend_update.go b/cmd/terraform/backend/backend_update.go new file mode 100644 index 0000000000..d4a6c2d55b --- /dev/null +++ b/cmd/terraform/backend/backend_update.go @@ -0,0 +1,39 @@ +package backend + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cloudposse/atmos/pkg/flags" +) + +var updateParser *flags.StandardParser + +var updateCmd = &cobra.Command{ + Use: "update ", + Short: "Update backend configuration", + Long: `Apply configuration changes to existing backend. + +This operation is idempotent and will update backend settings like +versioning, encryption, and public access blocking to match secure defaults.`, + Example: ` atmos terraform backend update vpc --stack dev`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ExecuteProvisionCommand(cmd, args, updateParser, "backend.update.RunE") + }, +} + +func init() { + updateCmd.DisableFlagParsing = false + + updateParser = flags.NewStandardParser( + flags.WithStackFlag(), + flags.WithIdentityFlag(), + ) + + updateParser.RegisterFlags(updateCmd) + + if err := updateParser.BindToViper(viper.GetViper()); err != nil { + panic(err) + } +} diff --git a/cmd/terraform/backend/backend_update_test.go b/cmd/terraform/backend/backend_update_test.go new file mode 100644 index 0000000000..16ee93d129 --- /dev/null +++ b/cmd/terraform/backend/backend_update_test.go @@ -0,0 +1,31 @@ +package backend + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUpdateCmd_Structure(t *testing.T) { + testCommandStructure(t, commandTestParams{ + cmd: updateCmd, + parser: updateParser, + expectedUse: "update ", + expectedShort: "Update backend configuration", + requiredFlags: []string{}, + }) +} + +func TestUpdateCmd_Init(t *testing.T) { + // Verify init() ran successfully by checking parser and flags are set up. + assert.NotNil(t, updateParser, "updateParser should be initialized") + assert.NotNil(t, updateCmd, "updateCmd should be initialized") + assert.False(t, updateCmd.DisableFlagParsing, "DisableFlagParsing should be false") + + // Verify flags are registered. + stackFlag := updateCmd.Flags().Lookup("stack") + assert.NotNil(t, stackFlag, "stack flag should be registered") + + identityFlag := updateCmd.Flags().Lookup("identity") + assert.NotNil(t, identityFlag, "identity flag should be registered") +} diff --git a/cmd/terraform/backend/mock_backend_helpers_test.go b/cmd/terraform/backend/mock_backend_helpers_test.go new file mode 100644 index 0000000000..0be071d76c --- /dev/null +++ b/cmd/terraform/backend/mock_backend_helpers_test.go @@ -0,0 +1,137 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: backend_helpers.go +// +// Generated by this command: +// +// mockgen -source=backend_helpers.go -destination=mock_backend_helpers_test.go -package=backend +// + +// Package backend is a generated GoMock package. +package backend + +import ( + reflect "reflect" + + schema "github.com/cloudposse/atmos/pkg/schema" + gomock "go.uber.org/mock/gomock" +) + +// MockConfigInitializer is a mock of ConfigInitializer interface. +type MockConfigInitializer struct { + ctrl *gomock.Controller + recorder *MockConfigInitializerMockRecorder + isgomock struct{} +} + +// MockConfigInitializerMockRecorder is the mock recorder for MockConfigInitializer. +type MockConfigInitializerMockRecorder struct { + mock *MockConfigInitializer +} + +// NewMockConfigInitializer creates a new mock instance. +func NewMockConfigInitializer(ctrl *gomock.Controller) *MockConfigInitializer { + mock := &MockConfigInitializer{ctrl: ctrl} + mock.recorder = &MockConfigInitializerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConfigInitializer) EXPECT() *MockConfigInitializerMockRecorder { + return m.recorder +} + +// InitConfigAndAuth mocks base method. +func (m *MockConfigInitializer) InitConfigAndAuth(component, stack, identity string) (*schema.AtmosConfiguration, *schema.AuthContext, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitConfigAndAuth", component, stack, identity) + ret0, _ := ret[0].(*schema.AtmosConfiguration) + ret1, _ := ret[1].(*schema.AuthContext) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// InitConfigAndAuth indicates an expected call of InitConfigAndAuth. +func (mr *MockConfigInitializerMockRecorder) InitConfigAndAuth(component, stack, identity any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitConfigAndAuth", reflect.TypeOf((*MockConfigInitializer)(nil).InitConfigAndAuth), component, stack, identity) +} + +// MockProvisioner is a mock of Provisioner interface. +type MockProvisioner struct { + ctrl *gomock.Controller + recorder *MockProvisionerMockRecorder + isgomock struct{} +} + +// MockProvisionerMockRecorder is the mock recorder for MockProvisioner. +type MockProvisionerMockRecorder struct { + mock *MockProvisioner +} + +// NewMockProvisioner creates a new mock instance. +func NewMockProvisioner(ctrl *gomock.Controller) *MockProvisioner { + mock := &MockProvisioner{ctrl: ctrl} + mock.recorder = &MockProvisionerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProvisioner) EXPECT() *MockProvisionerMockRecorder { + return m.recorder +} + +// CreateBackend mocks base method. +func (m *MockProvisioner) CreateBackend(params *CreateBackendParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateBackend", params) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateBackend indicates an expected call of CreateBackend. +func (mr *MockProvisionerMockRecorder) CreateBackend(params any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackend", reflect.TypeOf((*MockProvisioner)(nil).CreateBackend), params) +} + +// DeleteBackend mocks base method. +func (m *MockProvisioner) DeleteBackend(params *DeleteBackendParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBackend", params) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteBackend indicates an expected call of DeleteBackend. +func (mr *MockProvisionerMockRecorder) DeleteBackend(params any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBackend", reflect.TypeOf((*MockProvisioner)(nil).DeleteBackend), params) +} + +// DescribeBackend mocks base method. +func (m *MockProvisioner) DescribeBackend(atmosConfig *schema.AtmosConfiguration, component string, opts any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeBackend", atmosConfig, component, opts) + ret0, _ := ret[0].(error) + return ret0 +} + +// DescribeBackend indicates an expected call of DescribeBackend. +func (mr *MockProvisionerMockRecorder) DescribeBackend(atmosConfig, component, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeBackend", reflect.TypeOf((*MockProvisioner)(nil).DescribeBackend), atmosConfig, component, opts) +} + +// ListBackends mocks base method. +func (m *MockProvisioner) ListBackends(atmosConfig *schema.AtmosConfiguration, opts any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListBackends", atmosConfig, opts) + ret0, _ := ret[0].(error) + return ret0 +} + +// ListBackends indicates an expected call of ListBackends. +func (mr *MockProvisionerMockRecorder) ListBackends(atmosConfig, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBackends", reflect.TypeOf((*MockProvisioner)(nil).ListBackends), atmosConfig, opts) +} diff --git a/cmd/terraform_commands.go b/cmd/terraform_commands.go index 00eb732b50..5df8c3ae11 100644 --- a/cmd/terraform_commands.go +++ b/cmd/terraform_commands.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/cloudposse/atmos/cmd/terraform/backend" errUtils "github.com/cloudposse/atmos/errors" cfg "github.com/cloudposse/atmos/pkg/config" h "github.com/cloudposse/atmos/pkg/hooks" @@ -302,6 +303,9 @@ func attachTerraformCommands(parentCmd *cobra.Command) { "If set to 'false' (default), the target reference will be checked out instead\n"+ "This requires that the target reference is already cloned by Git, and the information about it exists in the '.git' directory") + // Add backend subcommand to terraform. + parentCmd.AddCommand(backend.GetBackendCommand()) + commands := getTerraformCommands() for _, cmd := range commands { diff --git a/demo/screengrabs/demo-stacks.txt b/demo/screengrabs/demo-stacks.txt index c401c7ba25..74f28e3862 100644 --- a/demo/screengrabs/demo-stacks.txt +++ b/demo/screengrabs/demo-stacks.txt @@ -62,6 +62,12 @@ atmos profile list --help atmos profile show --help atmos terraform --help atmos terraform apply --help +atmos terraform backend --help +atmos terraform backend create --help +atmos terraform backend delete --help +atmos terraform backend describe --help +atmos terraform backend list --help +atmos terraform backend update --help atmos terraform clean --help atmos terraform console --help atmos terraform deploy --help diff --git a/docs/prd/backend-provisioner.md b/docs/prd/backend-provisioner.md new file mode 100644 index 0000000000..12dbef496a --- /dev/null +++ b/docs/prd/backend-provisioner.md @@ -0,0 +1,1100 @@ +# PRD: Backend Provisioner + +**Status:** Draft for Review +**Version:** 1.0 +**Last Updated:** 2025-11-19 +**Author:** Erik Osterman + +--- + +## Executive Summary + +The Backend Provisioner is the first implementation of the Atmos Provisioner System. It automatically provisions Terraform state backends (S3, GCS, Azure Blob Storage) before Terraform initialization, eliminating cold-start friction for development and testing environments. + +**Key Principle:** Backend provisioning is infrastructure plumbing - opinionated, automatic, and invisible to users except via simple configuration flags. + +--- + +## Overview + +### What is a Backend Provisioner? + +The Backend Provisioner is a system hook that: +1. **Registers** for `before.terraform.init` hook event +2. **Checks** if `provision.backend.enabled: true` in component config +3. **Delegates** to backend-type-specific provisioner (S3, GCS, Azure) +4. **Provisions** minimal backend infrastructure with secure defaults + +### Scope + +#### In Scope + +- ✅ S3 backend provisioning (Phase 1 - see `s3-backend-provisioner.md`) +- ✅ GCS backend provisioning (Phase 2) +- ✅ Azure Blob backend provisioning (Phase 2) +- ✅ Secure defaults (encryption, versioning, public access blocking) +- ✅ Development/testing focus + +#### Out of Scope + +- ❌ Production-grade features (custom KMS, replication, lifecycle policies) +- ❌ DynamoDB table provisioning (Terraform 1.10+ has native S3 locking) +- ❌ Backend migration/destruction +- ❌ Backend drift detection + +--- + +## Architecture + +### Backend Provisioner Registration + +```go +// pkg/provisioner/backend/backend.go + +package backend + +import ( + "github.com/cloudposse/atmos/pkg/hooks" + "github.com/cloudposse/atmos/pkg/provisioner" + "github.com/cloudposse/atmos/pkg/schema" +) + +func init() { + // Backend provisioner registers for before.terraform.init + provisioner.RegisterProvisioner(provisioner.Provisioner{ + Type: "backend", + HookEvent: hooks.BeforeTerraformInit, // Self-declared timing + Func: ProvisionBackend, + }) +} +``` + +### Backend Provisioner Interface + +```go +// BackendProvisionerFunc defines the interface for backend-specific provisioners. +type BackendProvisionerFunc func( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error +``` + +### Backend Registry + +```go +// pkg/provisioner/backend/registry.go + +package backend + +import ( + "fmt" + "sync" + + "github.com/cloudposse/atmos/pkg/schema" +) + +// Backend provisioner registry maps backend type → provisioner function +var backendProvisioners = make(map[string]BackendProvisionerFunc) +var registerBackendProvisionersOnce sync.Once + +// RegisterBackendProvisioners registers all backend-specific provisioners. +func RegisterBackendProvisioners() { + registerBackendProvisionersOnce.Do(func() { + // Phase 1: S3 backend + backendProvisioners["s3"] = ProvisionS3Backend + + // Phase 2: Multi-cloud backends + // backendProvisioners["gcs"] = ProvisionGCSBackend + // backendProvisioners["azurerm"] = ProvisionAzureBackend + }) +} + +// GetBackendProvisioner retrieves a backend provisioner by type. +func GetBackendProvisioner(backendType string) (BackendProvisionerFunc, error) { + if provisioner, ok := backendProvisioners[backendType]; ok { + return provisioner, nil + } + return nil, fmt.Errorf("no provisioner for backend type: %s", backendType) +} +``` + +### Main Backend Provisioner Logic + +```go +// pkg/provisioner/backend/backend.go + +// ProvisionBackend is the main backend provisioner (delegates to backend-specific provisioners). +func ProvisionBackend( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error { + defer perf.Track(atmosConfig, "provisioner.backend.ProvisionBackend")() + + // 1. Check if backend provisioning is enabled + if !isBackendProvisioningEnabled(componentSections) { + return nil + } + + // 2. Register backend-specific provisioners + RegisterBackendProvisioners() + + // 3. Get backend type from component sections + backendType := getBackendType(componentSections) + if backendType == "" { + return fmt.Errorf("backend_type not specified") + } + + // 4. Get backend-specific provisioner + backendProvisioner, err := GetBackendProvisioner(backendType) + if err != nil { + ui.Warning(fmt.Sprintf("No provisioner available for backend type '%s'", backendType)) + return nil // Not an error - just skip provisioning + } + + // 5. Execute backend-specific provisioner + ui.Info(fmt.Sprintf("Provisioning %s backend...", backendType)) + if err := backendProvisioner(atmosConfig, componentSections, authContext); err != nil { + return fmt.Errorf("failed to provision %s backend: %w", backendType, err) + } + + ui.Success(fmt.Sprintf("Backend '%s' provisioned successfully", backendType)) + return nil +} + +// isBackendProvisioningEnabled checks if provision.backend.enabled is true. +func isBackendProvisioningEnabled(componentSections *map[string]any) bool { + provisionConfig, ok := (*componentSections)["provision"].(map[string]any) + if !ok { + return false + } + + backendConfig, ok := provisionConfig["backend"].(map[string]any) + if !ok { + return false + } + + enabled, ok := backendConfig["enabled"].(bool) + return ok && enabled +} + +// getBackendType extracts backend_type from component sections. +func getBackendType(componentSections *map[string]any) string { + backendType, ok := (*componentSections)["backend_type"].(string) + if !ok { + return "" + } + return backendType +} +``` + +--- + +## Configuration Schema + +### Stack Manifest Configuration + +```yaml +components: + terraform: + vpc: + # Backend type (standard Terraform) + backend_type: s3 + + # Backend configuration (standard Terraform) + backend: + bucket: my-terraform-state + key: vpc/terraform.tfstate + region: us-east-1 + encrypt: true + + # Optional: Role assumption (standard Terraform syntax) + assume_role: + role_arn: "arn:aws:iam::999999999999:role/TerraformStateAdmin" + session_name: "atmos-backend-provision" + + # Provisioning configuration (Atmos-specific, never serialized to backend.tf.json) + provision: + backend: + enabled: true # Enable auto-provisioning for this backend +``` + +### Global Configuration (atmos.yaml) + +```yaml +# atmos.yaml +settings: + backends: + auto_provision: + enabled: true # Global feature flag (default: false) +``` + +### Configuration Hierarchy + +The `provision.backend` configuration leverages Atmos's deep-merge system and can be specified at **multiple levels** in the stack hierarchy. This provides maximum flexibility for different organizational patterns. + +#### 1. Top-Level Terraform Defaults + +Enable provisioning for all components across all environments: + +```yaml +# stacks/_defaults.yaml or stacks/orgs/acme/_defaults.yaml +terraform: + provision: + backend: + enabled: true # Applies to all components +``` + +#### 2. Environment-Level Configuration + +Override defaults per environment: + +```yaml +# stacks/orgs/acme/plat/dev/_defaults.yaml +terraform: + provision: + backend: + enabled: true # Enable for all dev components + +# stacks/orgs/acme/plat/prod/_defaults.yaml +terraform: + provision: + backend: + enabled: false # Disable for production (use pre-provisioned backends) +``` + +#### 3. Component-Level Configuration + +Override at the component level: + +```yaml +# stacks/dev.yaml +components: + terraform: + vpc: + provision: + backend: + enabled: true # Enable for this specific component + + eks: + provision: + backend: + enabled: false # Disable for this specific component +``` + +#### 4. Inheritance via metadata.inherits + +Share provision configuration through catalog components: + +```yaml +# stacks/catalog/vpc/defaults.yaml +components: + terraform: + vpc/defaults: + provision: + backend: + enabled: true + +# stacks/dev.yaml +components: + terraform: + vpc: + metadata: + inherits: [vpc/defaults] + # Automatically inherits provision.backend.enabled: true +``` + +#### Deep-Merge Behavior + +Atmos deep-merges `provision` blocks across all hierarchy levels: + +```yaml +# 1. Top-level default +terraform: + provision: + backend: + enabled: true + +# 2. Component override +components: + terraform: + vpc: + provision: + backend: + enabled: false # Overrides top-level setting + +# Result after deep-merge: +# vpc component has provision.backend.enabled: false +``` + +#### Key Benefits + +- **DRY Principle**: Set defaults once at high levels +- **Environment Flexibility**: Dev uses auto-provision, prod uses pre-provisioned +- **Component Control**: Override per component when needed +- **Catalog Reuse**: Share provision settings through inherits + +### Configuration Filtering + +**Critical:** The `provision` block is **never serialized** to `backend.tf.json`: + +```go +// internal/exec/terraform_generate_backend.go + +func generateBackendConfig(backendConfig map[string]any) map[string]any { + // Remove Atmos-specific keys before serialization + filteredConfig := make(map[string]any) + for k, v := range backendConfig { + if k != "provision" { // Filter out provision block + filteredConfig[k] = v + } + } + return filteredConfig +} +``` + +--- + +## Role Assumption and Cross-Account Provisioning + +### How Role Assumption Works + +1. **Component identity** (from `auth.providers.aws.identity`) provides base credentials +2. **Role ARN** (from `backend.assume_role.role_arn`) specifies cross-account role +3. **Backend provisioner** assumes role using base credentials +4. **Provisioning** happens in target account with assumed role + +### Configuration Example + +```yaml +components: + terraform: + vpc: + # Source account identity + auth: + providers: + aws: + type: aws-sso + identity: dev-admin # Credentials in account 111111111111 + + # Target account backend + backend_type: s3 + backend: + bucket: prod-terraform-state + region: us-east-1 + + # Assume role in target account + assume_role: + role_arn: "arn:aws:iam::999999999999:role/TerraformStateAdmin" + session_name: "atmos-backend-provision" + + # Enable provisioning + provision: + backend: + enabled: true +``` + +#### Flow + +1. Auth system authenticates as `dev-admin` (account 111111111111) +2. Backend provisioner extracts `role_arn` from backend config +3. Provisioner assumes role in target account (999999999999) +4. S3 bucket created in target account + +### Implementation Pattern + +```go +// Backend provisioners follow this pattern for role assumption + +func ProvisionS3Backend( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error { + // Extract backend config + backendConfig := (*componentSections)["backend"].(map[string]any) + region := backendConfig["region"].(string) + + // Get role ARN from backend config (if specified) + roleArn := GetS3BackendAssumeRoleArn(&backendConfig) + + // Load AWS config with auth context + role assumption + cfg, err := awsUtils.LoadAWSConfigWithAuth( + ctx, + region, + roleArn, // From backend.assume_role.role_arn + 15*time.Minute, + authContext.AWS, // From component's auth.identity + ) + + // Create client and provision + client := s3.NewFromConfig(cfg) + return provisionBucket(client, bucket) +} + +// GetS3BackendAssumeRoleArn extracts role ARN from backend config (standard Terraform syntax) +func GetS3BackendAssumeRoleArn(backend *map[string]any) string { + // Try assume_role block first (standard Terraform) + if assumeRoleSection, ok := (*backend)["assume_role"].(map[string]any); ok { + if roleArn, ok := assumeRoleSection["role_arn"].(string); ok { + return roleArn + } + } + + // Fallback to top-level role_arn (legacy) + if roleArn, ok := (*backend)["role_arn"].(string); ok { + return roleArn + } + + return "" +} +``` + +--- + +## Backend-Specific Provisioner Requirements + +### Interface Contract + +All backend provisioners MUST: + +1. **Check if backend exists** (idempotent operation) +2. **Create backend with secure defaults** (if doesn't exist) +3. **Return nil** (no error) if backend already exists +4. **Use AuthContext** for authentication +5. **Support role assumption** from backend config +6. **Implement client caching** for performance +7. **Retry with exponential backoff** for transient failures +8. **Log provisioning actions** with ui package + +### Hardcoded Best Practices (No Configuration) + +All backends MUST create resources with: + +- ✅ **Encryption** - Always enabled (provider-managed keys) +- ✅ **Versioning** - Always enabled (recovery from accidental deletions) +- ✅ **Public Access** - Always blocked (security) +- ✅ **Resource Tags/Labels**: + - `ManagedBy: Atmos` + - `CreatedAt: ` + - `Purpose: TerraformState` + +### Provider-Specific Defaults + +#### AWS S3 +- Encryption: AES-256 or AWS-managed KMS +- Versioning: Enabled +- Public access: All 4 settings blocked +- State locking: Terraform 1.10+ native S3 locking (no DynamoDB) + +#### GCP GCS +- Encryption: Google-managed encryption keys +- Versioning: Enabled +- Access: Uniform bucket-level access +- State locking: Native GCS locking + +#### Azure Blob Storage +- Encryption: Microsoft-managed keys +- Versioning: Blob versioning enabled +- HTTPS: Required +- State locking: Native blob lease locking + +--- + +## Implementation Guide + +### Step 1: Implement Backend Provisioner + +```go +// pkg/provisioner/backend/mybackend.go + +package backend + +func ProvisionMyBackend( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error { + defer perf.Track(atmosConfig, "provisioner.backend.ProvisionMyBackend")() + + // 1. Extract backend configuration + backendConfig := (*componentSections)["backend"].(map[string]any) + containerName := backendConfig["container"].(string) + + // 2. Get authenticated client (with role assumption if needed) + client, err := getMyBackendClient(backendConfig, authContext) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + // 3. Check if backend exists (idempotent) + exists, err := checkMyBackendExists(client, containerName) + if err != nil { + return fmt.Errorf("failed to check backend existence: %w", err) + } + + if exists { + ui.Info(fmt.Sprintf("Backend '%s' already exists", containerName)) + return nil + } + + // 4. Create backend with hardcoded best practices + ui.Info(fmt.Sprintf("Creating backend '%s'...", containerName)) + if err := createMyBackend(client, containerName); err != nil { + return fmt.Errorf("failed to create backend: %w", err) + } + + ui.Success(fmt.Sprintf("Backend '%s' created successfully", containerName)) + return nil +} +``` + +### Step 2: Register Backend Provisioner + +```go +// pkg/provisioner/backend/registry.go + +func RegisterBackendProvisioners() { + registerBackendProvisionersOnce.Do(func() { + backendProvisioners["s3"] = ProvisionS3Backend + backendProvisioners["mybackend"] = ProvisionMyBackend // Add new backend + }) +} +``` + +### Step 3: Test Backend Provisioner + +```go +// pkg/provisioner/backend/mybackend_test.go + +func TestProvisionMyBackend_NewBackend(t *testing.T) { + // Mock client + mockClient := &MockMyBackendClient{} + + // Configure mock expectations + mockClient.EXPECT(). + CheckExists("my-container"). + Return(false, nil) + + mockClient.EXPECT(). + Create("my-container", gomock.Any()). + Return(nil) + + // Execute provisioner + err := ProvisionMyBackend(atmosConfig, componentSections, authContext) + + // Verify + assert.NoError(t, err) +} + +func TestProvisionMyBackend_ExistingBackend(t *testing.T) { + // Mock client + mockClient := &MockMyBackendClient{} + + // Backend already exists + mockClient.EXPECT(). + CheckExists("my-container"). + Return(true, nil) + + // Should NOT call Create + mockClient.EXPECT(). + Create(gomock.Any(), gomock.Any()). + Times(0) + + // Execute provisioner + err := ProvisionMyBackend(atmosConfig, componentSections, authContext) + + // Verify idempotent behavior + assert.NoError(t, err) +} +``` + +--- + +## Error Handling + +### Error Definitions + +```go +// pkg/provisioner/backend/errors.go + +package backend + +import "errors" + +var ( + // Backend provisioning errors + ErrBackendProvision = errors.New("backend provisioning failed") + ErrBackendCheck = errors.New("backend existence check failed") + ErrBackendConfig = errors.New("invalid backend configuration") + ErrBackendTypeUnsupported = errors.New("backend type not supported for provisioning") +) +``` + +### Error Examples + +```go +// Configuration error +if bucket == "" { + return fmt.Errorf("%w: bucket name is required", ErrBackendConfig) +} + +// Provisioning error with context +return errUtils.Build(ErrBackendProvision). + WithHint("Verify AWS credentials have s3:CreateBucket permission"). + WithContext("backend_type", "s3"). + WithContext("bucket", bucket). + WithContext("region", region). + WithExitCode(2). + Err() + +// Permission error (exit code 3 per exit code table) +return errUtils.Build(ErrBackendProvision). + WithHint("Required permissions: s3:CreateBucket, s3:PutBucketVersioning, s3:PutBucketEncryption"). + WithHintf("Check IAM policy for identity: %s", authContext.AWS.Profile). + WithContext("bucket", bucket). + WithExitCode(3). + Err() +``` + +--- + +## Testing Strategy + +### Unit Tests (per backend type) + +```go +func TestProvisionS3Backend_NewBucket(t *testing.T) +func TestProvisionS3Backend_ExistingBucket(t *testing.T) +func TestProvisionS3Backend_InvalidConfig(t *testing.T) +func TestProvisionS3Backend_PermissionDenied(t *testing.T) +func TestProvisionS3Backend_RoleAssumption(t *testing.T) +``` + +### Integration Tests + +```go +// tests/backend_provisioning_test.go + +func TestBackendProvisioning_S3_FreshAccount(t *testing.T) { + // Requires: localstack or real AWS account + tests.RequireAWSAccess(t) + + // Execute provisioning + err := ProvisionBackend(atmosConfig, componentSections, authContext) + + // Verify bucket created with correct settings + assert.NoError(t, err) + assertBucketExists(t, "my-test-bucket") + assertVersioningEnabled(t, "my-test-bucket") + assertEncryptionEnabled(t, "my-test-bucket") + assertPublicAccessBlocked(t, "my-test-bucket") +} + +func TestBackendProvisioning_S3_Idempotent(t *testing.T) { + // Create bucket manually first + createBucket(t, "my-test-bucket") + + // Execute provisioning + err := ProvisionBackend(atmosConfig, componentSections, authContext) + + // Should not error - idempotent + assert.NoError(t, err) +} +``` + +--- + +## Security Considerations + +### IAM Permissions + +Backend provisioners require specific permissions. Document these clearly: + +#### AWS S3 Backend + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:HeadBucket", + "s3:PutBucketVersioning", + "s3:PutBucketEncryption", + "s3:PutBucketPublicAccessBlock", + "s3:PutBucketTagging" + ], + "Resource": "arn:aws:s3:::*-terraform-state-*" + } + ] +} +``` + +#### GCP GCS Backend + +```yaml +roles: + - roles/storage.admin # For bucket creation +``` + +#### Azure Blob Backend + +```yaml +permissions: + - Microsoft.Storage/storageAccounts/write + - Microsoft.Storage/storageAccounts/blobServices/containers/write +``` + +### Security Defaults + +All backends MUST: +- ✅ Enable encryption at rest +- ✅ Block public access +- ✅ Enable versioning +- ✅ Use provider-managed keys (not custom KMS for simplicity) +- ✅ Apply resource tags for tracking + +--- + +## Performance Optimization + +### Client Caching + +```go +// Per-backend client cache +var s3ClientCache sync.Map + +func getCachedS3Client(region string, authContext *schema.AuthContext) (*s3.Client, error) { + // Build deterministic cache key + cacheKey := fmt.Sprintf("region=%s;profile=%s", region, authContext.AWS.Profile) + + // Check cache + if cached, ok := s3ClientCache.Load(cacheKey); ok { + return cached.(*s3.Client), nil + } + + // Create client with timeout + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cfg, err := LoadAWSConfigWithAuth(ctx, region, authContext) + if err != nil { + return nil, err + } + + client := s3.NewFromConfig(cfg) + s3ClientCache.Store(cacheKey, client) + return client, nil +} +``` + +### Retry Logic + +```go +const maxRetries = 3 + +func provisionWithRetry(fn func() error) error { + var lastErr error + + for attempt := 0; attempt < maxRetries; attempt++ { + if err := fn(); err == nil { + return nil + } else { + lastErr = err + backoff := time.Duration(attempt+1) * 2 * time.Second + time.Sleep(backoff) + } + } + + return fmt.Errorf("provisioning failed after %d attempts: %w", maxRetries, lastErr) +} +``` + +--- + +## Documentation Requirements + +Each backend provisioner MUST document: + +1. **Backend type** - What backend does it provision? +2. **Resources created** - What infrastructure is created? +3. **Hardcoded defaults** - What security settings are applied? +4. **Required permissions** - What IAM/RBAC permissions needed? +5. **Configuration example** - How to enable provisioning? +6. **Limitations** - What's NOT supported (custom KMS, etc.)? +7. **Migration path** - How to upgrade to production backend? + +--- + +## CLI Commands + +### Backend Management Commands + +```bash +# Create/provision backend explicitly +atmos terraform backend create --stack + +# Update existing backend configuration +atmos terraform backend update --stack + +# List all backends in a stack +atmos terraform backend list --stack + +# Describe backend configuration for a component +atmos terraform backend describe --stack + +# Delete backend (requires --force flag) +atmos terraform backend delete --stack --force + +# Examples +atmos terraform backend create vpc --stack dev +atmos terraform backend create eks --stack prod +atmos terraform backend list --stack dev --format json +atmos terraform backend describe vpc --stack dev --format yaml +atmos terraform backend delete vpc --stack dev --force +``` + +#### When to Use + +- Separate provisioning from Terraform execution (CI/CD pipelines) +- Troubleshoot provisioning issues +- Pre-provision backends for multiple components +- Manage backend lifecycle independently + +#### Automatic Provisioning (via Hooks) + +```bash +# Backend provisioned automatically if provision.backend.enabled: true +atmos terraform apply vpc --stack dev +``` + +### Error Handling in CLI + +#### Provisioning Failure Stops Execution + +```bash +$ atmos terraform backend create vpc --stack dev +Error: provisioner 'backend' failed: backend provisioning failed: +failed to create bucket: AccessDenied + +Hint: Verify AWS credentials have s3:CreateBucket permission +Context: bucket=acme-state-dev, region=us-east-1 + +Exit code: 3 +``` + +#### Terraform Won't Run if Provisioning Fails + +```bash +$ atmos terraform apply vpc --stack dev +Running backend provisioner... +Error: Provisioning failed - cannot proceed with terraform +provisioner 'backend' failed: backend provisioning failed + +Exit code: 2 +``` + +--- + +## Error Handling and Propagation + +### Error Handling Requirements + +All backend provisioners MUST: + +1. **Return errors (never panic)** + ```go + func ProvisionS3Backend(...) error { + if err := createBucket(); err != nil { + return fmt.Errorf("failed to create bucket: %w", err) + } + return nil + } + ``` + +2. **Return nil for idempotent operations** + ```go + if bucketExists { + ui.Info("Bucket already exists (idempotent)") + return nil // Not an error + } + ``` + +3. **Use error builder for detailed errors** + ```go + return errUtils.Build(errUtils.ErrBackendProvision). + WithHint("Verify AWS credentials have s3:CreateBucket permission"). + WithContext("bucket", bucket). + WithExitCode(3). + Err() + ``` + +4. **Fail fast on critical errors** + ```go + if bucket == "" { + return fmt.Errorf("%w: bucket name required", errUtils.ErrInvalidConfig) + } + ``` + +### Error Propagation Flow + +```text +Backend Provisioner (ProvisionS3Backend) + ↓ returns error +Backend Provisioner Wrapper (ProvisionBackend) + ↓ wraps and returns +Hook System (ExecuteProvisionerHooks) + ↓ propagates immediately (fail fast) +Terraform Execution (ExecuteTerraform) + ↓ stops before terraform init +Main (main.go) + ↓ exits with error code +CI/CD Pipeline + ↓ fails build +``` + +### Exit Codes + +| Exit Code | Error Type | Example | +|-----------|------------|---------| +| 0 | Success | Backend created or already exists | +| 1 | General error | Unexpected AWS SDK error | +| 2 | Configuration error | Missing bucket name in config | +| 3 | Permission error | IAM s3:CreateBucket denied | +| 4 | Resource conflict | Bucket name globally taken | +| 5 | Network error | Connection timeout to AWS API | + +### Error Examples by Backend Type + +#### S3 Backend Errors + +Configuration Error: + +```go +if bucket == "" { + return fmt.Errorf("%w: backend.bucket is required", errUtils.ErrBackendConfig) +} +``` + +Permission Error: + +```go +if isAccessDenied(err) { + return errUtils.Build(errUtils.ErrBackendProvision). + WithHint("Required IAM permissions: s3:CreateBucket, s3:PutBucketVersioning"). + WithHintf("Check policy for identity: %s", authContext.AWS.Profile). + WithContext("bucket", bucket). + WithExitCode(3). + Err() +} +``` + +Resource Conflict: + +```go +if isBucketNameTaken(err) { + return errUtils.Build(errUtils.ErrBackendProvision). + WithHint("S3 bucket names are globally unique across all AWS accounts"). + WithHintf("Try a different name: %s-%s", bucket, accountID). + WithContext("bucket", bucket). + WithExitCode(4). + Err() +} +``` + +#### GCS Backend Errors (Future) + +Permission Error: + +```go +return errUtils.Build(errUtils.ErrBackendProvision). + WithHint("Required GCP permissions: storage.buckets.create"). + WithContext("bucket", bucket). + WithContext("project", project). + WithExitCode(3). + Err() +``` + +#### Azure Backend Errors (Future) + +Permission Error: + +```go +return errUtils.Build(errUtils.ErrBackendProvision). + WithHint("Required Azure permissions: Microsoft.Storage/storageAccounts/write"). + WithContext("storage_account", storageAccount). + WithExitCode(3). + Err() +``` + +### Testing Error Handling + +Unit Tests: + +```go +func TestProvisionS3Backend_ConfigurationError(t *testing.T) { + componentSections := map[string]any{ + "backend": map[string]any{ + // Missing bucket name + "region": "us-east-1", + }, + } + + err := ProvisionS3Backend(atmosConfig, &componentSections, authContext) + + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBackendConfig) + assert.Contains(t, err.Error(), "bucket") +} + +func TestProvisionS3Backend_PermissionDenied(t *testing.T) { + mockClient := &MockS3Client{} + mockClient.EXPECT(). + CreateBucket(gomock.Any()). + Return(nil, awserr.New("AccessDenied", "Permission denied", nil)) + + err := ProvisionS3Backend(...) + + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBackendProvision) + exitCode := errUtils.GetExitCode(err) + assert.Equal(t, 3, exitCode) +} +``` + +--- + +## Related Documents + +- **[Provisioner System](./provisioner-system.md)** - Generic provisioner infrastructure +- **[S3 Backend Provisioner](./s3-backend-provisioner.md)** - S3 implementation (reference) + +--- + +## Appendix: Backend Type Registry + +| Backend Type | Status | Provisioner | Phase | +|--------------|--------|-------------|-------| +| `s3` | ✅ Implemented | `ProvisionS3Backend` | Phase 1 | +| `gcs` | 🔄 Planned | `ProvisionGCSBackend` | Phase 2 | +| `azurerm` | 🔄 Planned | `ProvisionAzureBackend` | Phase 2 | +| `local` | ❌ Not applicable | N/A | - | +| `cloud` | ❌ Not applicable | N/A (Terraform Cloud manages storage) | - | +| `remote` | ❌ Deprecated | N/A | - | + +--- + +## Conclusion + +**Status:** Implemented + +### Next Steps + +1. Review backend provisioner interface +2. Implement S3 backend provisioner (see `s3-backend-provisioner.md`) +3. Test with localstack/real AWS account +4. Add GCS and Azure provisioners (Phase 2) diff --git a/docs/prd/provisioner-system.md b/docs/prd/provisioner-system.md new file mode 100644 index 0000000000..d19ce299d4 --- /dev/null +++ b/docs/prd/provisioner-system.md @@ -0,0 +1,1274 @@ +# PRD: Provisioner System + +**Status:** Implemented +**Version:** 1.0 +**Last Updated:** 2025-11-19 +**Author:** Erik Osterman + +--- + +## Executive Summary + +The Provisioner System provides a generic, self-registering infrastructure for automatically provisioning resources needed by Atmos components. Provisioners declare when they need to run by registering themselves with specific hook events, creating a decentralized and extensible architecture. + +**Key Principle:** Each provisioner knows its own requirements and timing - the system provides discovery and execution infrastructure. + +--- + +## Overview + +### What is a Provisioner? + +A provisioner is a self-contained module that: +1. **Detects** if provisioning is needed (checks configuration) +2. **Provisions** required infrastructure (creates resources) +3. **Self-registers** with the hook system (declares timing) + +### Core Capabilities + +- **Self-Registration**: Provisioners declare when they need to run +- **Hook Integration**: Leverages existing hook system infrastructure +- **AuthContext Support**: Receives authentication from component config +- **Extensibility**: New provisioners added via simple registration +- **Discoverability**: Hook system queries "what runs at this event?" + +### First Implementation + +**Backend Provisioner** - Automatically provisions Terraform state backends (S3, GCS, Azure) before `terraform init`. See `backend-provisioner.md` for details. + +--- + +## Architecture + +### Provisioner Registration Pattern + +```go +// pkg/provisioner/provisioner.go + +package provisioner + +import ( + "github.com/cloudposse/atmos/pkg/hooks" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Provisioner defines a self-registering provisioner. +type Provisioner struct { + Type string // Provisioner type ("backend", "component", etc.) + HookEvent hooks.HookEvent // When to run (self-declared) + Func ProvisionerFunc // What to run +} + +// ProvisionerFunc is the function signature for all provisioners. +type ProvisionerFunc func( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error + +// Global registry: hook event → list of provisioners +var provisionersByEvent = make(map[hooks.HookEvent][]Provisioner) + +// RegisterProvisioner allows provisioners to self-register. +func RegisterProvisioner(p Provisioner) { + provisionersByEvent[p.HookEvent] = append( + provisionersByEvent[p.HookEvent], + p, + ) +} + +// GetProvisionersForEvent returns all provisioners registered for a hook event. +func GetProvisionersForEvent(event hooks.HookEvent) []Provisioner { + if provisioners, ok := provisionersByEvent[event]; ok { + return provisioners + } + return []Provisioner{} +} +``` + +### Self-Registration Example + +```go +// pkg/provisioner/backend/backend.go + +package backend + +import ( + "github.com/cloudposse/atmos/pkg/hooks" + "github.com/cloudposse/atmos/pkg/provisioner" +) + +func init() { + // Backend provisioner declares: "I need to run before terraform.init" + provisioner.RegisterProvisioner(provisioner.Provisioner{ + Type: "backend", + HookEvent: hooks.BeforeTerraformInit, // Self-declared timing + Func: ProvisionBackend, + }) +} + +func ProvisionBackend( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error { + // Check if provision.backend.enabled + if !isBackendProvisioningEnabled(componentSections) { + return nil + } + + // Provision backend (see backend-provisioner.md) + return provisionBackendInfrastructure(atmosConfig, componentSections, authContext) +} +``` + +--- + +## Hook System Integration + +### System Hook Execution + +```go +// pkg/hooks/system_hooks.go + +// ExecuteProvisionerHooks triggers all provisioners registered for a hook event. +func ExecuteProvisionerHooks( + event HookEvent, + atmosConfig *schema.AtmosConfiguration, + stackInfo *schema.ConfigAndStacksInfo, +) error { + provisioners := provisioner.GetProvisionersForEvent(event) + + for _, p := range provisioners { + // Check if this provisioner is enabled in component config + if shouldRunProvisioner(p.Type, stackInfo.ComponentSections) { + ui.Info(fmt.Sprintf("Running %s provisioner...", p.Type)) + + if err := p.Func( + atmosConfig, + &stackInfo.ComponentSections, + stackInfo.AuthContext, + ); err != nil { + return fmt.Errorf("provisioner '%s' failed: %w", p.Type, err) + } + } + } + + return nil +} + +// shouldRunProvisioner checks if provisioner is enabled in configuration. +func shouldRunProvisioner(provisionerType string, componentSections map[string]any) bool { + provisionConfig, ok := componentSections["provision"].(map[string]any) + if !ok { + return false + } + + typeConfig, ok := provisionConfig[provisionerType].(map[string]any) + if !ok { + return false + } + + enabled, ok := typeConfig["enabled"].(bool) + return ok && enabled +} +``` + +### Integration with Terraform Execution + +```go +// internal/exec/terraform.go + +func ExecuteTerraform(atmosConfig, stackInfo, ...) error { + // 1. Auth setup (existing system hook) + if err := auth.TerraformPreHook(atmosConfig, stackInfo); err != nil { + return err + } + + // 2. Provisioner system hooks (NEW) + if err := hooks.ExecuteProvisionerHooks( + hooks.BeforeTerraformInit, + atmosConfig, + stackInfo, + ); err != nil { + return err + } + + // 3. User-defined hooks (existing) + if err := hooks.ExecuteUserHooks(hooks.BeforeTerraformInit, ...); err != nil { + return err + } + + // 4. Terraform execution + return terraform.Init(...) +} +``` + +--- + +## Configuration Schema + +### Stack Manifest Configuration + +```yaml +# stacks/dev/us-east-1.yaml +components: + terraform: + vpc: + # Component authentication + auth: + providers: + aws: + type: aws-sso + identity: dev-admin + + # Provisioning configuration + provision: + backend: # Backend provisioner + enabled: true + + # Future provisioners: + # component: # Component provisioner + # vendor: true + # network: # Network provisioner + # vpc: true +``` + +### Schema Structure + +```yaml +provision: + : + enabled: boolean + # Provisioner-specific configuration +``` + +#### Key Points + +- `provision` block contains all provisioner configurations +- Each provisioner type has its own sub-block +- Provisioners check their own `enabled` flag +- Provisioner-specific options defined by implementation + +### Configuration Hierarchy and Deep-Merge + +The `provision` configuration follows Atmos's standard deep-merge behavior and can be specified at **any level** in the stack hierarchy. + +#### Top-Level Defaults + +Set provisioning defaults for all components: + +```yaml +# stacks/_defaults.yaml +terraform: + provision: + backend: + enabled: true +``` + +#### Environment-Specific Configuration + +Override at environment level: + +```yaml +# stacks/orgs/acme/plat/dev/_defaults.yaml +terraform: + provision: + backend: + enabled: true # Auto-provision in dev + +# stacks/orgs/acme/plat/prod/_defaults.yaml +terraform: + provision: + backend: + enabled: false # Pre-provisioned in prod +``` + +#### Component-Level Overrides + +Override for specific components: + +```yaml +components: + terraform: + vpc: + provision: + backend: + enabled: true + + eks: + provision: + backend: + enabled: false # Component-level override +``` + +#### Catalog Inheritance + +Share provision configuration through component catalogs: + +```yaml +# stacks/catalog/databases/defaults.yaml +components: + terraform: + rds/defaults: + provision: + backend: + enabled: true + +# stacks/prod.yaml +components: + terraform: + rds-primary: + metadata: + inherits: [rds/defaults] + # Inherits provision.backend.enabled: true + provision: + backend: + enabled: false # Override inherited value +``` + +#### Multi-Provisioner Configuration + +Configure multiple provisioners with different settings: + +```yaml +# Top-level: Enable backend provisioning everywhere +terraform: + provision: + backend: + enabled: true + +# Component: Enable specific provisioner types +components: + terraform: + app: + provision: + backend: + enabled: true # Inherited from top-level + component: # Future provisioner + vendor: true # Component-specific setting +``` + +**Implementation Note:** Provisioners receive the fully resolved `componentConfig` after deep-merge, so they automatically benefit from hierarchy without additional code. + +--- + +## AuthContext Integration + +### Authentication Flow + +```text +Component Definition (stack manifest) + ↓ +auth.providers.aws.identity: "dev-admin" + ↓ +TerraformPreHook (auth system hook) + ↓ +AuthContext populated with credentials + ↓ +ProvisionerHooks (receives AuthContext) + ↓ +Provisioners use AuthContext for cloud operations +``` + +### AuthContext Usage in Provisioners + +```go +func ProvisionBackend( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, // Populated by auth system +) error { + // Extract backend configuration + backendConfig := (*componentSections)["backend"].(map[string]any) + region := backendConfig["region"].(string) + + // Load AWS config using authContext (from component's identity) + cfg, err := awsUtils.LoadAWSConfigWithAuth( + ctx, + region, + "", // roleArn from backend config (if needed) + 15*time.Minute, + authContext.AWS, // Credentials from component's auth.identity + ) + + // Use config for provisioning + client := s3.NewFromConfig(cfg) + return provisionBucket(client, bucket) +} +``` + +### Identity Inheritance + +Provisioners inherit the component's identity: + +- Component defines `auth.providers.aws.identity: "dev-admin"` +- Auth system populates `AuthContext` +- Provisioners receive `AuthContext` automatically +- No separate provisioning identity needed + +Role assumption (if needed) extracted from provisioner-specific config: +- Backend: `backend.assume_role.role_arn` +- Component: `component.source.assume_role` (hypothetical) +- Each provisioner defines its own role assumption pattern + +--- + +## Package Structure + +```text +pkg/provisioner/ + ├── provisioner.go # Core registry and types + ├── provisioner_test.go # Registry tests + ├── backend/ # Backend provisioner + │ ├── backend.go # Backend provisioner implementation + │ ├── backend_test.go + │ ├── registry.go # Backend-specific registry (S3, GCS, Azure) + │ ├── s3.go # S3 backend provisioner + │ ├── gcs.go # GCS backend provisioner (future) + │ └── azurerm.go # Azure backend provisioner (future) + └── component/ # Future: Component provisioner + └── component.go + +pkg/hooks/ + ├── event.go # Hook event constants + ├── system_hooks.go # ExecuteProvisionerHooks() + └── hooks_test.go +``` + +--- + +## Adding a New Provisioner + +### Step 1: Define Provisioner Logic + +```go +// pkg/provisioner/myprovisioner/myprovisioner.go + +package myprovisioner + +import ( + "github.com/cloudposse/atmos/pkg/hooks" + "github.com/cloudposse/atmos/pkg/provisioner" + "github.com/cloudposse/atmos/pkg/schema" +) + +func init() { + // Self-register with hook system + provisioner.RegisterProvisioner(provisioner.Provisioner{ + Type: "myprovisioner", + HookEvent: hooks.BeforeComponentLoad, // Declare when to run + Func: ProvisionMyResource, + }) +} + +func ProvisionMyResource( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error { + // 1. Check if enabled + if !isEnabled(componentSections, "myprovisioner") { + return nil + } + + // 2. Extract configuration + config := extractConfig(componentSections) + + // 3. Provision resource + return provisionResource(config, authContext) +} + +func isEnabled(componentSections *map[string]any, provisionerType string) bool { + provisionConfig, ok := (*componentSections)["provision"].(map[string]any) + if !ok { + return false + } + + typeConfig, ok := provisionConfig[provisionerType].(map[string]any) + if !ok { + return false + } + + enabled, ok := typeConfig["enabled"].(bool) + return ok && enabled +} +``` + +### Step 2: Import Provisioner Package + +```go +// cmd/root.go or appropriate location + +import ( + _ "github.com/cloudposse/atmos/pkg/provisioner/backend" // Backend provisioner + _ "github.com/cloudposse/atmos/pkg/provisioner/myprovisioner" // Your provisioner +) +``` + +### Step 3: Configure in Stack Manifest + +```yaml +provision: + myprovisioner: + enabled: true + # Provisioner-specific configuration +``` + +**That's it!** The provisioner is now active. + +--- + +## Hook Events + +### Existing Hook Events (for reference) + +```go +// pkg/hooks/event.go + +const ( + BeforeTerraformPlan HookEvent = "before.terraform.plan" + AfterTerraformPlan HookEvent = "after.terraform.plan" + BeforeTerraformApply HookEvent = "before.terraform.apply" + AfterTerraformApply HookEvent = "after.terraform.apply" +) +``` + +### New Hook Events for Provisioners + +```go +const ( + // Provisioner hook events + BeforeTerraformInit HookEvent = "before.terraform.init" + AfterTerraformInit HookEvent = "after.terraform.init" + BeforeComponentLoad HookEvent = "before.component.load" + AfterComponentLoad HookEvent = "after.component.load" +) +``` + +**Provisioners can register for any hook event** - the system is fully extensible. + +--- + +## Testing Strategy + +### Unit Tests + +Registry Tests: + +```go +func TestRegisterProvisioner(t *testing.T) +func TestGetProvisionersForEvent(t *testing.T) +func TestGetProvisionersForEvent_NoProvisioners(t *testing.T) +func TestMultipleProvisionersForSameEvent(t *testing.T) +``` + +Hook Integration Tests: + +```go +func TestExecuteProvisionerHooks(t *testing.T) +func TestExecuteProvisionerHooks_ProvisionerDisabled(t *testing.T) +func TestExecuteProvisionerHooks_ProvisionerFails(t *testing.T) +func TestExecuteProvisionerHooks_MultipleProvisioners(t *testing.T) +``` + +### Integration Tests + +```go +func TestProvisionerSystemIntegration(t *testing.T) { + // Register test provisioner + provisioner.RegisterProvisioner(provisioner.Provisioner{ + Type: "test", + HookEvent: hooks.BeforeTerraformInit, + Func: func(atmosConfig, componentSections, authContext) error { + // Test provisioning logic + return nil + }, + }) + + // Execute hook system + err := hooks.ExecuteProvisionerHooks( + hooks.BeforeTerraformInit, + atmosConfig, + stackInfo, + ) + + assert.NoError(t, err) +} +``` + +--- + +## Future Provisioner Types + +### Component Provisioner + +**Purpose:** Auto-vendor components from remote sources + +**Hook Event:** `before.component.load` + +Configuration: + +```yaml +provision: + component: + vendor: true + source: "github.com/cloudposse/terraform-aws-components//modules/vpc" +``` + +Implementation: + +```go +func init() { + provisioner.RegisterProvisioner(provisioner.Provisioner{ + Type: "component", + HookEvent: hooks.BeforeComponentLoad, + Func: ProvisionComponent, + }) +} +``` + +### Network Provisioner + +**Purpose:** Auto-create VPCs/networks for testing + +**Hook Event:** `before.component.init` + +Configuration: + +```yaml +provision: + network: + vpc: true + cidr: "10.0.0.0/16" +``` + +### Workflow Provisioner + +**Purpose:** Auto-generate workflows from templates + +**Hook Event:** `before.workflow.execute` + +Configuration: + +```yaml +provision: + workflow: + template: "deploy-stack" +``` + +--- + +## Performance Considerations + +### Caching + +Provisioners should implement client caching: +```go +var clientCache sync.Map + +func getCachedClient(cacheKey string, authContext) (Client, error) { + if cached, ok := clientCache.Load(cacheKey); ok { + return cached.(Client), nil + } + + client := createClient(authContext) + clientCache.Store(cacheKey, client) + return client, nil +} +``` + +### Idempotency + +All provisioners must be idempotent: + +- Check if resource exists before creating +- Return nil (no error) if already provisioned +- Safe to run multiple times + +Example: + +```go +func ProvisionResource(config, authContext) error { + exists, err := checkResourceExists(config.Name) + if err != nil { + return err + } + + if exists { + ui.Info("Resource already exists (idempotent)") + return nil + } + + return createResource(config) +} +``` + +--- + +## Error Handling + +### Error Patterns + +```go +// Provisioner-specific errors +var ( + ErrProvisionerDisabled = errors.New("provisioner disabled") + ErrProvisionerFailed = errors.New("provisioner failed") + ErrResourceExists = errors.New("resource already exists") +) + +// Use error builder for detailed errors +func ProvisionResource() error { + return errUtils.Build(errUtils.ErrProvisionerFailed). + WithHint("Verify credentials have required permissions"). + WithContext("provisioner", "backend"). + WithContext("resource", "s3-bucket"). + WithExitCode(2). + Err() +} +``` + +--- + +## Security Considerations + +### AuthContext Requirements + +1. **Provisioners MUST use AuthContext** for cloud operations +2. **Never use ambient credentials** (environment variables, instance metadata) +3. **Respect component's identity** - don't override auth +4. **Role assumption** extracted from provisioner-specific config + +### Least Privilege + +Provisioners should: +- Document required IAM permissions +- Request minimal permissions +- Fail gracefully on permission denied +- Provide clear error messages with required permissions + +--- + +## Documentation Requirements + +### Each Provisioner Must Provide + +1. **Purpose** - What does it provision? +2. **Hook Event** - When does it run? +3. **Configuration** - What options are available? +4. **Requirements** - What permissions/dependencies needed? +5. **Examples** - Usage examples +6. **Migration** - How to migrate from manual provisioning + +--- + +## Success Metrics + +### Adoption Metrics + +- Number of provisioner types implemented +- Number of components using provisioners +- Provisioner invocation frequency + +### Performance Metrics + +- Provisioner execution time (p50, p95, p99) +- Cache hit rate +- Error rate per provisioner type + +### Quality Metrics + +- Test coverage for provisioner system (target: >90%) +- Test coverage per provisioner (target: >85%) +- Number of provisioner-related issues + +--- + +## CLI Commands for Provisioner Management + +### Overview + +Atmos provides dedicated CLI commands for managing provisioned resources throughout their lifecycle (SDLC): + +```bash +# Provision resources explicitly +atmos provision --stack + +# Examples +atmos provision backend vpc --stack dev +atmos provision component app --stack prod +atmos provision network vpc --stack test +``` + +### Command Structure + +**File:** `cmd/provision/provision.go` + +```go +// cmd/provision/provision.go + +package provision + +import ( + "github.com/spf13/cobra" + "github.com/cloudposse/atmos/cmd/internal/registry" +) + +type ProvisionCommandProvider struct{} + +func (p *ProvisionCommandProvider) ProvideCommands() []*cobra.Command { + return []*cobra.Command{ProvisionCmd} +} + +func (p *ProvisionCommandProvider) GetGroup() string { + return "Provisioning Commands" +} + +var ProvisionCmd = &cobra.Command{ + Use: "provision --stack ", + Short: "Provision infrastructure resources", + Long: `Provision infrastructure resources required by components. + +Provisioners create infrastructure that components depend on (backends, networks, etc.). +This command allows explicit provisioning outside of the automatic hook system. + +Examples: + # Provision S3 backend for vpc component + atmos provision backend vpc --stack dev + + # Provision network infrastructure + atmos provision network vpc --stack test + +Supported provisioner types: + backend - Provision Terraform state backends (S3, GCS, Azure) + component - Provision component dependencies (future) + network - Provision network infrastructure (future) +`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + provisionerType := args[0] + component := args[1] + stack, _ := cmd.Flags().GetString("stack") + + return exec.ExecuteProvision(provisionerType, component, stack) + }, +} + +func init() { + ProvisionCmd.Flags().StringP("stack", "s", "", "Atmos stack name (required)") + ProvisionCmd.MarkFlagRequired("stack") + + // Register with command registry + registry.Register(&ProvisionCommandProvider{}) +} +``` + +### Implementation: ExecuteProvision + +**File:** `internal/exec/provision.go` + +```go +// internal/exec/provision.go + +package exec + +import ( + "fmt" + + "github.com/cloudposse/atmos/pkg/provisioner" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui" +) + +// ExecuteProvision provisions infrastructure for a component. +func ExecuteProvision(provisionerType, component, stack string) error { + // 1. Load configuration and stacks + info, err := ProcessStacks(atmosConfig, stack, component, ...) + if err != nil { + return fmt.Errorf("failed to process stacks: %w", err) + } + + // 2. Setup authentication (TerraformPreHook) + if err := auth.TerraformPreHook(info.AtmosConfig, info); err != nil { + return fmt.Errorf("authentication failed: %w", err) + } + + // 3. Get provisioner for type + provisioners := provisioner.GetProvisionersForEvent(hooks.ManualProvision) + var targetProvisioner *provisioner.Provisioner + + for _, p := range provisioners { + if p.Type == provisionerType { + targetProvisioner = &p + break + } + } + + if targetProvisioner == nil { + return fmt.Errorf("provisioner type '%s' not found", provisionerType) + } + + // 4. Execute provisioner + ui.Info(fmt.Sprintf("Provisioning %s for component '%s' in stack '%s'...", + provisionerType, component, stack)) + + if err := targetProvisioner.Func( + info.AtmosConfig, + &info.ComponentSections, + info.AuthContext, + ); err != nil { + // CRITICAL: Propagate error to caller (exit with non-zero) + return fmt.Errorf("provisioning failed: %w", err) + } + + ui.Success(fmt.Sprintf("Successfully provisioned %s", provisionerType)) + return nil +} +``` + +### Usage Examples + +#### Explicit Backend Provisioning + +```bash +# Provision backend before applying +atmos provision backend vpc --stack dev + +# Then apply infrastructure +atmos terraform apply vpc --stack dev +``` + +#### Dry-Run Mode (Future Enhancement) + +```bash +# Preview what would be provisioned +atmos provision backend vpc --stack dev --dry-run + +# Output: +# Would create: +# - S3 bucket: acme-terraform-state-dev +# - Settings: versioning, encryption, public access block +# - Tags: ManagedBy=Atmos, Purpose=TerraformState +``` + +#### Multiple Components + +```bash +# Provision backend for multiple components +for comp in vpc eks rds; do + atmos provision backend $comp --stack dev +done +``` + +### Automatic vs Manual Provisioning + +Automatic (via hooks): + +```bash +# Provisioning happens automatically before terraform init +atmos terraform apply vpc --stack dev +# → TerraformPreHook (auth) +# → ProvisionerHook (backend provision if enabled) +# → terraform init +# → terraform apply +``` + +Manual (explicit command): + +```bash +# User explicitly provisions resources +atmos provision backend vpc --stack dev + +# Then runs terraform separately +atmos terraform apply vpc --stack dev +``` + +### When to Use Manual Provisioning + +1. **Separate provisioning step** - CI/CD pipelines with distinct stages +2. **Troubleshooting** - Isolate provisioning from application +3. **Batch operations** - Provision multiple backends at once +4. **Validation** - Verify provisioning without running terraform + +--- + +## Error Handling and Propagation + +### Error Handling Contract + +All provisioners MUST: + +1. Return `error` on failure (never panic) +2. Return `nil` on success or idempotent skip +3. Use wrapped errors with context +4. Provide actionable error messages + +### Error Propagation Flow + +```text +Provisioner fails + ↓ +Returns error with context + ↓ +Hook system catches error + ↓ +Propagates to main execution + ↓ +Atmos exits with non-zero code + ↓ +CI/CD pipeline fails +``` + +### Implementation: Hook System Error Handling + +```go +// pkg/hooks/system_hooks.go + +// ExecuteProvisionerHooks triggers all provisioners and PROPAGATES ERRORS. +func ExecuteProvisionerHooks( + event HookEvent, + atmosConfig *schema.AtmosConfiguration, + stackInfo *schema.ConfigAndStacksInfo, +) error { + provisioners := provisioner.GetProvisionersForEvent(event) + + for _, p := range provisioners { + if shouldRunProvisioner(p.Type, stackInfo.ComponentSections) { + ui.Info(fmt.Sprintf("Running %s provisioner...", p.Type)) + + // Execute provisioner + if err := p.Func( + atmosConfig, + &stackInfo.ComponentSections, + stackInfo.AuthContext, + ); err != nil { + // CRITICAL: Return error immediately (fail fast) + // Do NOT continue to next provisioner + ui.Error(fmt.Sprintf("Provisioner '%s' failed", p.Type)) + return fmt.Errorf("provisioner '%s' failed: %w", p.Type, err) + } + + ui.Success(fmt.Sprintf("Provisioner '%s' completed", p.Type)) + } + } + + return nil +} +``` + +### Error Examples + +Configuration Error: + +```go +if bucket == "" { + return fmt.Errorf("%w: bucket name is required in backend configuration", + errUtils.ErrInvalidConfig) +} +``` + +Provisioning Error: + +```go +if err := createBucket(bucket); err != nil { + return errUtils.Build(errUtils.ErrBackendProvision). + WithHint("Verify AWS credentials have s3:CreateBucket permission"). + WithContext("bucket", bucket). + WithContext("region", region). + WithExitCode(2). + Err() +} +``` + +Permission Error: + +```go +if isPermissionDenied(err) { + return errUtils.Build(errUtils.ErrBackendProvision). + WithHint("Required permissions: s3:CreateBucket, s3:PutBucketVersioning"). + WithHintf("Check IAM policy for identity: %s", authContext.AWS.Profile). + WithContext("action", "CreateBucket"). + WithContext("bucket", bucket). + WithExitCode(2). + Err() +} +``` + +### Exit Codes + +| Exit Code | Meaning | Example | +|-----------|---------|---------| +| 0 | Success or idempotent | Bucket already exists | +| 1 | General error | Unexpected failure | +| 2 | Configuration error | Missing required parameter | +| 3 | Permission error | IAM permission denied | +| 4 | Resource conflict | Bucket name already taken | + +### Terraform Execution Flow with Error Handling + +```go +// internal/exec/terraform.go + +func ExecuteTerraform(atmosConfig, stackInfo, ...) error { + // 1. Auth setup + if err := auth.TerraformPreHook(atmosConfig, stackInfo); err != nil { + return err // Fail fast - auth required + } + + // 2. Provisioner hooks + if err := hooks.ExecuteProvisionerHooks( + hooks.BeforeTerraformInit, + atmosConfig, + stackInfo, + ); err != nil { + // CRITICAL: If provisioning fails, DO NOT continue to terraform + ui.Error("Provisioning failed - cannot proceed with terraform") + return fmt.Errorf("provisioning failed: %w", err) + } + + // 3. User hooks + if err := hooks.ExecuteUserHooks(hooks.BeforeTerraformInit, ...); err != nil { + return err + } + + // 4. Terraform execution (only if provisioning succeeded) + return terraform.Init(...) +} +``` + +### CI/CD Integration + +GitHub Actions Example: + +```yaml +- name: Provision Backend + run: atmos provision backend vpc --stack dev + # If provisioning fails, pipeline stops here (exit code != 0) + +- name: Apply Infrastructure + run: atmos terraform apply vpc --stack dev + # Only runs if previous step succeeded +``` + +Error Output: + +```text +Error: provisioner 'backend' failed: backend provisioning failed: +failed to create bucket: operation error S3: CreateBucket, +https response error StatusCode: 403, AccessDenied + +Hint: Verify AWS credentials have s3:CreateBucket permission +Context: bucket=acme-terraform-state-dev, region=us-east-1 + +Exit code: 3 +``` + +--- + +## Related Documents + +- **[Backend Provisioner](./backend-provisioner.md)** - Backend provisioner interface and registry +- **[S3 Backend Provisioner](./s3-backend-provisioner.md)** - S3 backend implementation (reference implementation) + +--- + +## Appendix: Complete Example + +### Provisioner Implementation + +```go +// pkg/provisioner/example/example.go + +package example + +import ( + "github.com/cloudposse/atmos/pkg/hooks" + "github.com/cloudposse/atmos/pkg/provisioner" + "github.com/cloudposse/atmos/pkg/schema" +) + +func init() { + provisioner.RegisterProvisioner(provisioner.Provisioner{ + Type: "example", + HookEvent: hooks.BeforeTerraformInit, + Func: ProvisionExample, + }) +} + +func ProvisionExample( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error { + // 1. Check if enabled + provisionConfig, ok := (*componentSections)["provision"].(map[string]any) + if !ok { + return nil + } + + exampleConfig, ok := provisionConfig["example"].(map[string]any) + if !ok { + return nil + } + + enabled, ok := exampleConfig["enabled"].(bool) + if !ok || !enabled { + return nil + } + + // 2. Extract configuration + resourceName := exampleConfig["name"].(string) + + // 3. Check if already exists (idempotent) + exists, err := checkResourceExists(resourceName, authContext) + if err != nil { + return fmt.Errorf("failed to check resource: %w", err) + } + + if exists { + ui.Info(fmt.Sprintf("Resource '%s' already exists", resourceName)) + return nil + } + + // 4. Provision resource + ui.Info(fmt.Sprintf("Provisioning resource '%s'...", resourceName)) + if err := createResource(resourceName, authContext); err != nil { + return fmt.Errorf("failed to create resource: %w", err) + } + + ui.Success(fmt.Sprintf("Resource '%s' provisioned successfully", resourceName)) + return nil +} +``` + +### Stack Configuration + +```yaml +components: + terraform: + myapp: + auth: + providers: + aws: + type: aws-sso + identity: dev-admin + + provision: + example: + enabled: true + name: "my-example-resource" +``` + +--- + +## Conclusion + +This PRD has been implemented. The provisioner system is now part of Atmos. + +### Next Steps + +1. Review provisioner system architecture +2. Implement core registry (`pkg/provisioner/provisioner.go`) +3. Integrate with hook system (`pkg/hooks/system_hooks.go`) +4. See `backend-provisioner.md` for first provisioner implementation diff --git a/docs/prd/s3-backend-provisioner.md b/docs/prd/s3-backend-provisioner.md new file mode 100644 index 0000000000..30c535fd86 --- /dev/null +++ b/docs/prd/s3-backend-provisioner.md @@ -0,0 +1,1447 @@ +# PRD: S3 Backend Provisioner + +**Status:** Draft for Review +**Version:** 1.0 +**Last Updated:** 2025-11-19 +**Author:** Erik Osterman + +--- + +## Executive Summary + +The S3 Backend Provisioner automatically creates AWS S3 buckets for Terraform state storage with secure defaults. It's the reference implementation of the Backend Provisioner interface and eliminates cold-start friction for development and testing environments. + +**Key Principle:** Simple, opinionated S3 buckets with AWS best practices - not production-ready infrastructure. + +--- + +## Problem Statement + +### Current Pain Points + +1. **Manual Bucket Creation**: Users must create S3 buckets before running `terraform init` +2. **Inconsistent Security**: Manual bucket creation leads to varying security settings +3. **Onboarding Friction**: New developers need AWS console access or separate scripts +4. **Cold Start Delay**: Setting up new environments requires multiple manual steps + +### Target Users + +- **Development Teams**: Quick environment setup for testing +- **New Users**: First-time Terraform/Atmos users learning the system +- **CI/CD Pipelines**: Ephemeral environments that need automatic backend creation +- **POCs/Demos**: Rapid prototyping without infrastructure overhead + +### Non-Target Users + +- **Production Environments**: Should use `terraform-aws-tfstate-backend` module for: + - Custom KMS encryption + - Cross-region replication + - DynamoDB state locking + - Advanced lifecycle policies + - Compliance requirements (HIPAA, SOC2, etc.) + +--- + +## Goals & Non-Goals + +### Goals + +1. ✅ **Automatic S3 Bucket Creation**: Create bucket if doesn't exist +2. ✅ **Secure Defaults**: Encryption, versioning, public access blocking (always enabled) +3. ✅ **Idempotent Operations**: Safe to run multiple times +4. ✅ **Cross-Account Support**: Provision buckets via role assumption +5. ✅ **Zero Configuration**: No options beyond `enabled: true` +6. ✅ **Fast Implementation**: ~1 week timeline +7. ✅ **Backend Deletion**: Delete backend infrastructure with safety checks + +### Non-Goals + +1. ❌ **DynamoDB Tables**: Use Terraform 1.10+ native S3 locking +2. ❌ **Custom KMS Keys**: Use AWS-managed encryption +3. ❌ **Replication**: No cross-region bucket replication +4. ❌ **Lifecycle Policies**: No object expiration/transitions +5. ❌ **Access Logging**: No S3 access logs +6. ❌ **Production Features**: Not competing with terraform-aws-tfstate-backend module + +--- + +## What Gets Created + +### S3 Bucket with Hardcoded Best Practices + +When `provision.backend.enabled: true` and bucket doesn't exist: + +#### Always Enabled (No Configuration) + +1. **Versioning**: Enabled for state file recovery +2. **Encryption**: Server-side encryption with SSE-S3 (AES-256, AWS-managed keys) +3. **Public Access**: All 4 public access settings blocked +4. **Resource Tags**: + - `ManagedBy: Atmos` + - `Name: ` + +> **Note**: Bucket Key is not enabled because it only applies to SSE-KMS encryption. +> The implementation uses SSE-S3 (AES-256) for simplicity and zero additional cost. + +#### NOT Created + +- ❌ DynamoDB table (Terraform 1.10+ has native S3 state locking) +- ❌ Custom KMS key +- ❌ Replication configuration +- ❌ Lifecycle rules +- ❌ Access logging bucket +- ❌ Object lock/WORM +- ❌ Bucket policies (beyond public access block) + +--- + +## Backend Deletion + +### Delete Command + +The `atmos terraform backend delete` command permanently removes backend infrastructure. + +```shell +# Delete empty backend +atmos terraform backend delete vpc --stack dev --force + +# Command will error if bucket contains objects (unless --force) +``` + +### Safety Mechanisms + +#### Force Flag Required + +The `--force` flag is **always required** for deletion to prevent accidental removal: + +```shell +# This command requires --force +atmos terraform backend delete vpc --stack dev --force +``` + +#### Non-Empty Bucket Handling + +The `--force` flag is always required. When provided, the command: + +- Lists all objects and versions in bucket +- Shows count of objects and state files to be deleted +- Displays warning if `.tfstate` files are present +- Deletes all objects (including versions) +- Deletes the bucket itself +- Operation is irreversible + +**Without `--force` flag:** +- Command exits with error: "the --force flag is required for deletion" +- No bucket inspection or deletion occurs + +### Delete Process + +When you run `atmos terraform backend delete --force`: + +1. **Validate Configuration** - Load component's stack configuration +2. **Check Backend Type** - Verify supported backend type (s3, gcs, azurerm) +3. **List Objects** - Enumerate all objects and versions in bucket +4. **Detect State Files** - Count `.tfstate` files for warning message +5. **Warn User** - Display count of objects and state files to be deleted +6. **Delete Objects** - Remove all objects and versions (batch operations) +7. **Delete Bucket** - Remove the empty bucket +8. **Confirm Success** - Report completion + +### Error Scenarios + +- **Bucket Not Found**: Error if backend doesn't exist +- **Permission Denied**: AWS IAM permissions insufficient +- **Deletion Failure**: Partial delete (objects removed but bucket remains) +- **Force Required**: User didn't provide `--force` flag + +### Best Practices + +1. **Backup State Files**: Download `.tfstate` files before deletion +2. **Verify Component**: Use `describe` to confirm correct backend +3. **Check Stack**: Ensure you're targeting the right environment +4. **Document Deletion**: Record why backend was deleted +5. **Cross-Account**: Ensure role assumption permissions for delete operations + +### What Gets Deleted + +- ✅ S3 bucket and all objects +- ✅ All object versions (if versioning enabled) +- ✅ Terraform state files (`.tfstate`) +- ✅ Delete markers +- ❌ DynamoDB tables (not created by provisioner) +- ❌ KMS keys (not created by provisioner) +- ❌ IAM roles/policies (not created by provisioner) + +--- + +## Configuration + +### Stack Manifest Example + +```yaml +# stacks/dev/us-east-1.yaml +components: + terraform: + vpc: + # Component authentication + auth: + providers: + aws: + type: aws-sso + identity: dev-admin + + # Backend configuration (standard Terraform) + backend_type: s3 + backend: + bucket: acme-terraform-state-dev-use1 + key: vpc/terraform.tfstate + region: us-east-1 + encrypt: true + + # Provisioning configuration (Atmos-only) + provision: + backend: + enabled: true # Enable automatic S3 bucket creation +``` + +### Cross-Account Provisioning + +```yaml +components: + terraform: + vpc: + # Source account identity + auth: + providers: + aws: + type: aws-sso + identity: dev-admin # Credentials in account 111111111111 + + # Target account backend + backend_type: s3 + backend: + bucket: prod-terraform-state + key: vpc/terraform.tfstate + region: us-east-1 + + # Assume role in target account (standard Terraform syntax) + assume_role: + role_arn: "arn:aws:iam::999999999999:role/TerraformStateAdmin" + session_name: "atmos-backend-provision" + + # Enable provisioning + provision: + backend: + enabled: true +``` + +**Flow:** +1. Authenticate as `dev-admin` in account 111111111111 +2. Assume `TerraformStateAdmin` role in account 999999999999 +3. Create S3 bucket in account 999999999999 + +### Multi-Environment Setup with Inheritance + +Leverage Atmos's deep-merge system to configure provisioning at different hierarchy levels: + +#### Organization-Level Defaults + +Enable provisioning for all development and staging environments: + +```yaml +# stacks/orgs/acme/_defaults.yaml +terraform: + backend_type: s3 + backend: + region: us-east-1 + encrypt: true + +# stacks/orgs/acme/plat/dev/_defaults.yaml +terraform: + backend: + bucket: acme-terraform-state-dev # Dev bucket + provision: + backend: + enabled: true # Auto-provision in dev + +# stacks/orgs/acme/plat/staging/_defaults.yaml +terraform: + backend: + bucket: acme-terraform-state-staging # Staging bucket + provision: + backend: + enabled: true # Auto-provision in staging + +# stacks/orgs/acme/plat/prod/_defaults.yaml +terraform: + backend: + bucket: acme-terraform-state-prod # Prod bucket + provision: + backend: + enabled: false # Pre-provisioned in prod (managed by Terraform) +``` + +#### Catalog Inheritance Pattern + +Share provision configuration through component catalogs: + +```yaml +# stacks/catalog/networking/vpc.yaml +components: + terraform: + vpc/defaults: + backend_type: s3 + backend: + key: vpc/terraform.tfstate + region: us-east-1 + provision: + backend: + enabled: true # Default: auto-provision + +# stacks/dev/us-east-1.yaml +components: + terraform: + vpc-dev: + metadata: + inherits: [vpc/defaults] + # Inherits provision.backend.enabled: true + backend: + bucket: acme-terraform-state-dev # Dev-specific bucket + +# stacks/prod/us-east-1.yaml +components: + terraform: + vpc-prod: + metadata: + inherits: [vpc/defaults] + backend: + bucket: acme-terraform-state-prod # Prod-specific bucket + provision: + backend: + enabled: false # Override: disable for production +``` + +#### Per-Component Override + +Override provisioning for specific components: + +```yaml +# stacks/dev/us-east-1.yaml +components: + terraform: + # VPC uses auto-provisioning (inherits from environment defaults) + vpc: + backend: + bucket: acme-terraform-state-dev + key: vpc/terraform.tfstate + # provision.backend.enabled: true (inherited) + + # EKS explicitly disables auto-provisioning + eks: + backend: + bucket: acme-terraform-state-dev + key: eks/terraform.tfstate + provision: + backend: + enabled: false # Component-level override +``` + +**Benefits of Hierarchy:** +- **DRY**: Configure once at organization/environment level +- **Flexibility**: Override per component when needed +- **Consistency**: All dev environments auto-provision, all prod environments use pre-provisioned backends +- **Maintainability**: Change provisioning policy in one place + +--- + +## Implementation + +### Package Structure + +```text +pkg/provisioner/backend/ + ├── s3.go # S3 backend provisioner + ├── s3_test.go # Unit tests + ├── s3_integration_test.go # Integration tests +``` + +### Core Implementation + +```go +// pkg/provisioner/backend/s3.go + +package backend + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + + awsUtils "github.com/cloudposse/atmos/internal/aws" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui" +) + +// S3 client cache (performance optimization) +var s3ProvisionerClientCache sync.Map + +// ProvisionS3Backend provisions an S3 bucket for Terraform state. +func ProvisionS3Backend( + atmosConfig *schema.AtmosConfiguration, + componentSections *map[string]any, + authContext *schema.AuthContext, +) error { + defer perf.Track(atmosConfig, "provisioner.backend.ProvisionS3Backend")() + + // 1. Extract backend configuration + backendConfig, ok := (*componentSections)["backend"].(map[string]any) + if !ok { + return fmt.Errorf("backend configuration not found") + } + + bucket, ok := backendConfig["bucket"].(string) + if !ok || bucket == "" { + return fmt.Errorf("bucket name is required in backend configuration") + } + + region, ok := backendConfig["region"].(string) + if !ok || region == "" { + return fmt.Errorf("region is required in backend configuration") + } + + // 2. Get or create S3 client (with role assumption if needed) + client, err := getCachedS3ProvisionerClient(region, &backendConfig, authContext) + if err != nil { + return fmt.Errorf("failed to create S3 client: %w", err) + } + + // 3. Check if bucket exists (idempotent) + ctx := context.Background() + exists, err := checkS3BucketExists(ctx, client, bucket) + if err != nil { + return fmt.Errorf("failed to check bucket existence: %w", err) + } + + if exists { + ui.Info(fmt.Sprintf("S3 bucket '%s' already exists (idempotent)", bucket)) + return nil + } + + // 4. Create bucket with hardcoded best practices + ui.Info(fmt.Sprintf("Creating S3 bucket '%s' with secure defaults...", bucket)) + if err := provisionS3BucketWithDefaults(ctx, client, bucket, region); err != nil { + return fmt.Errorf("failed to provision S3 bucket: %w", err) + } + + ui.Success(fmt.Sprintf("Successfully created S3 bucket '%s'", bucket)) + return nil +} + +// getCachedS3ProvisionerClient returns a cached or new S3 client. +func getCachedS3ProvisionerClient( + region string, + backendConfig *map[string]any, + authContext *schema.AuthContext, +) (*s3.Client, error) { + defer perf.Track(nil, "provisioner.backend.getCachedS3ProvisionerClient")() + + // Extract role ARN if specified + roleArn := GetS3BackendAssumeRoleArn(backendConfig) + + // Build deterministic cache key + cacheKey := fmt.Sprintf("region=%s", region) + if authContext != nil && authContext.AWS != nil { + cacheKey += fmt.Sprintf(";profile=%s", authContext.AWS.Profile) + } + if roleArn != "" { + cacheKey += fmt.Sprintf(";role=%s", roleArn) + } + + // Check cache + if cached, ok := s3ProvisionerClientCache.Load(cacheKey); ok { + return cached.(*s3.Client), nil + } + + // Create new client with timeout + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Load AWS config with auth context + role assumption + cfg, err := awsUtils.LoadAWSConfigWithAuth( + ctx, + region, + roleArn, + 15*time.Minute, + authContext.AWS, + ) + if err != nil { + return nil, err + } + + // Create S3 client + client := s3.NewFromConfig(cfg) + s3ProvisionerClientCache.Store(cacheKey, client) + return client, nil +} + +// checkS3BucketExists checks if an S3 bucket exists. +// Returns: +// - (true, nil) if bucket exists and is accessible +// - (false, nil) if bucket does not exist (404/NotFound) +// - (false, error) if access denied (403) or other errors occur +func checkS3BucketExists(ctx context.Context, client *s3.Client, bucket string) (bool, error) { + defer perf.Track(nil, "provisioner.backend.checkS3BucketExists")() + + _, err := client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucket), + }) + + if err != nil { + // Check for specific error types to distinguish between "not found" and "access denied". + var notFound *s3types.NotFound + var noSuchBucket *s3types.NoSuchBucket + if errors.As(err, ¬Found) || errors.As(err, &noSuchBucket) { + // Bucket genuinely doesn't exist - safe to proceed with creation. + return false, nil + } + // For AccessDenied (403) or other errors, return the error. + // This prevents attempting to create a bucket we can't access. + return false, fmt.Errorf("failed to check bucket existence: %w", err) + } + + return true, nil +} + +// provisionS3BucketWithDefaults creates an S3 bucket with hardcoded best practices. +func provisionS3BucketWithDefaults( + ctx context.Context, + client *s3.Client, + bucket, region string, +) error { + defer perf.Track(nil, "provisioner.backend.provisionS3BucketWithDefaults")() + + // 1. Create bucket + createInput := &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + } + + // For regions other than us-east-1, must specify location constraint + if region != "us-east-1" { + createInput.CreateBucketConfiguration = &s3types.CreateBucketConfiguration{ + LocationConstraint: s3types.BucketLocationConstraint(region), + } + } + + if _, err := client.CreateBucket(ctx, createInput); err != nil { + return fmt.Errorf("failed to create bucket: %w", err) + } + + // Wait for bucket to be available (eventual consistency) + time.Sleep(2 * time.Second) + + // 2. Enable versioning (ALWAYS) + ui.Info("Enabling bucket versioning...") + if _, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &s3types.VersioningConfiguration{ + Status: s3types.BucketVersioningStatusEnabled, + }, + }); err != nil { + return fmt.Errorf("failed to enable versioning: %w", err) + } + + // 3. Enable encryption (ALWAYS - AES-256) + ui.Info("Enabling bucket encryption (AES-256)...") + if _, err := client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(bucket), + ServerSideEncryptionConfiguration: &s3types.ServerSideEncryptionConfiguration{ + Rules: []s3types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &s3types.ServerSideEncryptionByDefault{ + SSEAlgorithm: s3types.ServerSideEncryptionAes256, + }, + BucketKeyEnabled: aws.Bool(true), + }, + }, + }, + }); err != nil { + return fmt.Errorf("failed to enable encryption: %w", err) + } + + // 4. Block public access (ALWAYS) + ui.Info("Blocking public access...") + if _, err := client.PutPublicAccessBlock(ctx, &s3.PutPublicAccessBlockInput{ + Bucket: aws.String(bucket), + PublicAccessBlockConfiguration: &s3types.PublicAccessBlockConfiguration{ + BlockPublicAcls: aws.Bool(true), + BlockPublicPolicy: aws.Bool(true), + IgnorePublicAcls: aws.Bool(true), + RestrictPublicBuckets: aws.Bool(true), + }, + }); err != nil { + return fmt.Errorf("failed to block public access: %w", err) + } + + // 5. Apply standard tags (ALWAYS) + ui.Info("Applying resource tags...") + if _, err := client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{ + Bucket: aws.String(bucket), + Tagging: &s3types.Tagging{ + TagSet: []s3types.Tag{ + {Key: aws.String("ManagedBy"), Value: aws.String("Atmos")}, + {Key: aws.String("CreatedAt"), Value: aws.String(time.Now().Format(time.RFC3339))}, + {Key: aws.String("Purpose"), Value: aws.String("TerraformState")}, + }, + }, + }); err != nil { + return fmt.Errorf("failed to apply tags: %w", err) + } + + return nil +} + +// GetS3BackendAssumeRoleArn extracts role ARN from backend config (standard Terraform syntax). +func GetS3BackendAssumeRoleArn(backend *map[string]any) string { + // Try assume_role block first (standard Terraform) + if assumeRoleSection, ok := (*backend)["assume_role"].(map[string]any); ok { + if roleArn, ok := assumeRoleSection["role_arn"].(string); ok && roleArn != "" { + return roleArn + } + } + + // Fallback to top-level role_arn (legacy) + if roleArn, ok := (*backend)["role_arn"].(string); ok && roleArn != "" { + return roleArn + } + + return "" +} +``` + +--- + +## Testing Strategy + +### Unit Tests + +**File:** `pkg/provisioner/backend/s3_test.go` + +```go +func TestProvisionS3Backend_NewBucket(t *testing.T) { + // Test: Bucket doesn't exist → create bucket with all settings +} + +func TestProvisionS3Backend_ExistingBucket(t *testing.T) { + // Test: Bucket exists → return nil (idempotent) +} + +func TestProvisionS3Backend_InvalidConfig(t *testing.T) { + // Test: Missing bucket/region → return error +} + +func TestProvisionS3Backend_RoleAssumption(t *testing.T) { + // Test: Role ARN specified → assume role and create bucket +} + +func TestCheckS3BucketExists(t *testing.T) { + // Test: HeadBucket returns 200 → true + // Test: HeadBucket returns 404 → false +} + +func TestProvisionS3BucketWithDefaults(t *testing.T) { + // Test: All bucket settings applied correctly + // Test: Versioning enabled + // Test: Encryption enabled + // Test: Public access blocked + // Test: Tags applied +} + +func TestGetCachedS3ProvisionerClient(t *testing.T) { + // Test: Client cached and reused + // Test: Different cache key per region/profile/role +} + +func TestGetS3BackendAssumeRoleArn(t *testing.T) { + // Test: Extract from assume_role.role_arn + // Test: Fallback to top-level role_arn + // Test: Return empty string if not specified +} +``` + +**Mocking Strategy:** +- Use `go.uber.org/mock/mockgen` for AWS SDK interfaces +- Mock S3 client for unit tests +- Table-driven tests for configuration variants + +### Integration Tests + +**File:** `pkg/provisioner/backend/s3_integration_test.go` + +```go +func TestS3BackendProvisioning_Localstack(t *testing.T) { + // Requires: Docker with localstack + tests.RequireLocalstack(t) + + // Create S3 bucket via provisioner + // Verify bucket exists + // Verify versioning enabled + // Verify encryption enabled + // Verify public access blocked + // Verify tags applied +} + +func TestS3BackendProvisioning_RealAWS(t *testing.T) { + // Requires: Real AWS account with credentials + tests.RequireAWSAccess(t) + + // Create unique bucket name + bucket := fmt.Sprintf("atmos-test-%s", randomString()) + + // Provision bucket + // Verify bucket created with all settings + // Cleanup: delete bucket +} + +func TestS3BackendProvisioning_Idempotent(t *testing.T) { + // Create bucket first + // Run provisioner again + // Verify no error (idempotent) +} +``` + +**Test Infrastructure:** +- Docker Compose with localstack for local testing +- Real AWS account for integration tests (optional) +- Cleanup helpers to delete test buckets + +### Manual Testing Checklist + +- [ ] Fresh AWS account (verify bucket created) +- [ ] Existing bucket (verify idempotent, no errors) +- [ ] Cross-region bucket creation (us-east-1, us-west-2, eu-west-1) +- [ ] Cross-account provisioning (role assumption) +- [ ] Permission denied (verify clear error message) +- [ ] Invalid bucket name (verify error handling) +- [ ] Bucket name conflict (globally unique names) +- [ ] Integration with `atmos terraform init` +- [ ] Integration with `atmos terraform apply` + +--- + +## Security + +### Required IAM Permissions + +**Minimal permissions for S3 backend provisioning:** + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "S3BackendProvisioning", + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:HeadBucket", + "s3:PutBucketVersioning", + "s3:GetBucketVersioning", + "s3:PutBucketEncryption", + "s3:GetBucketEncryption", + "s3:PutBucketPublicAccessBlock", + "s3:GetBucketPublicAccessBlock", + "s3:PutBucketTagging", + "s3:GetBucketTagging" + ], + "Resource": "arn:aws:s3:::*-terraform-state-*" + } + ] +} +``` + +**Cross-account role trust policy:** + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::111111111111:role/DevAdminRole" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "sts:ExternalId": "atmos-backend-provision" + } + } + } + ] +} +``` + +### Security Best Practices (Hardcoded) + +Every auto-provisioned S3 bucket includes: + +1. **Encryption at Rest**: Server-side encryption with AES-256 +2. **Versioning**: Enabled for state file recovery +3. **Public Access**: All 4 settings blocked +4. **Bucket Key**: Enabled for cost reduction +5. **Resource Tags**: Tracking and attribution + +**What's NOT Included (Use terraform-aws-tfstate-backend for Production):** +- ❌ Custom KMS keys +- ❌ Access logging +- ❌ Object lock (WORM) +- ❌ MFA delete +- ❌ Bucket policies (beyond public access) +- ❌ Lifecycle rules +- ❌ Replication + +--- + +## Error Handling + +### Common Errors and Solutions + +#### 1. Bucket Name Already Taken + +**Error:** +```text +failed to provision S3 bucket: BucketAlreadyExists: The requested bucket name is not available +``` + +**Cause:** S3 bucket names are globally unique across all AWS accounts. + +**Solution:** +```yaml +# Use more specific bucket name +backend: + bucket: acme-terraform-state-dev-12345678 # Add account ID or random suffix +``` + +#### 2. Permission Denied + +**Error:** +```text +failed to create S3 client: operation error HeadBucket: AccessDenied +``` + +**Cause:** IAM identity lacks required S3 permissions. + +**Solution:** +- Attach IAM policy with required permissions (see Security section) +- Verify identity: `aws sts get-caller-identity` +- Check CloudTrail for specific permission denied + +#### 3. Invalid Region + +**Error:** +```text +failed to create bucket: InvalidLocationConstraint +``` + +**Cause:** Region specified doesn't exist or is invalid. + +**Solution:** +```yaml +backend: + region: us-east-1 # Use valid AWS region +``` + +#### 4. Cross-Account Role Assumption Failed + +**Error:** +```text +failed to create S3 client: operation error STS: AssumeRole, AccessDenied +``` + +**Cause:** Trust policy doesn't allow source identity to assume role. + +**Solution:** +- Verify trust policy allows source account/role +- Check external ID if required +- Verify role ARN is correct + +--- + +## Migration Guide + +### Enabling S3 Backend Provisioning + +**Step 1: Enable global feature flag (optional)** + +```yaml +# atmos.yaml +settings: + backends: + auto_provision: + enabled: true +``` + +**Step 2: Enable per-component** + +```yaml +# stacks/dev.yaml +components: + terraform: + vpc: + backend: + bucket: acme-terraform-state-dev + key: vpc/terraform.tfstate + region: us-east-1 + + provision: # ADD THIS + backend: + enabled: true +``` + +**Step 3: Generate backend** + +```bash +atmos terraform generate backend vpc --stack dev +``` + +**What happens:** +1. Atmos checks if bucket exists +2. Bucket doesn't exist → creates with secure defaults +3. Generates `backend.tf.json` +4. Ready for `terraform init` + +### Upgrading to Production Backend + +**Scenario:** Moving from auto-provisioned dev bucket to production-grade backend. + +**Step 1: Provision production backend via Terraform module** + +```yaml +# stacks/prod.yaml +components: + terraform: + # Provision production backend first + prod-backend: + component: terraform-aws-tfstate-backend + backend_type: local # Bootstrap with local state + backend: + path: ./local-state/backend.tfstate + + vars: + bucket: acme-terraform-state-prod + dynamodb_table: terraform-locks-prod + s3_replication_enabled: true + s3_replica_bucket_arn: "arn:aws:s3:::acme-terraform-state-prod-dr" + enable_point_in_time_recovery: true + sse_algorithm: "aws:kms" + kms_master_key_id: "arn:aws:kms:us-east-1:123456789012:key/..." +``` + +**Step 2: Apply backend infrastructure** + +```bash +atmos terraform apply prod-backend --stack prod +``` + +**Step 3: Update component to use production backend** + +```yaml +# stacks/prod.yaml +components: + terraform: + vpc: + backend: + bucket: acme-terraform-state-prod # New production bucket + key: vpc/terraform.tfstate + dynamodb_table: terraform-locks-prod + kms_key_id: "arn:aws:kms:us-east-1:123456789012:key/..." + # Remove provision block - backend already exists +``` + +**Step 4: Migrate state** + +```bash +# Re-initialize with new backend +atmos terraform init vpc --stack prod -migrate-state + +# Verify migration +atmos terraform state list vpc --stack prod +``` + +**Step 5: (Optional) Delete old auto-provisioned bucket** + +```bash +# Only after confirming migration successful +aws s3 rb s3://acme-dev-state --force +``` + +--- + +## Performance Benchmarks + +### Target Metrics + +- **Bucket existence check**: <2 seconds +- **Bucket creation**: <30 seconds (including all settings) +- **Total provisioning time**: <1 minute +- **Cache hit rate**: >80% for repeated operations + +### Optimization Strategies + +1. **Client Caching**: Reuse S3 clients across operations +2. **Concurrent Settings**: Apply bucket settings in parallel (future optimization) +3. **Retry with Backoff**: Handle transient AWS API failures +4. **Context Timeouts**: Prevent hanging on slow API calls + +--- + +## FAQ + +### Q: Why not support DynamoDB table provisioning? + +**A:** Terraform 1.10+ includes native S3 state locking, eliminating the need for DynamoDB. For users requiring DynamoDB (Terraform <1.10 or advanced features like point-in-time recovery), use the `terraform-aws-tfstate-backend` module. + +### Q: Can I use custom KMS keys? + +**A:** Not with auto-provisioning. Auto-provisioning uses AWS-managed encryption keys for simplicity. For custom KMS keys, use the `terraform-aws-tfstate-backend` module. + +### Q: Is auto-provisioned bucket suitable for production? + +**A:** No. Auto-provisioning is designed for development and testing. Production backends should use the `terraform-aws-tfstate-backend` Terraform module for advanced features like replication, custom KMS, lifecycle policies, and compliance controls. + +### Q: What happens if bucket name is already taken? + +**A:** S3 bucket names are globally unique across all AWS accounts. If the name is taken, you'll receive a clear error message. Use a more specific name (e.g., add account ID or random suffix). + +### Q: Can I migrate from auto-provisioned to production backend? + +**A:** Yes. Provision your production backend using the `terraform-aws-tfstate-backend` module, update your stack manifest, then run `terraform init -migrate-state`. See Migration Guide for detailed steps. + +### Q: Does this work with Terraform Cloud? + +**A:** The `cloud` backend type doesn't require provisioning (Terraform Cloud manages storage). Auto-provisioning only applies to self-managed backends (S3, GCS, Azure). + +### Q: What permissions are required? + +**A:** Minimal permissions: `s3:CreateBucket`, `s3:HeadBucket`, `s3:PutBucketVersioning`, `s3:PutBucketEncryption`, `s3:PutBucketPublicAccessBlock`, `s3:PutBucketTagging`. See Security section for complete IAM policy. + +### Q: Can I provision buckets in different AWS accounts? + +**A:** Yes, using role assumption. Configure `backend.assume_role.role_arn` to specify the target account role. The provisioner will assume the role and create the bucket in the target account. + +### Q: What if I already have a bucket? + +**A:** The provisioner is idempotent - if the bucket already exists, it returns without error. It will NOT modify existing bucket settings. + +--- + +## CLI Usage + +### Automatic Provisioning (Recommended) + +Backend provisioned automatically when running Terraform commands: + +```bash +# Backend provisioned automatically if provision.backend.enabled: true +atmos terraform apply vpc --stack dev + +# Execution flow: +# 1. Auth setup (TerraformPreHook) +# 2. Backend provisioning (if enabled) ← Automatic +# 3. Terraform init +# 4. Terraform apply +``` + +### Manual Provisioning + +Explicitly provision backend before Terraform execution: + +```bash +# Provision S3 backend explicitly +atmos terraform backend create vpc --stack dev + +# Then run Terraform +atmos terraform apply vpc --stack dev +``` + +**When to use manual provisioning:** +- CI/CD pipelines with separate provisioning stages +- Troubleshooting provisioning issues +- Batch provisioning for multiple components +- Pre-provisioning before large-scale deployments + +### CI/CD Integration Examples + +#### GitHub Actions + +```yaml +name: Deploy Infrastructure + +on: + push: + branches: [main] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::123456789012:role/GitHubActions + aws-region: us-east-1 + + - name: Provision Backend + run: | + atmos terraform backend create vpc --stack dev + atmos terraform backend create eks --stack dev + atmos terraform backend create rds --stack dev + # If any provisioning fails, workflow stops here + + - name: Deploy Infrastructure + run: | + atmos terraform apply vpc --stack dev + atmos terraform apply eks --stack dev + atmos terraform apply rds --stack dev + # Only runs if provisioning succeeded +``` + +#### GitLab CI + +```yaml +stages: + - provision + - deploy + +provision_backend: + stage: provision + script: + - atmos terraform backend create vpc --stack dev + # Pipeline fails if exit code != 0 + +deploy_infrastructure: + stage: deploy + script: + - atmos terraform apply vpc --stack dev + # Only runs if provision stage succeeded +``` + +### Error Handling in CLI + +**Provisioning failure stops execution:** + +```bash +$ atmos terraform backend create vpc --stack dev + +Running backend provisioner... +Creating S3 bucket 'acme-terraform-state-dev'... +Error: backend provisioning failed: failed to create bucket: +operation error S3: CreateBucket, https response error StatusCode: 403, AccessDenied + +Hint: Verify AWS credentials have s3:CreateBucket permission +Required IAM permissions: s3:CreateBucket, s3:PutBucketVersioning, s3:PutBucketEncryption +Context: + bucket: acme-terraform-state-dev + region: us-east-1 + identity: dev-admin + +Exit code: 3 +``` + +**Terraform blocked if provisioning fails:** + +```bash +$ atmos terraform apply vpc --stack dev + +Authenticating... +Running backend provisioner... +Error: Provisioning failed - cannot proceed with terraform +provisioner 'backend' failed: backend provisioning failed + +Exit code: 2 +``` + +**Success output:** + +```bash +$ atmos terraform backend create vpc --stack dev + +Running backend provisioner... +Creating S3 bucket 'acme-terraform-state-dev' with secure defaults... +Enabling bucket versioning... +Enabling bucket encryption (AES-256)... +Blocking public access... +Applying resource tags... +✓ Successfully created S3 bucket 'acme-terraform-state-dev' + +Exit code: 0 +``` + +**Idempotent operation:** + +```bash +$ atmos terraform backend create vpc --stack dev + +Running backend provisioner... +S3 bucket 'acme-terraform-state-dev' already exists (idempotent) +✓ Backend provisioning completed + +Exit code: 0 +``` + +--- + +## Error Categories and Exit Codes + +### Error Categories + +#### 1. Configuration Errors (Exit Code 2) + +**Missing bucket name:** +```text +Error: backend.bucket is required in backend configuration + +Hint: Add bucket name to stack manifest +Example: + backend: + bucket: my-terraform-state + key: vpc/terraform.tfstate + region: us-east-1 + +Exit code: 2 +``` + +**Missing region:** +```text +Error: backend.region is required in backend configuration + +Hint: Specify AWS region for S3 bucket +Example: + backend: + region: us-east-1 + +Exit code: 2 +``` + +#### 2. Permission Errors (Exit Code 3) + +**IAM permission denied:** +```text +Error: failed to create bucket: AccessDenied + +Hint: Verify AWS credentials have s3:CreateBucket permission +Required IAM permissions: + - s3:CreateBucket + - s3:HeadBucket + - s3:PutBucketVersioning + - s3:PutBucketEncryption + - s3:PutBucketPublicAccessBlock + - s3:PutBucketTagging + +Check IAM policy for identity: dev-admin +Context: + bucket: acme-terraform-state-dev + region: us-east-1 + +Exit code: 3 +``` + +**Cross-account role assumption failed:** +```text +Error: failed to create S3 client: operation error STS: AssumeRole, AccessDenied + +Hint: Verify trust policy allows source identity to assume role +Required: + - Trust policy in target account must allow source account + - Source identity must have sts:AssumeRole permission + +Context: + source_identity: dev-admin + target_role: arn:aws:iam::999999999999:role/TerraformStateAdmin + +Exit code: 3 +``` + +#### 3. Resource Conflicts (Exit Code 4) + +**Bucket name already taken:** +```text +Error: failed to create bucket: BucketAlreadyExists + +Hint: S3 bucket names are globally unique across all AWS accounts +Try a different bucket name, for example: + - acme-terraform-state-dev-123456789012 (add account ID) + - acme-terraform-state-dev-us-east-1 (add region) + - acme-terraform-state-dev-a1b2c3 (add random suffix) + +Context: + bucket: acme-terraform-state-dev + region: us-east-1 + +Exit code: 4 +``` + +#### 4. Network Errors (Exit Code 5) + +**Connection timeout:** +```text +Error: failed to create bucket: RequestTimeout + +Hint: Check network connectivity to AWS API endpoints +Possible causes: + - Network firewall blocking AWS API access + - VPN/proxy configuration issues + - AWS service outage in region + +Context: + bucket: acme-terraform-state-dev + region: us-east-1 + endpoint: s3.us-east-1.amazonaws.com + +Exit code: 5 +``` + +### Error Recovery Strategies + +**Permission Issues:** +1. Check IAM policy attached to identity +2. Verify trust policy for cross-account roles +3. Check CloudTrail for specific denied actions +4. Attach required permissions (see Security section) + +**Bucket Name Conflicts:** +1. Use more specific naming (add account ID or region) +2. Add random suffix for uniqueness +3. Check existing buckets: `aws s3 ls` + +**Network Issues:** +1. Verify AWS CLI connectivity: `aws s3 ls` +2. Check firewall/proxy settings +3. Try different region +4. Check AWS service health dashboard + +### Exit Code Summary + +| Exit Code | Category | Action | +|-----------|----------|--------| +| 0 | Success | Continue to Terraform execution | +| 1 | General error | Check error message for details | +| 2 | Configuration | Fix stack manifest configuration | +| 3 | Permission | Grant required IAM permissions | +| 4 | Resource conflict | Change resource name | +| 5 | Network | Check network connectivity | + +--- + +## Timeline + +### Week 1: Implementation + +- **Day 1-2**: Core provisioner implementation + - `ProvisionS3Backend()` function + - `checkS3BucketExists()` helper + - `provisionS3BucketWithDefaults()` helper + - Client caching logic + +- **Day 3-4**: Unit tests + - Mock S3 client + - Test bucket creation + - Test idempotency + - Test error handling + - Test role assumption + +- **Day 5**: Integration tests + - Localstack setup + - Real AWS tests (optional) + - Cleanup helpers + +- **Weekend**: Documentation + - User guide + - Migration guide + - FAQ + - Examples + +### Success Criteria + +- ✅ All unit tests passing (>90% coverage) +- ✅ Integration tests passing (localstack) +- ✅ Manual testing complete +- ✅ Documentation published +- ✅ PR reviewed and approved + +--- + +## Related Documents + +- **[Provisioner System](./provisioner-system.md)** - Generic provisioner infrastructure +- **[Backend Provisioner](./backend-provisioner.md)** - Backend provisioner interface + +--- + +## Appendix: Example Usage + +### Development Workflow + +```bash +# 1. Configure stack with auto-provision +vim stacks/dev.yaml + +# 2. Generate backend (creates bucket if needed) +atmos terraform generate backend vpc --stack dev + +# 3. Initialize Terraform (backend exists) +atmos terraform init vpc --stack dev + +# 4. Apply infrastructure +atmos terraform apply vpc --stack dev +``` + +### Multi-Environment Setup + +```yaml +# Base configuration with auto-provision +# stacks/catalog/terraform/base.yaml +components: + terraform: + base: + provision: + backend: + enabled: true # All dev/test environments inherit + +# Development environment +# stacks/dev.yaml +components: + terraform: + vpc: + metadata: + inherits: [base] + backend: + bucket: acme-dev-state + # provision.backend.enabled: true inherited + +# Production environment (no auto-provision) +# stacks/prod.yaml +components: + terraform: + vpc: + backend: + bucket: acme-prod-state # Provisioned via terraform-aws-tfstate-backend + # No provision block +``` + +--- + +**End of PRD** + +**Status:** Ready for Implementation +**Estimated Timeline:** 1 week +**Next Steps:** Begin implementation of core provisioner functions diff --git a/errors/errors.go b/errors/errors.go index d084a6715f..a7be612645 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -107,9 +107,9 @@ var ( ErrTerraformBackendAPIError = errors.New("terraform backend API error") ErrUnsupportedBackendType = errors.New("unsupported backend type") ErrProcessTerraformStateFile = errors.New("error processing terraform state file") - ErrLoadAWSConfig = errors.New("failed to load AWS config") ErrGetObjectFromS3 = errors.New("failed to get object from S3") ErrReadS3ObjectBody = errors.New("failed to read S3 object body") + ErrS3BucketAccessDenied = errors.New("access denied to S3 bucket") ErrCreateGCSClient = errors.New("failed to create GCS client") ErrGetObjectFromGCS = errors.New("failed to get object from GCS") ErrReadGCSObjectBody = errors.New("failed to read GCS object body") @@ -615,6 +615,28 @@ var ( ErrProviderNotInConfig = errors.New("provider not found in configuration") ErrInvalidLogoutOption = errors.New("invalid logout option") + // Backend provisioning errors. + ErrBucketRequired = errors.New("backend.bucket is required") + ErrRegionRequired = errors.New("backend.region is required") + ErrBackendNotFound = errors.New("backend configuration not found") + ErrCreateNotImplemented = errors.New("create not implemented for backend type") + ErrDeleteNotImplemented = errors.New("delete not implemented for backend type") + ErrProvisionerFailed = errors.New("provisioner failed") + ErrLoadAWSConfig = errors.New("failed to load AWS config") + ErrCheckBucketExist = errors.New("failed to check bucket existence") + ErrCreateBucket = errors.New("failed to create bucket") + ErrApplyBucketDefaults = errors.New("failed to apply bucket defaults") + ErrEnableVersioning = errors.New("failed to enable versioning") + ErrEnableEncryption = errors.New("failed to enable encryption") + ErrBlockPublicAccess = errors.New("failed to block public access") + ErrApplyTags = errors.New("failed to apply tags") + ErrForceRequired = errors.New("--force flag required for backend deletion") + ErrBucketNotEmpty = errors.New("bucket contains objects and cannot be deleted") + ErrStateFilesExist = errors.New("bucket contains terraform state files") + ErrDeleteObjects = errors.New("failed to delete objects from bucket") + ErrDeleteBucket = errors.New("failed to delete bucket") + ErrListObjects = errors.New("failed to list bucket objects") + // Component path resolution errors. ErrPathNotInComponentDir = errors.New("path is not within Atmos component directories") ErrComponentTypeMismatch = errors.New("path component type does not match command") diff --git a/internal/exec/terraform.go b/internal/exec/terraform.go index 16c75a95e2..2542d46b83 100644 --- a/internal/exec/terraform.go +++ b/internal/exec/terraform.go @@ -1,12 +1,14 @@ package exec import ( + "context" "errors" "fmt" "os" osexec "os/exec" "path/filepath" "strings" + "time" errUtils "github.com/cloudposse/atmos/errors" auth "github.com/cloudposse/atmos/pkg/auth" @@ -15,11 +17,20 @@ import ( log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/pro" + "github.com/cloudposse/atmos/pkg/provisioner" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" + + // Import backend provisioner to register S3 provisioner. + _ "github.com/cloudposse/atmos/pkg/provisioner/backend" ) const ( + // BeforeTerraformInitEvent is the hook event name for provisioners that run before terraform init. + // This matches the hook event registered by backend provisioners in pkg/provisioner/backend/backend.go. + // See pkg/hooks/event.go (hooks.BeforeTerraformInit) for the canonical definition. + beforeTerraformInitEvent = "before.terraform.init" + autoApproveFlag = "-auto-approve" outFlag = "-out" varFileFlag = "-var-file" @@ -408,6 +419,16 @@ func ExecuteTerraform(info schema.ConfigAndStacksInfo) error { // Before executing `terraform init`, delete the `.terraform/environment` file from the component directory. cleanTerraformWorkspace(atmosConfig, componentPath) + // Execute provisioners registered for before.terraform.init hook event. + // This runs backend provisioners to ensure backends exist before Terraform tries to configure them. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + err = provisioner.ExecuteProvisioners(ctx, provisioner.HookEvent(beforeTerraformInitEvent), &atmosConfig, info.ComponentSection, info.AuthContext) + if err != nil { + return fmt.Errorf("provisioner execution failed: %w", err) + } + err = ExecuteShellCommand( atmosConfig, info.Command, @@ -500,6 +521,16 @@ func ExecuteTerraform(info schema.ConfigAndStacksInfo) error { // Before executing `terraform init`, delete the `.terraform/environment` file from the component directory. cleanTerraformWorkspace(atmosConfig, componentPath) + // Execute provisioners registered for before.terraform.init hook event. + // This runs backend provisioners to ensure backends exist before Terraform tries to configure them. + initCtx, initCancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer initCancel() + + err = provisioner.ExecuteProvisioners(initCtx, provisioner.HookEvent(beforeTerraformInitEvent), &atmosConfig, info.ComponentSection, info.AuthContext) + if err != nil { + return fmt.Errorf("provisioner execution failed: %w", err) + } + if atmosConfig.Components.Terraform.InitRunReconfigure { allArgsAndFlags = append(allArgsAndFlags, []string{"-reconfigure"}...) } diff --git a/internal/tui/templates/help_printer.go b/internal/tui/templates/help_printer.go index 06aa9e735a..c1848afe4c 100644 --- a/internal/tui/templates/help_printer.go +++ b/internal/tui/templates/help_printer.go @@ -126,15 +126,18 @@ func (p *HelpFlagPrinter) PrintHelpFlag(flag *pflag.Flag) { lines = lines[1:] } - if _, err := fmt.Fprintf(p.out, "%-*s%s\n", descIndent, flagSection, lines[0]); err != nil { - return - } - - // Print remaining lines with proper indentation - for _, line := range lines[1:] { - if _, err := fmt.Fprintf(p.out, "%s%s\n", strings.Repeat(" ", descIndent), line); err != nil { + // Check if there are any lines remaining after removing the first line. + if len(lines) > 0 { + if _, err := fmt.Fprintf(p.out, "%-*s%s\n", descIndent, flagSection, lines[0]); err != nil { return } + + // Print remaining lines with proper indentation. + for _, line := range lines[1:] { + if _, err := fmt.Fprintf(p.out, "%s%s\n", strings.Repeat(" ", descIndent), line); err != nil { + return + } + } } if _, err := fmt.Fprintln(p.out); err != nil { diff --git a/internal/tui/templates/help_printer_test.go b/internal/tui/templates/help_printer_test.go index 93b50d7881..015525ab28 100644 --- a/internal/tui/templates/help_printer_test.go +++ b/internal/tui/templates/help_printer_test.go @@ -2,6 +2,7 @@ package templates import ( "bytes" + "io" "testing" "github.com/spf13/pflag" @@ -94,3 +95,315 @@ type boolValue struct { func (b *boolValue) String() string { return "false" } func (b *boolValue) Set(string) error { return nil } func (b *boolValue) Type() string { return "bool" } + +// assertErrorCase is a helper function to assert error cases in NewHelpFlagPrinter tests. +func assertErrorCase(t *testing.T, err error, printer *HelpFlagPrinter, expectedMsg string) { + t.Helper() + assert.Error(t, err) + assert.Nil(t, printer) + if expectedMsg != "" { + assert.Contains(t, err.Error(), expectedMsg) + } +} + +// assertSuccessCase is a helper function to assert success cases in NewHelpFlagPrinter tests. +func assertSuccessCase(t *testing.T, err error, printer *HelpFlagPrinter, wrapLimit uint) { + t.Helper() + assert.NoError(t, err) + assert.NotNil(t, printer) + if wrapLimit < minWidth { + assert.Equal(t, uint(minWidth), printer.wrapLimit) + } else { + assert.Equal(t, wrapLimit, printer.wrapLimit) + } +} + +func TestNewHelpFlagPrinter(t *testing.T) { + tests := []struct { + name string + setupOut func() io.Writer + wrapLimit uint + flags *pflag.FlagSet + expectError bool + expectedMsg string + }{ + { + name: "valid printer with standard width", + setupOut: func() io.Writer { + return &bytes.Buffer{} + }, + wrapLimit: 120, + flags: pflag.NewFlagSet("test", pflag.ContinueOnError), + }, + { + name: "nil output writer", + setupOut: func() io.Writer { + return nil + }, + wrapLimit: 80, + flags: pflag.NewFlagSet("test", pflag.ContinueOnError), + expectError: true, + expectedMsg: "invalid argument: output writer cannot be nil", + }, + { + name: "nil flag set", + setupOut: func() io.Writer { + return &bytes.Buffer{} + }, + wrapLimit: 80, + flags: nil, + expectError: true, + expectedMsg: "invalid argument: flag set cannot be nil", + }, + { + name: "below minimum width uses default", + setupOut: func() io.Writer { + return &bytes.Buffer{} + }, + wrapLimit: 50, // Below minWidth (80) + flags: pflag.NewFlagSet("test", pflag.ContinueOnError), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := tt.setupOut() + printer, err := NewHelpFlagPrinter(out, tt.wrapLimit, tt.flags) + + if tt.expectError { + assertErrorCase(t, err, printer, tt.expectedMsg) + return + } + + assertSuccessCase(t, err, printer, tt.wrapLimit) + }) + } +} + +func TestCalculateMaxFlagLength(t *testing.T) { + tests := []struct { + name string + setupFlags func(*pflag.FlagSet) + expectedMaxLen int + description string + }{ + { + name: "empty flag set", + setupFlags: func(fs *pflag.FlagSet) { + // No flags added. + }, + expectedMaxLen: 0, + description: "should return 0 for empty flag set", + }, + { + name: "single bool flag with shorthand", + setupFlags: func(fs *pflag.FlagSet) { + fs.BoolP("verbose", "v", false, "enable verbose output") + }, + expectedMaxLen: len(" -v, --verbose"), + description: "bool flag with shorthand", + }, + { + name: "string flag with shorthand", + setupFlags: func(fs *pflag.FlagSet) { + fs.StringP("output", "o", "", "output file") + }, + expectedMaxLen: len(" -o, --output string"), + description: "string flag with shorthand and type", + }, + { + name: "bool flag without shorthand", + setupFlags: func(fs *pflag.FlagSet) { + fs.Bool("debug", false, "enable debug mode") + }, + expectedMaxLen: len(" --debug"), + description: "bool flag without shorthand", + }, + { + name: "string flag without shorthand", + setupFlags: func(fs *pflag.FlagSet) { + fs.String("config", "", "config file path") + }, + expectedMaxLen: len(" --config string"), + description: "string flag without shorthand", + }, + { + name: "mixed flags returns longest", + setupFlags: func(fs *pflag.FlagSet) { + fs.BoolP("verbose", "v", false, "enable verbose") + fs.StringP("configuration-file", "c", "", "config file path") + }, + expectedMaxLen: len(" -c, --configuration-file string"), + description: "should return length of longest flag", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + tt.setupFlags(fs) + + maxLen := calculateMaxFlagLength(fs) + assert.Equal(t, tt.expectedMaxLen, maxLen, tt.description) + }) + } +} + +func TestPrintHelpFlag_EmptyLinesAfterFirstLineRemoval(t *testing.T) { + // This test validates behavior for empty/whitespace/newline usage values. + // It should not panic and should still print the flag row. + tests := []struct { + name string + flag *pflag.Flag + wrapLimit uint + maxFlagLen int + }{ + { + name: "empty usage results in single empty line after split", + flag: &pflag.Flag{ + Name: "test", + Shorthand: "t", + Usage: "", + Value: &boolValue{value: false}, + DefValue: "", + }, + wrapLimit: 80, + maxFlagLen: 20, + }, + { + name: "whitespace only usage", + flag: &pflag.Flag{ + Name: "whitespace", + Shorthand: "w", + Usage: " ", + Value: &boolValue{value: false}, + DefValue: "", + }, + wrapLimit: 80, + maxFlagLen: 20, + }, + { + name: "newline only usage", + flag: &pflag.Flag{ + Name: "newline", + Shorthand: "n", + Usage: "\n", + Value: &boolValue{value: false}, + DefValue: "", + }, + wrapLimit: 80, + maxFlagLen: 20, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + printer := &HelpFlagPrinter{ + out: &buf, + wrapLimit: tt.wrapLimit, + maxFlagLen: tt.maxFlagLen, + } + + // This should not panic due to the empty lines check. + assert.NotPanics(t, func() { + printer.PrintHelpFlag(tt.flag) + }, "PrintHelpFlag should not panic with empty or minimal content") + + // Verify trailing newline is always written. + output := buf.String() + assert.True(t, len(output) > 0, "output should not be empty") + assert.True(t, output[len(output)-1] == '\n', "output should end with newline") + }) + } +} + +func TestPrintHelpFlag_EdgeCases(t *testing.T) { + tests := []struct { + name string + flag *pflag.Flag + wrapLimit uint + maxFlagLen int + description string + }{ + { + name: "flag without shorthand string type", + flag: &pflag.Flag{ + Name: "config", + Usage: "configuration file path", + Value: &stringValue{value: "config.yaml"}, + DefValue: "config.yaml", + }, + wrapLimit: 80, + maxFlagLen: 25, + description: "should format flag without shorthand with type", + }, + { + name: "flag without shorthand bool type", + flag: &pflag.Flag{ + Name: "debug", + Usage: "enable debug mode", + Value: &boolValue{value: false}, + DefValue: "false", + }, + wrapLimit: 80, + maxFlagLen: 25, + description: "should format bool flag without shorthand", + }, + { + name: "narrow width triggers multi-line layout", + flag: &pflag.Flag{ + Name: "very-long-flag-name", + Shorthand: "l", + Usage: "this is a long description that should wrap", + Value: &stringValue{value: "default"}, + DefValue: "default", + }, + wrapLimit: 60, + maxFlagLen: 50, + description: "should handle narrow width gracefully", + }, + { + name: "empty description after markdown rendering", + flag: &pflag.Flag{ + Name: "empty", + Shorthand: "e", + Usage: "", + Value: &boolValue{value: false}, + DefValue: "", + }, + wrapLimit: 80, + maxFlagLen: 20, + description: "should handle empty lines after first line removal without panic", + }, + { + name: "single character description", + flag: &pflag.Flag{ + Name: "single", + Shorthand: "s", + Usage: "x", + Value: &boolValue{value: false}, + DefValue: "", + }, + wrapLimit: 80, + maxFlagLen: 20, + description: "should handle single character description", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + printer := &HelpFlagPrinter{ + out: &buf, + wrapLimit: tt.wrapLimit, + maxFlagLen: tt.maxFlagLen, + } + + printer.PrintHelpFlag(tt.flag) + + // Verify output was written. + assert.NotEmpty(t, buf.String(), "output should not be empty") + }) + } +} diff --git a/pkg/config/load_flags_test.go b/pkg/config/load_flags_test.go index 626a14b19e..1f9afef274 100644 --- a/pkg/config/load_flags_test.go +++ b/pkg/config/load_flags_test.go @@ -214,7 +214,7 @@ func TestGetProfilesFromFlagsOrEnv(t *testing.T) { viper.Reset() t.Cleanup(viper.Reset) - // Setup Viper (for tests that still need it) + // Setup Viper. tt.setupViper() // Setup environment variables using t.Setenv for automatic cleanup diff --git a/pkg/hooks/event.go b/pkg/hooks/event.go index 561c77368d..59e38bc4fc 100644 --- a/pkg/hooks/event.go +++ b/pkg/hooks/event.go @@ -3,6 +3,7 @@ package hooks type HookEvent string const ( + BeforeTerraformInit HookEvent = "before.terraform.init" AfterTerraformApply HookEvent = "after.terraform.apply" BeforeTerraformApply HookEvent = "before.terraform.apply" AfterTerraformPlan HookEvent = "after.terraform.plan" diff --git a/pkg/provisioner/backend/backend.go b/pkg/provisioner/backend/backend.go new file mode 100644 index 0000000000..5fcd43dedd --- /dev/null +++ b/pkg/provisioner/backend/backend.go @@ -0,0 +1,137 @@ +package backend + +import ( + "context" + "fmt" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// BackendCreateFunc is a function that creates a Terraform backend. +type BackendCreateFunc func( + ctx context.Context, + atmosConfig *schema.AtmosConfiguration, + backendConfig map[string]any, + authContext *schema.AuthContext, +) error + +// BackendDeleteFunc is a function that deletes a Terraform backend. +type BackendDeleteFunc func( + ctx context.Context, + atmosConfig *schema.AtmosConfiguration, + backendConfig map[string]any, + authContext *schema.AuthContext, + force bool, +) error + +var ( + // BackendCreators maps backend type (s3, gcs, azurerm) to create function. + backendCreators = make(map[string]BackendCreateFunc) + // BackendDeleters maps backend type (s3, gcs, azurerm) to delete function. + backendDeleters = make(map[string]BackendDeleteFunc) + registryMu sync.RWMutex +) + +// RegisterBackendCreate registers a backend create function for a specific backend type. +func RegisterBackendCreate(backendType string, fn BackendCreateFunc) { + defer perf.Track(nil, "backend.RegisterBackendCreate")() + + registryMu.Lock() + defer registryMu.Unlock() + + backendCreators[backendType] = fn +} + +// GetBackendCreate returns the create function for a backend type. +// Returns nil if no create function is registered for the type. +func GetBackendCreate(backendType string) BackendCreateFunc { + defer perf.Track(nil, "backend.GetBackendCreate")() + + registryMu.RLock() + defer registryMu.RUnlock() + + return backendCreators[backendType] +} + +// RegisterBackendDelete registers a backend delete function for a specific backend type. +func RegisterBackendDelete(backendType string, fn BackendDeleteFunc) { + defer perf.Track(nil, "backend.RegisterBackendDelete")() + + registryMu.Lock() + defer registryMu.Unlock() + + backendDeleters[backendType] = fn +} + +// GetBackendDelete returns the delete function for a backend type. +// Returns nil if no delete function is registered for the type. +func GetBackendDelete(backendType string) BackendDeleteFunc { + defer perf.Track(nil, "backend.GetBackendDelete")() + + registryMu.RLock() + defer registryMu.RUnlock() + + return backendDeleters[backendType] +} + +// ResetRegistryForTesting clears the backend provisioner registry. +// This function is intended for use in tests to ensure test isolation. +// It should be called via t.Cleanup() to restore clean state after each test. +func ResetRegistryForTesting() { + defer perf.Track(nil, "backend.ResetRegistryForTesting")() + + registryMu.Lock() + defer registryMu.Unlock() + backendCreators = make(map[string]BackendCreateFunc) + backendDeleters = make(map[string]BackendDeleteFunc) +} + +// ProvisionBackend provisions a backend if provisioning is enabled. +// Returns an error if provisioning fails or no provisioner is registered. +func ProvisionBackend( + ctx context.Context, + atmosConfig *schema.AtmosConfiguration, + componentConfig map[string]any, + authContext *schema.AuthContext, +) error { + defer perf.Track(atmosConfig, "backend.ProvisionBackend")() + + // Check if provisioning is enabled. + provision, ok := componentConfig["provision"].(map[string]any) + if !ok { + return nil // No provisioning configuration + } + + backend, ok := provision["backend"].(map[string]any) + if !ok { + return nil // No backend provisioning configuration + } + + enabled, ok := backend["enabled"].(bool) + if !ok || !enabled { + return nil // Provisioning not enabled + } + + // Get backend configuration. + backendConfig, ok := componentConfig["backend"].(map[string]any) + if !ok { + return fmt.Errorf("%w: backend configuration not found", errUtils.ErrBackendNotFound) + } + + backendType, ok := componentConfig["backend_type"].(string) + if !ok { + return fmt.Errorf("%w: backend_type not specified", errUtils.ErrBackendTypeRequired) + } + + // Get create function for backend type. + createFunc := GetBackendCreate(backendType) + if createFunc == nil { + return fmt.Errorf("%w: %s", errUtils.ErrCreateNotImplemented, backendType) + } + + // Execute create function. + return createFunc(ctx, atmosConfig, backendConfig, authContext) +} diff --git a/pkg/provisioner/backend/backend_test.go b/pkg/provisioner/backend/backend_test.go new file mode 100644 index 0000000000..70fe7a315b --- /dev/null +++ b/pkg/provisioner/backend/backend_test.go @@ -0,0 +1,622 @@ +package backend + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// resetBackendRegistry clears the backend provisioner registry for testing. +func resetBackendRegistry() { + registryMu.Lock() + defer registryMu.Unlock() + backendCreators = make(map[string]BackendCreateFunc) +} + +func TestRegisterBackendCreate(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return nil + } + + RegisterBackendCreate("s3", mockProvisioner) + + provisioner := GetBackendCreate("s3") + assert.NotNil(t, provisioner) +} + +func TestGetBackendCreate_NotFound(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + provisioner := GetBackendCreate("nonexistent") + assert.Nil(t, provisioner) +} + +func TestGetBackendCreate_MultipleTypes(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + s3Provisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return nil + } + + gcsProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return nil + } + + RegisterBackendCreate("s3", s3Provisioner) + RegisterBackendCreate("gcs", gcsProvisioner) + + assert.NotNil(t, GetBackendCreate("s3")) + assert.NotNil(t, GetBackendCreate("gcs")) + assert.Nil(t, GetBackendCreate("azurerm")) +} + +func TestRegisterBackendDelete(t *testing.T) { + // Reset registry before test. + ResetRegistryForTesting() + t.Cleanup(ResetRegistryForTesting) + + mockDeleter := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext, force bool) error { + return nil + } + + RegisterBackendDelete("s3", mockDeleter) + + deleter := GetBackendDelete("s3") + assert.NotNil(t, deleter) +} + +func TestGetBackendDelete_NotFound(t *testing.T) { + // Reset registry before test. + ResetRegistryForTesting() + t.Cleanup(ResetRegistryForTesting) + + deleter := GetBackendDelete("nonexistent") + assert.Nil(t, deleter) +} + +func TestGetBackendDelete_MultipleTypes(t *testing.T) { + // Reset registry before test. + ResetRegistryForTesting() + t.Cleanup(ResetRegistryForTesting) + + s3Deleter := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext, force bool) error { + return nil + } + + gcsDeleter := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext, force bool) error { + return nil + } + + RegisterBackendDelete("s3", s3Deleter) + RegisterBackendDelete("gcs", gcsDeleter) + + assert.NotNil(t, GetBackendDelete("s3")) + assert.NotNil(t, GetBackendDelete("gcs")) + assert.Nil(t, GetBackendDelete("azurerm")) +} + +func TestResetRegistryForTesting(t *testing.T) { + // Register some functions first. + mockCreator := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return nil + } + mockDeleter := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext, force bool) error { + return nil + } + + RegisterBackendCreate("test-backend", mockCreator) + RegisterBackendDelete("test-backend", mockDeleter) + + // Verify they're registered. + assert.NotNil(t, GetBackendCreate("test-backend")) + assert.NotNil(t, GetBackendDelete("test-backend")) + + // Reset the registry. + ResetRegistryForTesting() + + // Verify they're cleared. + assert.Nil(t, GetBackendCreate("test-backend")) + assert.Nil(t, GetBackendDelete("test-backend")) +} + +func TestResetRegistryForTesting_ClearsAllEntries(t *testing.T) { + // Reset at start. + ResetRegistryForTesting() + + mockCreator := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return nil + } + mockDeleter := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext, force bool) error { + return nil + } + + // Register multiple backends. + RegisterBackendCreate("s3", mockCreator) + RegisterBackendCreate("gcs", mockCreator) + RegisterBackendCreate("azurerm", mockCreator) + RegisterBackendDelete("s3", mockDeleter) + RegisterBackendDelete("gcs", mockDeleter) + + // Verify all are registered. + assert.NotNil(t, GetBackendCreate("s3")) + assert.NotNil(t, GetBackendCreate("gcs")) + assert.NotNil(t, GetBackendCreate("azurerm")) + assert.NotNil(t, GetBackendDelete("s3")) + assert.NotNil(t, GetBackendDelete("gcs")) + + // Reset. + ResetRegistryForTesting() + + // Verify all are cleared. + assert.Nil(t, GetBackendCreate("s3")) + assert.Nil(t, GetBackendCreate("gcs")) + assert.Nil(t, GetBackendCreate("azurerm")) + assert.Nil(t, GetBackendDelete("s3")) + assert.Nil(t, GetBackendDelete("gcs")) +} + +func TestProvisionBackend_NoProvisioningConfiguration(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + // Component config without provision block. + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.NoError(t, err, "Should return nil when no provisioning configuration exists") +} + +func TestProvisionBackend_NoBackendProvisioningConfiguration(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + // Component config with provision block but no backend sub-block. + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "other": map[string]any{ + "enabled": true, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.NoError(t, err, "Should return nil when no backend provisioning configuration exists") +} + +func TestProvisionBackend_ProvisioningDisabled(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + // Component config with provisioning explicitly disabled. + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": false, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.NoError(t, err, "Should return nil when provisioning is disabled") +} + +func TestProvisionBackend_ProvisioningEnabledMissingField(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + // Component config with backend block but no enabled field (defaults to false). + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{}, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.NoError(t, err, "Should return nil when enabled field is missing") +} + +func TestProvisionBackend_MissingBackendConfiguration(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + // Component config with provisioning enabled but no backend configuration. + componentConfig := map[string]any{ + "backend_type": "s3", + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBackendNotFound) + assert.Contains(t, err.Error(), "backend configuration not found") +} + +func TestProvisionBackend_MissingBackendType(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + // Component config with provisioning enabled but no backend_type. + componentConfig := map[string]any{ + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBackendTypeRequired) + assert.Contains(t, err.Error(), "backend_type not specified") +} + +func TestProvisionBackend_UnsupportedBackendType(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + // Component config with unsupported backend type. + componentConfig := map[string]any{ + "backend_type": "unsupported", + "backend": map[string]any{ + "bucket": "test-bucket", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrCreateNotImplemented) + assert.Contains(t, err.Error(), "unsupported") +} + +func TestProvisionBackend_Success(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + provisionerCalled := false + var capturedBackendConfig map[string]any + var capturedAuthContext *schema.AuthContext + + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + provisionerCalled = true + capturedBackendConfig = backendConfig + capturedAuthContext = authContext + return nil + } + + RegisterBackendCreate("s3", mockProvisioner) + + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.NoError(t, err) + assert.True(t, provisionerCalled, "Provisioner should have been called") + assert.NotNil(t, capturedBackendConfig) + assert.Equal(t, "test-bucket", capturedBackendConfig["bucket"]) + assert.Equal(t, "us-west-2", capturedBackendConfig["region"]) + assert.Nil(t, capturedAuthContext) +} + +func TestProvisionBackend_WithAuthContext(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + var capturedAuthContext *schema.AuthContext + + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + capturedAuthContext = authContext + return nil + } + + RegisterBackendCreate("s3", mockProvisioner) + + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + authContext := &schema.AuthContext{ + AWS: &schema.AWSAuthContext{ + Profile: "test-profile", + Region: "us-west-2", + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, authContext) + require.NoError(t, err) + require.NotNil(t, capturedAuthContext) + require.NotNil(t, capturedAuthContext.AWS) + assert.Equal(t, "test-profile", capturedAuthContext.AWS.Profile) + assert.Equal(t, "us-west-2", capturedAuthContext.AWS.Region) +} + +func TestProvisionBackend_ProvisionerFailure(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return errors.New("bucket creation failed: permission denied") + } + + RegisterBackendCreate("s3", mockProvisioner) + + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "bucket creation failed") + assert.Contains(t, err.Error(), "permission denied") +} + +func TestProvisionBackend_MultipleBackendTypes(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + s3Called := false + gcsCalled := false + + mockS3Provisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + s3Called = true + return nil + } + + mockGCSProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + gcsCalled = true + return nil + } + + RegisterBackendCreate("s3", mockS3Provisioner) + RegisterBackendCreate("gcs", mockGCSProvisioner) + + // Test S3 backend. + componentConfigS3 := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfigS3, nil) + require.NoError(t, err) + assert.True(t, s3Called, "S3 provisioner should have been called") + assert.False(t, gcsCalled, "GCS provisioner should not have been called") + + // Reset flags. + s3Called = false + gcsCalled = false + + // Test GCS backend. + componentConfigGCS := map[string]any{ + "backend_type": "gcs", + "backend": map[string]any{ + "bucket": "test-bucket", + "prefix": "terraform/state", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + err = ProvisionBackend(ctx, atmosConfig, componentConfigGCS, nil) + require.NoError(t, err) + assert.False(t, s3Called, "S3 provisioner should not have been called") + assert.True(t, gcsCalled, "GCS provisioner should have been called") +} + +func TestConcurrentBackendProvisioning(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + var callCount int + var mu sync.Mutex + + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + mu.Lock() + callCount++ + mu.Unlock() + return nil + } + + RegisterBackendCreate("s3", mockProvisioner) + + // Base config template - each goroutine will get its own copy. + baseComponentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + } + + // Run 10 concurrent provisioning operations. + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + // Create per-goroutine copy to avoid data race if ProvisionBackend mutates the map. + componentConfig := map[string]any{ + "backend_type": baseComponentConfig["backend_type"], + "backend": baseComponentConfig["backend"], + "provision": baseComponentConfig["provision"], + } + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + assert.NoError(t, err) + }() + } + + wg.Wait() + + // Verify all 10 calls executed. + assert.Equal(t, 10, callCount) +} + +func TestProvisionBackend_EnabledWrongType(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + + tests := []struct { + name string + enabledValue any + shouldProvision bool + }{ + { + name: "enabled is string 'true'", + enabledValue: "true", + shouldProvision: false, // Type assertion fails, treated as not enabled + }, + { + name: "enabled is int 1", + enabledValue: 1, + shouldProvision: false, // Type assertion fails, treated as not enabled + }, + { + name: "enabled is true", + enabledValue: true, + shouldProvision: true, + }, + { + name: "enabled is false", + enabledValue: false, + shouldProvision: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Reset registry before test. + resetBackendRegistry() + + provisionerCalled := false + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + provisionerCalled = true + return nil + } + + RegisterBackendCreate("s3", mockProvisioner) + + componentConfig := map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": tt.enabledValue, + }, + }, + } + + err := ProvisionBackend(ctx, atmosConfig, componentConfig, nil) + require.NoError(t, err) + assert.Equal(t, tt.shouldProvision, provisionerCalled) + }) + } +} diff --git a/pkg/provisioner/backend/s3.go b/pkg/provisioner/backend/s3.go new file mode 100644 index 0000000000..0466356c14 --- /dev/null +++ b/pkg/provisioner/backend/s3.go @@ -0,0 +1,362 @@ +package backend + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/internal/aws_utils" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui" +) + +const errFormat = "%w: %w" + +// S3ClientAPI defines the interface for S3 operations. +// This interface allows for mocking in tests. +// +//nolint:dupl // Interface mirrors AWS SDK client signatures - intentional design for testability. +type S3ClientAPI interface { + HeadBucket(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) + CreateBucket(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) + PutBucketVersioning(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) + PutBucketEncryption(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) + PutPublicAccessBlock(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) + PutBucketTagging(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) + ListObjectVersions(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) + DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) + DeleteBucket(ctx context.Context, params *s3.DeleteBucketInput, optFns ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) +} + +// s3Config holds S3 backend configuration. +type s3Config struct { + bucket string + region string + roleArn string +} + +func init() { + // Register S3 backend create function. + RegisterBackendCreate("s3", CreateS3Backend) + // Register S3 backend delete function. + RegisterBackendDelete("s3", DeleteS3Backend) +} + +// CreateS3Backend creates an S3 backend with opinionated, hardcoded defaults. +// +// Hardcoded features: +// - Versioning: ENABLED (always) +// - Encryption: AES-256 (AWS-managed keys, always) +// - Public Access: BLOCKED (all 4 settings, always) +// - Locking: Native S3 locking (Terraform 1.10+, no DynamoDB) +// - Tags: Standard tags (Name, ManagedBy, always) +// +// No configuration options beyond enabled: true. +// For production use, migrate to terraform-aws-tfstate-backend module. +func CreateS3Backend( + ctx context.Context, + atmosConfig *schema.AtmosConfiguration, + backendConfig map[string]any, + authContext *schema.AuthContext, +) error { + defer perf.Track(atmosConfig, "backend.CreateS3Backend")() + + // Extract and validate required configuration. + config, err := extractS3Config(backendConfig) + if err != nil { + return err + } + + _ = ui.Info(fmt.Sprintf("Provisioning S3 backend: bucket=%s region=%s", config.bucket, config.region)) + + // Load AWS configuration with auth context. + awsConfig, err := loadAWSConfigWithAuth(ctx, config.region, config.roleArn, authContext) + if err != nil { + return errUtils.Build(errUtils.ErrLoadAWSConfig). + WithHint("Check AWS credentials are configured correctly"). + WithHintf("Verify AWS region '%s' is valid", config.region). + WithHint("If using --identity flag, ensure the identity is authenticated"). + WithContext("region", config.region). + WithContext("bucket", config.bucket). + Err() + } + + // Create S3 client. + client := s3.NewFromConfig(awsConfig) + + // Check if bucket exists and create if needed. + bucketAlreadyExisted, err := ensureBucket(ctx, client, config.bucket, config.region) + if err != nil { + return err + } + + // Apply hardcoded defaults. + // If bucket already existed, warn that settings may be overwritten. + if err := applyS3BucketDefaults(ctx, client, config.bucket, bucketAlreadyExisted); err != nil { + return fmt.Errorf(errFormat, errUtils.ErrApplyBucketDefaults, err) + } + + _ = ui.Success(fmt.Sprintf("S3 backend provisioned successfully: %s", config.bucket)) + return nil +} + +// extractS3Config extracts and validates required S3 configuration. +func extractS3Config(backendConfig map[string]any) (*s3Config, error) { + // Extract bucket name. + bucketVal, ok := backendConfig["bucket"].(string) + if !ok || bucketVal == "" { + return nil, fmt.Errorf("%w", errUtils.ErrBucketRequired) + } + + // Extract region. + regionVal, ok := backendConfig["region"].(string) + if !ok || regionVal == "" { + return nil, fmt.Errorf("%w", errUtils.ErrRegionRequired) + } + + // Extract role ARN if specified (optional). + var roleArnVal string + if assumeRole, ok := backendConfig["assume_role"].(map[string]any); ok { + if arn, ok := assumeRole["role_arn"].(string); ok { + roleArnVal = arn + } + } + + return &s3Config{ + bucket: bucketVal, + region: regionVal, + roleArn: roleArnVal, + }, nil +} + +// ensureBucket checks if bucket exists and creates it if needed. +// Returns (true, nil) if bucket already existed, (false, nil) if bucket was created, (_, error) on failure. +func ensureBucket(ctx context.Context, client S3ClientAPI, bucket, region string) (bool, error) { + exists, err := bucketExists(ctx, client, bucket) + if err != nil { + return false, fmt.Errorf(errFormat, errUtils.ErrCheckBucketExist, err) + } + + if exists { + _ = ui.Info(fmt.Sprintf("S3 bucket %s already exists, skipping creation", bucket)) + return true, nil + } + + // Create bucket. + if err := createBucket(ctx, client, bucket, region); err != nil { + return false, fmt.Errorf(errFormat, errUtils.ErrCreateBucket, err) + } + _ = ui.Success(fmt.Sprintf("Created S3 bucket: %s", bucket)) + return false, nil +} + +// loadAWSConfigWithAuth loads AWS configuration with optional role assumption. +func loadAWSConfigWithAuth(ctx context.Context, region, roleArn string, authContext *schema.AuthContext) (aws.Config, error) { + // Extract AWS auth context if available. + var awsAuthContext *schema.AWSAuthContext + if authContext != nil && authContext.AWS != nil { + awsAuthContext = authContext.AWS + } + + // Use 1-hour duration for assumed role (default). + assumeRoleDuration := 1 * time.Hour + + // Load AWS config with auth context and optional role assumption. + return aws_utils.LoadAWSConfigWithAuth(ctx, region, roleArn, assumeRoleDuration, awsAuthContext) +} + +// bucketExists checks if an S3 bucket exists. +// Returns (false, nil) if bucket doesn't exist (404). +// Returns (false, error) for permission denied, network issues, or other errors. +func bucketExists(ctx context.Context, client S3ClientAPI, bucket string) (bool, error) { + _, err := client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + // Check if error is bucket not found (404). + var notFound *types.NotFound + var noSuchBucket *types.NoSuchBucket + if errors.As(err, ¬Found) || errors.As(err, &noSuchBucket) { + return false, nil + } + + // Check for HTTP status code to distinguish between different error types. + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "Forbidden", "AccessDenied": + // 403 Forbidden - permission denied. + return false, errUtils.Build(errUtils.ErrS3BucketAccessDenied). + WithHint("Check AWS IAM permissions for s3:ListBucket action"). + WithHintf("Verify that your credentials have access to bucket '%s'", bucket). + WithContext("bucket", bucket). + WithContext("operation", "HeadBucket"). + Err() + } + } + + // Check for HTTP-level errors using response metadata. + var respErr interface{ HTTPStatusCode() int } + if errors.As(err, &respErr) { + statusCode := respErr.HTTPStatusCode() + switch statusCode { + case http.StatusForbidden: + // 403 Forbidden. + return false, errUtils.Build(errUtils.ErrS3BucketAccessDenied). + WithHint("Check AWS IAM permissions for s3:ListBucket action"). + WithHintf("Verify that your credentials have access to bucket '%s'", bucket). + WithContext("bucket", bucket). + WithContext("status_code", fmt.Sprintf("%d", statusCode)). + Err() + case http.StatusNotFound: + // 404 Not Found (shouldn't reach here due to type checks above, but be defensive). + return false, nil + } + } + + // Network or other transient error. + return false, errUtils.Build(errUtils.ErrCheckBucketExist). + WithHint("Check network connectivity to AWS S3"). + WithHint("Verify AWS region is correct"). + WithHintf("Try again - this may be a transient network issue"). + WithContext("bucket", bucket). + WithContext("error", err.Error()). + Err() + } + + return true, nil +} + +// createBucket creates an S3 bucket. +func createBucket(ctx context.Context, client S3ClientAPI, bucket, region string) error { + input := &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + } + + // LocationConstraint is required for all regions except us-east-1. + if region != "us-east-1" { + input.CreateBucketConfiguration = &types.CreateBucketConfiguration{ + LocationConstraint: types.BucketLocationConstraint(region), + } + } + + _, err := client.CreateBucket(ctx, input) + return err +} + +// applyS3BucketDefaults applies hardcoded defaults to an S3 bucket. +// +// IMPORTANT: This function always overwrites existing settings with opinionated defaults: +// - Versioning: ENABLED +// - Encryption: AES-256 (replaces any existing encryption including KMS) +// - Public Access: BLOCKED (all 4 settings) +// - Tags: Replaces entire tag set with Name and ManagedBy=Atmos only +// +// If the bucket already existed (alreadyExisted=true), warnings are logged to inform the user +// that existing settings are being modified. +func applyS3BucketDefaults(ctx context.Context, client S3ClientAPI, bucket string, alreadyExisted bool) error { + // Warn user if modifying pre-existing bucket settings. + if alreadyExisted { + _ = ui.Warning(fmt.Sprintf("Applying Atmos defaults to existing bucket '%s'", bucket)) + _ = ui.Write(" - Versioning will be ENABLED") + _ = ui.Write(" - Encryption will be set to AES-256 (existing KMS encryption will be replaced)") + _ = ui.Write(" - Public access will be BLOCKED (all 4 settings)") + _ = ui.Write(" - Tags will be replaced with: Name, ManagedBy=Atmos") + } + + // 1. Enable versioning (ALWAYS). + if err := enableVersioning(ctx, client, bucket); err != nil { + return fmt.Errorf(errFormat, errUtils.ErrEnableVersioning, err) + } + + // 2. Enable encryption with AES-256 (ALWAYS). + // NOTE: This replaces any existing encryption configuration, including KMS. + if err := enableEncryption(ctx, client, bucket); err != nil { + return fmt.Errorf(errFormat, errUtils.ErrEnableEncryption, err) + } + + // 3. Block public access (ALWAYS). + if err := blockPublicAccess(ctx, client, bucket); err != nil { + return fmt.Errorf(errFormat, errUtils.ErrBlockPublicAccess, err) + } + + // 4. Apply standard tags (ALWAYS). + // NOTE: This replaces the entire tag set. Existing tags are not preserved. + if err := applyTags(ctx, client, bucket); err != nil { + return fmt.Errorf(errFormat, errUtils.ErrApplyTags, err) + } + + return nil +} + +// enableVersioning enables versioning on an S3 bucket. +func enableVersioning(ctx context.Context, client S3ClientAPI, bucket string) error { + _, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + return err +} + +// enableEncryption enables AES-256 encryption on an S3 bucket. +func enableEncryption(ctx context.Context, client S3ClientAPI, bucket string) error { + _, err := client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{ + Bucket: aws.String(bucket), + ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{ + Rules: []types.ServerSideEncryptionRule{ + { + ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{ + SSEAlgorithm: types.ServerSideEncryptionAes256, + }, + // Note: BucketKeyEnabled only applies to SSE-KMS, not AES-256. + }, + }, + }, + }) + return err +} + +// blockPublicAccess blocks all public access to an S3 bucket. +func blockPublicAccess(ctx context.Context, client S3ClientAPI, bucket string) error { + _, err := client.PutPublicAccessBlock(ctx, &s3.PutPublicAccessBlockInput{ + Bucket: aws.String(bucket), + PublicAccessBlockConfiguration: &types.PublicAccessBlockConfiguration{ + BlockPublicAcls: aws.Bool(true), + BlockPublicPolicy: aws.Bool(true), + IgnorePublicAcls: aws.Bool(true), + RestrictPublicBuckets: aws.Bool(true), + }, + }) + return err +} + +// applyTags applies standard tags to an S3 bucket. +func applyTags(ctx context.Context, client S3ClientAPI, bucket string) error { + _, err := client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{ + Bucket: aws.String(bucket), + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(bucket), + }, + { + Key: aws.String("ManagedBy"), + Value: aws.String("Atmos"), + }, + }, + }, + }) + return err +} diff --git a/pkg/provisioner/backend/s3_delete.go b/pkg/provisioner/backend/s3_delete.go new file mode 100644 index 0000000000..0bd98c5ded --- /dev/null +++ b/pkg/provisioner/backend/s3_delete.go @@ -0,0 +1,281 @@ +package backend + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui" +) + +// DeleteS3Backend deletes an S3 backend and all its contents. +// +// Safety mechanisms include requiring force=true flag, listing all objects and versions +// before deletion, detecting and counting .tfstate files, warning user about data loss, +// and deleting all objects/versions before bucket deletion. +// +// The process validates bucket configuration, checks bucket exists, lists all objects +// and versions, counts state files for warning, deletes all objects in batches +// (AWS limit: 1000 per request), and finally deletes the bucket itself. +// +// This operation is irreversible. State files will be permanently lost. +func DeleteS3Backend( + ctx context.Context, + atmosConfig *schema.AtmosConfiguration, + backendConfig map[string]any, + authContext *schema.AuthContext, + force bool, +) error { + defer perf.Track(atmosConfig, "backend.DeleteS3Backend")() + + if !force { + return errForceRequired() + } + + config, err := extractS3Config(backendConfig) + if err != nil { + return err + } + + _ = ui.Info(fmt.Sprintf("Deleting S3 backend: bucket=%s region=%s", config.bucket, config.region)) + + client, err := createS3ClientForDeletion(ctx, config, authContext) + if err != nil { + return err + } + + if err := validateBucketExistsForDeletion(ctx, client, config); err != nil { + return err + } + + if err := deleteS3BucketAndContents(ctx, client, config.bucket); err != nil { + return err + } + + _ = ui.Success(fmt.Sprintf("✓ Backend deleted: bucket '%s' and all contents removed", config.bucket)) + return nil +} + +// errForceRequired returns an error indicating --force flag is required. +func errForceRequired() error { + return errUtils.Build(errUtils.ErrForceRequired). + WithExplanation("Backend deletion requires explicit confirmation"). + WithHint("Use --force flag to confirm you want to permanently delete the backend"). + Err() +} + +// createS3ClientForDeletion loads AWS config and creates an S3 client. +func createS3ClientForDeletion(ctx context.Context, config *s3Config, authContext *schema.AuthContext) (S3ClientAPI, error) { + awsConfig, err := loadAWSConfigWithAuth(ctx, config.region, config.roleArn, authContext) + if err != nil { + return nil, errUtils.Build(errUtils.ErrLoadAWSConfig). + WithCause(err). + WithExplanation("Failed to load AWS configuration for backend deletion"). + WithContext("region", config.region). + WithHint("Check AWS credentials and region configuration"). + Err() + } + return s3.NewFromConfig(awsConfig), nil +} + +// validateBucketExistsForDeletion checks if the bucket exists before deletion. +func validateBucketExistsForDeletion(ctx context.Context, client S3ClientAPI, config *s3Config) error { + exists, err := bucketExists(ctx, client, config.bucket) + if err != nil { + return err + } + if !exists { + return errUtils.Build(errUtils.ErrBackendNotFound). + WithExplanation("Cannot delete backend - bucket does not exist"). + WithContext("bucket", config.bucket). + WithContext("region", config.region). + WithHint("Verify the bucket name in your backend configuration"). + Err() + } + return nil +} + +// deleteS3BucketAndContents lists, warns, deletes objects, and deletes the bucket. +func deleteS3BucketAndContents(ctx context.Context, client S3ClientAPI, bucket string) error { + objectCount, stateFileCount, err := listAllObjects(ctx, client, bucket) + if err != nil { + return err + } + + if err := deleteBackendContents(ctx, client, bucket, objectCount, stateFileCount); err != nil { + return err + } + + return deleteBucket(ctx, client, bucket) +} + +// deleteBackendContents displays warnings and deletes all objects from a bucket. +func deleteBackendContents(ctx context.Context, client S3ClientAPI, bucket string, objectCount, stateFileCount int) error { + if objectCount == 0 { + return nil + } + + // Show warning about what will be deleted. + showDeletionWarning(bucket, objectCount, stateFileCount) + + // Delete all objects and versions. + if err := deleteAllObjects(ctx, client, bucket); err != nil { + return err + } + + _ = ui.Success(fmt.Sprintf("Deleted %d object(s) from bucket '%s'", objectCount, bucket)) + return nil +} + +// showDeletionWarning displays a warning message about pending deletion. +func showDeletionWarning(bucket string, objectCount, stateFileCount int) { + msg := fmt.Sprintf("⚠ Deleting backend will permanently remove %d object(s) from bucket '%s'", + objectCount, bucket) + if stateFileCount > 0 { + msg += fmt.Sprintf(" (including %d Terraform state file(s))", stateFileCount) + } + _ = ui.Warning(msg) + _ = ui.Warning("This action cannot be undone") +} + +// listAllObjects lists all objects and versions in a bucket, returning counts. +func listAllObjects(ctx context.Context, client S3ClientAPI, bucket string) (totalObjects int, stateFiles int, err error) { + var continuationKeyMarker *string + var continuationVersionMarker *string + + for { + output, err := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + KeyMarker: continuationKeyMarker, + VersionIdMarker: continuationVersionMarker, + }) + if err != nil { + return 0, 0, errUtils.Build(errUtils.ErrListObjects). + WithCause(err). + WithExplanation("Failed to list objects in bucket"). + WithContext("bucket", bucket). + WithHint("Check IAM permissions for s3:ListBucketVersions"). + Err() + } + + // Count versions (actual objects). + totalObjects += len(output.Versions) + for i := range output.Versions { + if output.Versions[i].Key != nil && strings.HasSuffix(*output.Versions[i].Key, ".tfstate") { + stateFiles++ + } + } + + // Count delete markers (also need to be deleted). + totalObjects += len(output.DeleteMarkers) + + // Check if there are more pages. + if !aws.ToBool(output.IsTruncated) { + break + } + + continuationKeyMarker = output.NextKeyMarker + continuationVersionMarker = output.NextVersionIdMarker + } + + return totalObjects, stateFiles, nil +} + +// collectObjectIdentifiers builds a list of object identifiers from versions and delete markers. +func collectObjectIdentifiers(output *s3.ListObjectVersionsOutput) []types.ObjectIdentifier { + objects := make([]types.ObjectIdentifier, 0, len(output.Versions)+len(output.DeleteMarkers)) + for i := range output.Versions { + objects = append(objects, types.ObjectIdentifier{ + Key: output.Versions[i].Key, VersionId: output.Versions[i].VersionId, + }) + } + for i := range output.DeleteMarkers { + objects = append(objects, types.ObjectIdentifier{ + Key: output.DeleteMarkers[i].Key, VersionId: output.DeleteMarkers[i].VersionId, + }) + } + return objects +} + +// deleteBatch deletes a batch of objects and handles partial failures. +func deleteBatch(ctx context.Context, client S3ClientAPI, bucket string, objects []types.ObjectIdentifier) error { + if len(objects) == 0 { + return nil + } + resp, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)}, + }) + if err != nil { + return errUtils.Build(errUtils.ErrDeleteObjects). + WithCause(err). + WithExplanation("Failed to delete objects from bucket"). + WithContext("bucket", bucket). + WithHint("Check IAM permissions for s3:DeleteObject and s3:DeleteObjectVersion"). + Err() + } + // Handle partial failures - DeleteObjects can return HTTP 200 with per-key errors. + if resp != nil && len(resp.Errors) > 0 { + e := resp.Errors[0] + return errUtils.Build(errUtils.ErrDeleteObjects). + WithExplanation("Partial failure when deleting objects"). + WithContext("bucket", bucket). + WithContext("key", aws.ToString(e.Key)). + WithContext("version", aws.ToString(e.VersionId)). + WithContext("code", aws.ToString(e.Code)). + WithContext("message", aws.ToString(e.Message)). + WithHint("Check object-level permissions or bucket policies"). + Err() + } + return nil +} + +// deleteAllObjects deletes all objects and versions from a bucket in batches. +func deleteAllObjects(ctx context.Context, client S3ClientAPI, bucket string) error { + var keyMarker, versionMarker *string + for { + output, err := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), KeyMarker: keyMarker, + VersionIdMarker: versionMarker, MaxKeys: aws.Int32(1000), + }) + if err != nil { + return errUtils.Build(errUtils.ErrListObjects). + WithCause(err). + WithExplanation("Failed to list object versions for deletion"). + WithContext("bucket", bucket). + WithHint("Check IAM permissions for s3:ListBucketVersions"). + Err() + } + if err := deleteBatch(ctx, client, bucket, collectObjectIdentifiers(output)); err != nil { + return err + } + if !aws.ToBool(output.IsTruncated) { + break + } + keyMarker, versionMarker = output.NextKeyMarker, output.NextVersionIdMarker + } + return nil +} + +// deleteBucket deletes an empty S3 bucket. +func deleteBucket(ctx context.Context, client S3ClientAPI, bucket string) error { + _, err := client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + return errUtils.Build(errUtils.ErrDeleteBucket). + WithCause(err). + WithExplanation("Failed to delete S3 bucket"). + WithContext("bucket", bucket). + WithHint("Check IAM permissions for s3:DeleteBucket and ensure bucket is empty"). + Err() + } + return nil +} diff --git a/pkg/provisioner/backend/s3_test.go b/pkg/provisioner/backend/s3_test.go new file mode 100644 index 0000000000..f857148109 --- /dev/null +++ b/pkg/provisioner/backend/s3_test.go @@ -0,0 +1,1291 @@ +package backend + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// mockS3Client provides a manual mock for S3ClientAPI interface. +// Manual mock used instead of mockgen because the S3ClientAPI interface wraps +// external AWS SDK types with variadic options that mockgen doesn't handle well. +// +//nolint:dupl // Mock struct mirrors interface definition - intentional for test clarity. +type mockS3Client struct { + headBucketFunc func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) + createBucketFunc func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) + putBucketVersioningFunc func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) + putBucketEncryptionFunc func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) + putPublicAccessBlockFunc func(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) + putBucketTaggingFunc func(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) + listObjectVersionsFunc func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) + deleteObjectsFunc func(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) + deleteBucketFunc func(ctx context.Context, params *s3.DeleteBucketInput, optFns ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) +} + +func (m *mockS3Client) HeadBucket(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + if m.headBucketFunc != nil { + return m.headBucketFunc(ctx, params, optFns...) + } + return &s3.HeadBucketOutput{}, nil +} + +func (m *mockS3Client) CreateBucket(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + if m.createBucketFunc != nil { + return m.createBucketFunc(ctx, params, optFns...) + } + return &s3.CreateBucketOutput{}, nil +} + +func (m *mockS3Client) PutBucketVersioning(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + if m.putBucketVersioningFunc != nil { + return m.putBucketVersioningFunc(ctx, params, optFns...) + } + return &s3.PutBucketVersioningOutput{}, nil +} + +func (m *mockS3Client) PutBucketEncryption(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + if m.putBucketEncryptionFunc != nil { + return m.putBucketEncryptionFunc(ctx, params, optFns...) + } + return &s3.PutBucketEncryptionOutput{}, nil +} + +func (m *mockS3Client) PutPublicAccessBlock(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) { + if m.putPublicAccessBlockFunc != nil { + return m.putPublicAccessBlockFunc(ctx, params, optFns...) + } + return &s3.PutPublicAccessBlockOutput{}, nil +} + +func (m *mockS3Client) PutBucketTagging(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) { + if m.putBucketTaggingFunc != nil { + return m.putBucketTaggingFunc(ctx, params, optFns...) + } + return &s3.PutBucketTaggingOutput{}, nil +} + +func (m *mockS3Client) ListObjectVersions(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + if m.listObjectVersionsFunc != nil { + return m.listObjectVersionsFunc(ctx, params, optFns...) + } + return &s3.ListObjectVersionsOutput{}, nil +} + +func (m *mockS3Client) DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + if m.deleteObjectsFunc != nil { + return m.deleteObjectsFunc(ctx, params, optFns...) + } + return &s3.DeleteObjectsOutput{}, nil +} + +func (m *mockS3Client) DeleteBucket(ctx context.Context, params *s3.DeleteBucketInput, optFns ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) { + if m.deleteBucketFunc != nil { + return m.deleteBucketFunc(ctx, params, optFns...) + } + return &s3.DeleteBucketOutput{}, nil +} + +func TestExtractS3Config(t *testing.T) { + tests := []struct { + name string + backendConfig map[string]any + want *s3Config + wantErr error + }{ + { + name: "valid config with all fields", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + "region": "us-west-2", + "assume_role": map[string]any{ + "role_arn": "arn:aws:iam::123456789012:role/TerraformRole", + }, + }, + want: &s3Config{ + bucket: "my-terraform-state", + region: "us-west-2", + roleArn: "arn:aws:iam::123456789012:role/TerraformRole", + }, + wantErr: nil, + }, + { + name: "valid config without role ARN", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + "region": "us-east-1", + }, + want: &s3Config{ + bucket: "my-terraform-state", + region: "us-east-1", + roleArn: "", + }, + wantErr: nil, + }, + { + name: "missing bucket", + backendConfig: map[string]any{ + "region": "us-west-2", + }, + want: nil, + wantErr: errUtils.ErrBucketRequired, + }, + { + name: "empty bucket", + backendConfig: map[string]any{ + "bucket": "", + "region": "us-west-2", + }, + want: nil, + wantErr: errUtils.ErrBucketRequired, + }, + { + name: "missing region", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + }, + want: nil, + wantErr: errUtils.ErrRegionRequired, + }, + { + name: "empty region", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + "region": "", + }, + want: nil, + wantErr: errUtils.ErrRegionRequired, + }, + { + name: "invalid bucket type", + backendConfig: map[string]any{ + "bucket": 12345, + "region": "us-west-2", + }, + want: nil, + wantErr: errUtils.ErrBucketRequired, + }, + { + name: "invalid region type", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + "region": 12345, + }, + want: nil, + wantErr: errUtils.ErrRegionRequired, + }, + { + name: "assume_role with empty role_arn", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + "region": "us-west-2", + "assume_role": map[string]any{ + "role_arn": "", + }, + }, + want: &s3Config{ + bucket: "my-terraform-state", + region: "us-west-2", + roleArn: "", + }, + wantErr: nil, + }, + { + name: "assume_role with invalid type", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + "region": "us-west-2", + "assume_role": "not-a-map", + }, + want: &s3Config{ + bucket: "my-terraform-state", + region: "us-west-2", + roleArn: "", + }, + wantErr: nil, + }, + { + name: "complex role ARN", + backendConfig: map[string]any{ + "bucket": "my-terraform-state", + "region": "eu-west-1", + "assume_role": map[string]any{ + "role_arn": "arn:aws:iam::987654321098:role/CrossAccountRole", + "session_name": "terraform-session", // Extra field (ignored). + }, + }, + want: &s3Config{ + bucket: "my-terraform-state", + region: "eu-west-1", + roleArn: "arn:aws:iam::987654321098:role/CrossAccountRole", + }, + wantErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractS3Config(tt.backendConfig) + + if tt.wantErr != nil { + require.Error(t, err) + assert.ErrorIs(t, err, tt.wantErr) + assert.Nil(t, got) + } else { + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func TestS3ProvisionerRegistration(t *testing.T) { + // Test that S3 provisioner is registered in init(). + provisioner := GetBackendCreate("s3") + assert.NotNil(t, provisioner, "S3 provisioner should be registered") +} + +func TestS3Config_FieldValues(t *testing.T) { + // Test s3Config structure holds correct values. + config := &s3Config{ + bucket: "test-bucket", + region: "us-west-2", + roleArn: "arn:aws:iam::123456789012:role/TestRole", + } + + assert.Equal(t, "test-bucket", config.bucket) + assert.Equal(t, "us-west-2", config.region) + assert.Equal(t, "arn:aws:iam::123456789012:role/TestRole", config.roleArn) +} + +func TestExtractS3Config_AllRegions(t *testing.T) { + // Test various AWS regions. + regions := []string{ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "eu-west-1", + "eu-central-1", + "ap-southeast-1", + "ap-northeast-1", + } + + for _, region := range regions { + t.Run(region, func(t *testing.T) { + backendConfig := map[string]any{ + "bucket": "test-bucket", + "region": region, + } + + got, err := extractS3Config(backendConfig) + require.NoError(t, err) + assert.Equal(t, region, got.region) + }) + } +} + +func TestExtractS3Config_BucketNameValidation(t *testing.T) { + // Test various bucket name scenarios. + tests := []struct { + name string + bucketName any + shouldPass bool + }{ + { + name: "valid bucket name", + bucketName: "my-terraform-state-bucket", + shouldPass: true, + }, + { + name: "bucket with dots", + bucketName: "my.terraform.state.bucket", + shouldPass: true, + }, + { + name: "bucket with numbers", + bucketName: "terraform-state-123456", + shouldPass: true, + }, + { + name: "nil bucket", + bucketName: nil, + shouldPass: false, + }, + { + name: "empty string bucket", + bucketName: "", + shouldPass: false, + }, + { + name: "int bucket", + bucketName: 123, + shouldPass: false, + }, + { + name: "bool bucket", + bucketName: true, + shouldPass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + backendConfig := map[string]any{ + "bucket": tt.bucketName, + "region": "us-west-2", + } + + _, err := extractS3Config(backendConfig) + if tt.shouldPass { + assert.NoError(t, err) + } else { + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBucketRequired) + } + }) + } +} + +func TestErrFormatConstant(t *testing.T) { + // Verify error format constant. + assert.Equal(t, "%w: %w", errFormat) +} + +// Tests for S3 operations using mock client. + +func TestBucketExists_BucketExists(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + assert.Equal(t, "test-bucket", *params.Bucket) + return &s3.HeadBucketOutput{}, nil + }, + } + + exists, err := bucketExists(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.True(t, exists) +} + +func TestBucketExists_BucketNotFound(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &types.NotFound{} + }, + } + + exists, err := bucketExists(ctx, mockClient, "nonexistent-bucket") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestBucketExists_NoSuchBucket(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &types.NoSuchBucket{} + }, + } + + exists, err := bucketExists(ctx, mockClient, "nonexistent-bucket") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestBucketExists_NetworkError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, errors.New("network timeout") + }, + } + + exists, err := bucketExists(ctx, mockClient, "test-bucket") + require.Error(t, err) + assert.False(t, exists) + assert.ErrorIs(t, err, errUtils.ErrCheckBucketExist) +} + +func TestCreateBucket_Success(t *testing.T) { + ctx := context.Background() + var capturedInput *s3.CreateBucketInput + mockClient := &mockS3Client{ + createBucketFunc: func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + capturedInput = params + return &s3.CreateBucketOutput{}, nil + }, + } + + err := createBucket(ctx, mockClient, "test-bucket", "us-west-2") + require.NoError(t, err) + assert.Equal(t, "test-bucket", *capturedInput.Bucket) + assert.Equal(t, types.BucketLocationConstraint("us-west-2"), capturedInput.CreateBucketConfiguration.LocationConstraint) +} + +func TestCreateBucket_UsEast1NoLocationConstraint(t *testing.T) { + ctx := context.Background() + var capturedInput *s3.CreateBucketInput + mockClient := &mockS3Client{ + createBucketFunc: func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + capturedInput = params + return &s3.CreateBucketOutput{}, nil + }, + } + + err := createBucket(ctx, mockClient, "test-bucket", "us-east-1") + require.NoError(t, err) + assert.Equal(t, "test-bucket", *capturedInput.Bucket) + // us-east-1 should not have LocationConstraint. + assert.Nil(t, capturedInput.CreateBucketConfiguration) +} + +func TestCreateBucket_Failure(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + createBucketFunc: func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + // Simulate AWS SDK error (third-party error, no sentinel). + return nil, errors.New("bucket already exists in another region") + }, + } + + err := createBucket(ctx, mockClient, "test-bucket", "us-west-2") + require.Error(t, err) + // String matching OK for third-party errors per repo standards. + assert.Contains(t, err.Error(), "bucket already exists") +} + +func TestEnsureBucket_BucketAlreadyExists(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return &s3.HeadBucketOutput{}, nil + }, + } + + alreadyExisted, err := ensureBucket(ctx, mockClient, "existing-bucket", "us-west-2") + require.NoError(t, err) + assert.True(t, alreadyExisted) +} + +func TestEnsureBucket_CreateNewBucket(t *testing.T) { + ctx := context.Background() + createCalled := false + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &types.NotFound{} + }, + createBucketFunc: func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + createCalled = true + return &s3.CreateBucketOutput{}, nil + }, + } + + alreadyExisted, err := ensureBucket(ctx, mockClient, "new-bucket", "us-west-2") + require.NoError(t, err) + assert.False(t, alreadyExisted) + assert.True(t, createCalled, "CreateBucket should have been called") +} + +func TestEnsureBucket_HeadBucketError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, errors.New("network error") + }, + } + + _, err := ensureBucket(ctx, mockClient, "test-bucket", "us-west-2") + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrCheckBucketExist) +} + +func TestEnsureBucket_CreateBucketError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &types.NotFound{} + }, + createBucketFunc: func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + return nil, errors.New("permission denied") + }, + } + + _, err := ensureBucket(ctx, mockClient, "new-bucket", "us-west-2") + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrCreateBucket) +} + +func TestEnableVersioning_Success(t *testing.T) { + ctx := context.Background() + var capturedInput *s3.PutBucketVersioningInput + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + capturedInput = params + return &s3.PutBucketVersioningOutput{}, nil + }, + } + + err := enableVersioning(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, "test-bucket", *capturedInput.Bucket) + assert.Equal(t, types.BucketVersioningStatusEnabled, capturedInput.VersioningConfiguration.Status) +} + +func TestEnableVersioning_Failure(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + return nil, errors.New("permission denied") + }, + } + + err := enableVersioning(ctx, mockClient, "test-bucket") + require.Error(t, err) +} + +func TestEnableEncryption_Success(t *testing.T) { + ctx := context.Background() + var capturedInput *s3.PutBucketEncryptionInput + mockClient := &mockS3Client{ + putBucketEncryptionFunc: func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + capturedInput = params + return &s3.PutBucketEncryptionOutput{}, nil + }, + } + + err := enableEncryption(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, "test-bucket", *capturedInput.Bucket) + require.Len(t, capturedInput.ServerSideEncryptionConfiguration.Rules, 1) + assert.Equal(t, types.ServerSideEncryptionAes256, capturedInput.ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault.SSEAlgorithm) + // BucketKeyEnabled should not be set for AES-256 (only applies to SSE-KMS). + assert.Nil(t, capturedInput.ServerSideEncryptionConfiguration.Rules[0].BucketKeyEnabled) +} + +func TestEnableEncryption_Failure(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putBucketEncryptionFunc: func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + return nil, errors.New("permission denied") + }, + } + + err := enableEncryption(ctx, mockClient, "test-bucket") + require.Error(t, err) +} + +func TestBlockPublicAccess_Success(t *testing.T) { + ctx := context.Background() + var capturedInput *s3.PutPublicAccessBlockInput + mockClient := &mockS3Client{ + putPublicAccessBlockFunc: func(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) { + capturedInput = params + return &s3.PutPublicAccessBlockOutput{}, nil + }, + } + + err := blockPublicAccess(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, "test-bucket", *capturedInput.Bucket) + assert.True(t, *capturedInput.PublicAccessBlockConfiguration.BlockPublicAcls) + assert.True(t, *capturedInput.PublicAccessBlockConfiguration.BlockPublicPolicy) + assert.True(t, *capturedInput.PublicAccessBlockConfiguration.IgnorePublicAcls) + assert.True(t, *capturedInput.PublicAccessBlockConfiguration.RestrictPublicBuckets) +} + +func TestBlockPublicAccess_Failure(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putPublicAccessBlockFunc: func(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) { + return nil, errors.New("permission denied") + }, + } + + err := blockPublicAccess(ctx, mockClient, "test-bucket") + require.Error(t, err) +} + +func TestApplyTags_Success(t *testing.T) { + ctx := context.Background() + var capturedInput *s3.PutBucketTaggingInput + mockClient := &mockS3Client{ + putBucketTaggingFunc: func(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) { + capturedInput = params + return &s3.PutBucketTaggingOutput{}, nil + }, + } + + err := applyTags(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, "test-bucket", *capturedInput.Bucket) + require.Len(t, capturedInput.Tagging.TagSet, 2) + + // Find Name and ManagedBy tags. + var nameTag, managedByTag *types.Tag + for i := range capturedInput.Tagging.TagSet { + tag := &capturedInput.Tagging.TagSet[i] + if *tag.Key == "Name" { + nameTag = tag + } + if *tag.Key == "ManagedBy" { + managedByTag = tag + } + } + + require.NotNil(t, nameTag) + assert.Equal(t, "test-bucket", *nameTag.Value) + require.NotNil(t, managedByTag) + assert.Equal(t, "Atmos", *managedByTag.Value) +} + +func TestApplyTags_Failure(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putBucketTaggingFunc: func(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) { + return nil, errors.New("permission denied") + }, + } + + err := applyTags(ctx, mockClient, "test-bucket") + require.Error(t, err) +} + +func TestApplyS3BucketDefaults_NewBucket(t *testing.T) { + ctx := context.Background() + versioningCalled := false + encryptionCalled := false + publicAccessCalled := false + taggingCalled := false + + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + versioningCalled = true + return &s3.PutBucketVersioningOutput{}, nil + }, + putBucketEncryptionFunc: func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + encryptionCalled = true + return &s3.PutBucketEncryptionOutput{}, nil + }, + putPublicAccessBlockFunc: func(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) { + publicAccessCalled = true + return &s3.PutPublicAccessBlockOutput{}, nil + }, + putBucketTaggingFunc: func(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) { + taggingCalled = true + return &s3.PutBucketTaggingOutput{}, nil + }, + } + + err := applyS3BucketDefaults(ctx, mockClient, "new-bucket", false) + require.NoError(t, err) + assert.True(t, versioningCalled, "Versioning should be enabled") + assert.True(t, encryptionCalled, "Encryption should be enabled") + assert.True(t, publicAccessCalled, "Public access should be blocked") + assert.True(t, taggingCalled, "Tags should be applied") +} + +func TestApplyS3BucketDefaults_ExistingBucket(t *testing.T) { + ctx := context.Background() + callCount := 0 + + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + callCount++ + return &s3.PutBucketVersioningOutput{}, nil + }, + putBucketEncryptionFunc: func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + callCount++ + return &s3.PutBucketEncryptionOutput{}, nil + }, + putPublicAccessBlockFunc: func(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) { + callCount++ + return &s3.PutPublicAccessBlockOutput{}, nil + }, + putBucketTaggingFunc: func(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) { + callCount++ + return &s3.PutBucketTaggingOutput{}, nil + }, + } + + // With alreadyExisted=true, all operations should still be called. + err := applyS3BucketDefaults(ctx, mockClient, "existing-bucket", true) + require.NoError(t, err) + assert.Equal(t, 4, callCount, "All 4 operations should be called") +} + +func TestApplyS3BucketDefaults_VersioningFails(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + return nil, errors.New("versioning failed") + }, + } + + err := applyS3BucketDefaults(ctx, mockClient, "test-bucket", false) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrEnableVersioning) +} + +func TestApplyS3BucketDefaults_EncryptionFails(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + return &s3.PutBucketVersioningOutput{}, nil + }, + putBucketEncryptionFunc: func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + return nil, errors.New("encryption failed") + }, + } + + err := applyS3BucketDefaults(ctx, mockClient, "test-bucket", false) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrEnableEncryption) +} + +func TestApplyS3BucketDefaults_PublicAccessFails(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + return &s3.PutBucketVersioningOutput{}, nil + }, + putBucketEncryptionFunc: func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + return &s3.PutBucketEncryptionOutput{}, nil + }, + putPublicAccessBlockFunc: func(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) { + return nil, errors.New("public access block failed") + }, + } + + err := applyS3BucketDefaults(ctx, mockClient, "test-bucket", false) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBlockPublicAccess) +} + +func TestApplyS3BucketDefaults_TaggingFails(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + putBucketVersioningFunc: func(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error) { + return &s3.PutBucketVersioningOutput{}, nil + }, + putBucketEncryptionFunc: func(ctx context.Context, params *s3.PutBucketEncryptionInput, optFns ...func(*s3.Options)) (*s3.PutBucketEncryptionOutput, error) { + return &s3.PutBucketEncryptionOutput{}, nil + }, + putPublicAccessBlockFunc: func(ctx context.Context, params *s3.PutPublicAccessBlockInput, optFns ...func(*s3.Options)) (*s3.PutPublicAccessBlockOutput, error) { + return &s3.PutPublicAccessBlockOutput{}, nil + }, + putBucketTaggingFunc: func(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) { + return nil, errors.New("tagging failed") + }, + } + + err := applyS3BucketDefaults(ctx, mockClient, "test-bucket", false) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrApplyTags) +} + +// Verify mock implements interface. +var _ S3ClientAPI = (*mockS3Client)(nil) + +// Additional tests for interface compliance. + +func TestS3ClientAPI_InterfaceCompliance(t *testing.T) { + // This test verifies that our mock properly implements the interface. + var client S3ClientAPI = &mockS3Client{} + assert.NotNil(t, client) +} + +func TestCreateBucket_AllRegions(t *testing.T) { + // Test bucket creation with various regions. + regions := map[string]bool{ + "us-east-1": false, // No location constraint. + "us-west-2": true, // Has location constraint. + "eu-west-1": true, + "ap-northeast-1": true, + } + + for region, shouldHaveConstraint := range regions { + t.Run(region, func(t *testing.T) { + ctx := context.Background() + var capturedInput *s3.CreateBucketInput + mockClient := &mockS3Client{ + createBucketFunc: func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + capturedInput = params + return &s3.CreateBucketOutput{}, nil + }, + } + + err := createBucket(ctx, mockClient, "test-bucket", region) + require.NoError(t, err) + + if shouldHaveConstraint { + require.NotNil(t, capturedInput.CreateBucketConfiguration) + assert.Equal(t, types.BucketLocationConstraint(region), capturedInput.CreateBucketConfiguration.LocationConstraint) + } else { + assert.Nil(t, capturedInput.CreateBucketConfiguration) + } + }) + } +} + +// Tests for DeleteS3Backend. + +func TestDeleteS3Backend_ForceRequired(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + backendConfig := map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + } + + // Test that force=false returns error. + err := DeleteS3Backend(ctx, atmosConfig, backendConfig, nil, false) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrForceRequired) +} + +func TestDeleteS3Backend_MissingBucket(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + backendConfig := map[string]any{ + "region": "us-west-2", + } + + err := DeleteS3Backend(ctx, atmosConfig, backendConfig, nil, true) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBucketRequired) +} + +func TestDeleteS3Backend_MissingRegion(t *testing.T) { + ctx := context.Background() + atmosConfig := &schema.AtmosConfiguration{} + backendConfig := map[string]any{ + "bucket": "test-bucket", + } + + err := DeleteS3Backend(ctx, atmosConfig, backendConfig, nil, true) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrRegionRequired) +} + +// TestDeleteS3Backend_BucketNotFound is tested via integration tests since it requires AWS SDK calls. + +func TestListAllObjects_EmptyBucket(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{}, + DeleteMarkers: []types.DeleteMarkerEntry{}, + IsTruncated: aws.Bool(false), + }, nil + }, + } + + totalObjects, stateFiles, err := listAllObjects(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, 0, totalObjects) + assert.Equal(t, 0, stateFiles) +} + +func TestListAllObjects_WithObjects(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String("file1.txt"), VersionId: aws.String("v1")}, + {Key: aws.String("terraform.tfstate"), VersionId: aws.String("v2")}, + {Key: aws.String("env/prod/terraform.tfstate"), VersionId: aws.String("v3")}, + }, + DeleteMarkers: []types.DeleteMarkerEntry{ + {Key: aws.String("deleted.txt"), VersionId: aws.String("d1")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + } + + totalObjects, stateFiles, err := listAllObjects(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, 4, totalObjects) // 3 versions + 1 delete marker. + assert.Equal(t, 2, stateFiles) // 2 files ending with .tfstate. +} + +func TestListAllObjects_Pagination(t *testing.T) { + ctx := context.Background() + callCount := 0 + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + callCount++ + if callCount == 1 { + // First page. + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String("file1.txt"), VersionId: aws.String("v1")}, + }, + IsTruncated: aws.Bool(true), + NextKeyMarker: aws.String("file1.txt"), + NextVersionIdMarker: aws.String("v1"), + }, nil + } + // Second page. + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String("file2.txt"), VersionId: aws.String("v2")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + } + + totalObjects, stateFiles, err := listAllObjects(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, 2, totalObjects) + assert.Equal(t, 0, stateFiles) + assert.Equal(t, 2, callCount, "Should make 2 API calls for pagination") +} + +func TestDeleteAllObjects_Success(t *testing.T) { + ctx := context.Background() + var deletedObjects []types.ObjectIdentifier + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String("file1.txt"), VersionId: aws.String("v1")}, + {Key: aws.String("file2.txt"), VersionId: aws.String("v2")}, + }, + DeleteMarkers: []types.DeleteMarkerEntry{ + {Key: aws.String("deleted.txt"), VersionId: aws.String("d1")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + deleteObjectsFunc: func(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + deletedObjects = params.Delete.Objects + return &s3.DeleteObjectsOutput{}, nil + }, + } + + err := deleteAllObjects(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Len(t, deletedObjects, 3, "Should delete 2 versions + 1 delete marker") +} + +func TestDeleteBucket_Success(t *testing.T) { + ctx := context.Background() + var deletedBucket string + mockClient := &mockS3Client{ + deleteBucketFunc: func(ctx context.Context, params *s3.DeleteBucketInput, optFns ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) { + deletedBucket = *params.Bucket + return &s3.DeleteBucketOutput{}, nil + }, + } + + err := deleteBucket(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, "test-bucket", deletedBucket) +} + +func TestDeleteBucket_Failure(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + deleteBucketFunc: func(ctx context.Context, params *s3.DeleteBucketInput, optFns ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) { + return nil, errors.New("bucket not empty") + }, + } + + err := deleteBucket(ctx, mockClient, "test-bucket") + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrDeleteBucket) +} + +// Note: Integration tests for S3 bucket operations (bucketExists, createBucket, etc.) +// with real AWS credentials would be placed in tests/ directory. +// Tests for deleteBackendContents helper function. + +func TestDeleteBackendContents_EmptyBucket(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{} + + // With objectCount=0, function should return nil without doing anything. + err := deleteBackendContents(ctx, mockClient, "test-bucket", 0, 0) + require.NoError(t, err) +} + +func TestDeleteBackendContents_Success(t *testing.T) { + tests := []struct { + name string + objectCount int + stateFileCount int + objectKey string + expectedSuccess bool + }{ + { + name: "with regular objects", + objectCount: 1, + stateFileCount: 0, + objectKey: "file1.txt", + expectedSuccess: true, + }, + { + name: "with state files", + objectCount: 1, + stateFileCount: 1, + objectKey: "terraform.tfstate", + expectedSuccess: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String(tt.objectKey), VersionId: aws.String("v1")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + deleteObjectsFunc: func(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + return &s3.DeleteObjectsOutput{}, nil + }, + } + + err := deleteBackendContents(ctx, mockClient, "test-bucket", tt.objectCount, tt.stateFileCount) + require.NoError(t, err) + }) + } +} + +func TestDeleteBackendContents_DeleteError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String("file1.txt"), VersionId: aws.String("v1")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + deleteObjectsFunc: func(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + return nil, errors.New("delete failed") + }, + } + + err := deleteBackendContents(ctx, mockClient, "test-bucket", 1, 0) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrDeleteObjects) +} + +// Tests for showDeletionWarning helper function. + +func TestShowDeletionWarning_WithoutStateFiles(t *testing.T) { + // This function only produces UI output, so we just verify it doesn't panic. + showDeletionWarning("test-bucket", 5, 0) +} + +func TestShowDeletionWarning_WithStateFiles(t *testing.T) { + // This function only produces UI output, so we just verify it doesn't panic. + showDeletionWarning("test-bucket", 10, 3) +} + +// The tests above provide comprehensive unit test coverage using mocked S3 client. + +// Additional tests for improved coverage. + +// mockAPIError implements smithy.APIError for testing error code paths. +type mockAPIError struct { + code string + message string +} + +func (e *mockAPIError) Error() string { return e.message } +func (e *mockAPIError) ErrorCode() string { return e.code } +func (e *mockAPIError) ErrorMessage() string { return e.message } +func (e *mockAPIError) ErrorFault() smithy.ErrorFault { return smithy.FaultUnknown } + +// mockHTTPError implements an interface with HTTPStatusCode for testing. +type mockHTTPError struct { + statusCode int + message string +} + +func (e *mockHTTPError) Error() string { return e.message } +func (e *mockHTTPError) HTTPStatusCode() int { return e.statusCode } + +func TestBucketExists_AccessDeniedAPIError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &mockAPIError{code: "AccessDenied", message: "access denied"} + }, + } + + exists, err := bucketExists(ctx, mockClient, "test-bucket") + require.Error(t, err) + assert.False(t, exists) + assert.ErrorIs(t, err, errUtils.ErrS3BucketAccessDenied) +} + +func TestBucketExists_ForbiddenAPIError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &mockAPIError{code: "Forbidden", message: "forbidden"} + }, + } + + exists, err := bucketExists(ctx, mockClient, "test-bucket") + require.Error(t, err) + assert.False(t, exists) + assert.ErrorIs(t, err, errUtils.ErrS3BucketAccessDenied) +} + +func TestBucketExists_HTTPForbiddenStatusCode(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &mockHTTPError{statusCode: 403, message: "forbidden"} + }, + } + + exists, err := bucketExists(ctx, mockClient, "test-bucket") + require.Error(t, err) + assert.False(t, exists) + assert.ErrorIs(t, err, errUtils.ErrS3BucketAccessDenied) +} + +func TestBucketExists_HTTPNotFoundStatusCode(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + headBucketFunc: func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, &mockHTTPError{statusCode: 404, message: "not found"} + }, + } + + exists, err := bucketExists(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestListAllObjects_ListError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return nil, errors.New("list failed") + }, + } + + totalObjects, stateFiles, err := listAllObjects(ctx, mockClient, "test-bucket") + require.Error(t, err) + assert.Equal(t, 0, totalObjects) + assert.Equal(t, 0, stateFiles) + assert.ErrorIs(t, err, errUtils.ErrListObjects) +} + +func TestDeleteAllObjects_ListError(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return nil, errors.New("list failed") + }, + } + + err := deleteAllObjects(ctx, mockClient, "test-bucket") + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrListObjects) +} + +func TestDeleteAllObjects_EmptyBucket(t *testing.T) { + ctx := context.Background() + deleteObjectsCalled := false + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{}, + DeleteMarkers: []types.DeleteMarkerEntry{}, + IsTruncated: aws.Bool(false), + }, nil + }, + deleteObjectsFunc: func(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + deleteObjectsCalled = true + return &s3.DeleteObjectsOutput{}, nil + }, + } + + err := deleteAllObjects(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.False(t, deleteObjectsCalled, "DeleteObjects should not be called for empty bucket") +} + +func TestDeleteAllObjects_Pagination(t *testing.T) { + ctx := context.Background() + listCallCount := 0 + deleteCallCount := 0 + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + listCallCount++ + if listCallCount == 1 { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String("file1.txt"), VersionId: aws.String("v1")}, + }, + IsTruncated: aws.Bool(true), + NextKeyMarker: aws.String("file1.txt"), + NextVersionIdMarker: aws.String("v1"), + }, nil + } + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: aws.String("file2.txt"), VersionId: aws.String("v2")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + deleteObjectsFunc: func(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + deleteCallCount++ + return &s3.DeleteObjectsOutput{}, nil + }, + } + + err := deleteAllObjects(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, 2, listCallCount, "Should list twice for pagination") + assert.Equal(t, 2, deleteCallCount, "Should delete twice for pagination") +} + +func TestListAllObjects_NilKeyHandling(t *testing.T) { + ctx := context.Background() + mockClient := &mockS3Client{ + listObjectVersionsFunc: func(ctx context.Context, params *s3.ListObjectVersionsInput, optFns ...func(*s3.Options)) (*s3.ListObjectVersionsOutput, error) { + return &s3.ListObjectVersionsOutput{ + Versions: []types.ObjectVersion{ + {Key: nil, VersionId: aws.String("v1")}, // Nil key - should not panic. + {Key: aws.String("terraform.tfstate"), VersionId: aws.String("v2")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + } + + totalObjects, stateFiles, err := listAllObjects(ctx, mockClient, "test-bucket") + require.NoError(t, err) + assert.Equal(t, 2, totalObjects) + assert.Equal(t, 1, stateFiles) // Only the non-nil key ending with .tfstate. +} diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go new file mode 100644 index 0000000000..9b5110a7e1 --- /dev/null +++ b/pkg/provisioner/provisioner.go @@ -0,0 +1,193 @@ +package provisioner + +import ( + "context" + "errors" + "fmt" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/provisioner/backend" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui" +) + +// Error types for provisioning operations. +var ErrUnsupportedProvisionerType = errors.New("unsupported provisioner type") + +// getAtmosConfigFromProvisionParams safely extracts AtmosConfig from ProvisionParams. +func getAtmosConfigFromProvisionParams(params *ProvisionParams) *schema.AtmosConfiguration { + if params == nil { + return nil + } + return params.AtmosConfig +} + +// getAtmosConfigFromDeleteParams safely extracts AtmosConfig from DeleteBackendParams. +func getAtmosConfigFromDeleteParams(params *DeleteBackendParams) *schema.AtmosConfiguration { + if params == nil { + return nil + } + return params.AtmosConfig +} + +// ExecuteDescribeComponentFunc is a function that describes a component from a stack. +// This allows us to inject the describe component logic without circular dependencies. +type ExecuteDescribeComponentFunc func( + component string, + stack string, +) (map[string]any, error) + +// ProvisionParams contains parameters for the Provision function. +type ProvisionParams struct { + AtmosConfig *schema.AtmosConfiguration + ProvisionerType string + Component string + Stack string + DescribeComponent ExecuteDescribeComponentFunc + AuthContext *schema.AuthContext +} + +// Provision provisions infrastructure resources using a params struct. +// It validates the provisioner type, loads component configuration, and executes the provisioner. +func ProvisionWithParams(params *ProvisionParams) error { + defer perf.Track(getAtmosConfigFromProvisionParams(params), "provisioner.ProvisionWithParams")() + + if params == nil { + return fmt.Errorf("%w: provision params", errUtils.ErrNilParam) + } + + if params.DescribeComponent == nil { + return fmt.Errorf("%w: DescribeComponent callback", errUtils.ErrNilParam) + } + + _ = ui.Info(fmt.Sprintf("Provisioning %s '%s' in stack '%s'", params.ProvisionerType, params.Component, params.Stack)) + + // Get component configuration from stack. + componentConfig, err := params.DescribeComponent(params.Component, params.Stack) + if err != nil { + return fmt.Errorf("failed to describe component: %w", err) + } + + // Validate provisioner type. + if params.ProvisionerType != "backend" { + return fmt.Errorf("%w: %s (supported: backend)", ErrUnsupportedProvisionerType, params.ProvisionerType) + } + + // Execute backend provisioner. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Pass AuthContext from params directly to backend provisioner. + // This enables in-process SDK calls with Atmos-managed credentials. + // The AuthContext was populated by the command layer through InitConfigAndAuth, + // which merges component-level auth with global auth and respects default identity settings. + authContext := params.AuthContext + + err = backend.ProvisionBackend(ctx, params.AtmosConfig, componentConfig, authContext) + if err != nil { + return fmt.Errorf("backend provisioning failed: %w", err) + } + + _ = ui.Success(fmt.Sprintf("Successfully provisioned %s '%s' in stack '%s'", params.ProvisionerType, params.Component, params.Stack)) + return nil +} + +// ListBackends lists all backends in a stack. +func ListBackends(atmosConfig *schema.AtmosConfiguration, opts interface{}) error { + defer perf.Track(atmosConfig, "provision.ListBackends")() + + return errUtils.Build(errUtils.ErrNotImplemented). + WithExplanation("List backends functionality is not yet implemented"). + WithHint("This feature is planned for a future release"). + Err() +} + +// DescribeBackend returns the backend configuration from stack. +func DescribeBackend(atmosConfig *schema.AtmosConfiguration, component string, opts interface{}) error { + defer perf.Track(atmosConfig, "provision.DescribeBackend")() + + return errUtils.Build(errUtils.ErrNotImplemented). + WithExplanation("Describe backend functionality is not yet implemented"). + WithHint("This feature is planned for a future release"). + WithContext("component", component). + Err() +} + +// DeleteBackendParams contains parameters for the DeleteBackend function. +type DeleteBackendParams struct { + AtmosConfig *schema.AtmosConfiguration + Component string + Stack string + Force bool + DescribeComponent ExecuteDescribeComponentFunc + AuthContext *schema.AuthContext +} + +// validateDeleteParams validates DeleteBackendParams and returns an error if invalid. +func validateDeleteParams(params *DeleteBackendParams) error { + if params == nil { + return errUtils.Build(errUtils.ErrNilParam).WithExplanation("delete backend params cannot be nil").Err() + } + if params.DescribeComponent == nil { + return errUtils.Build(errUtils.ErrNilParam).WithExplanation("DescribeComponent callback cannot be nil").Err() + } + return nil +} + +// getBackendConfigFromComponent extracts backend configuration from component config. +func getBackendConfigFromComponent(componentConfig map[string]any, component, stack string) (map[string]any, string, error) { + backendConfig, ok := componentConfig["backend"].(map[string]any) + if !ok { + return nil, "", errUtils.Build(errUtils.ErrBackendNotFound). + WithExplanation("Backend configuration not found in component"). + WithContext("component", component).WithContext("stack", stack). + WithHint("Ensure the component has a 'backend' block configured").Err() + } + backendType, ok := componentConfig["backend_type"].(string) + if !ok { + return nil, "", errUtils.Build(errUtils.ErrBackendTypeRequired). + WithExplanation("Backend type not specified in component configuration"). + WithContext("component", component).WithContext("stack", stack). + WithHint("Add 'backend_type' (e.g., 's3', 'gcs', 'azurerm') to the component configuration").Err() + } + return backendConfig, backendType, nil +} + +// DeleteBackendWithParams deletes a backend using a params struct. +func DeleteBackendWithParams(params *DeleteBackendParams) error { + defer perf.Track(getAtmosConfigFromDeleteParams(params), "provisioner.DeleteBackendWithParams")() + + if err := validateDeleteParams(params); err != nil { + return err + } + + _ = ui.Info(fmt.Sprintf("Deleting backend for component '%s' in stack '%s'", params.Component, params.Stack)) + + componentConfig, err := params.DescribeComponent(params.Component, params.Stack) + if err != nil { + return errUtils.Build(errUtils.ErrDescribeComponent).WithCause(err). + WithExplanation("Failed to describe component"). + WithContext("component", params.Component).WithContext("stack", params.Stack). + WithHint("Verify the component exists in the specified stack").Err() + } + + backendConfig, backendType, err := getBackendConfigFromComponent(componentConfig, params.Component, params.Stack) + if err != nil { + return err + } + + deleteFunc := backend.GetBackendDelete(backendType) + if deleteFunc == nil { + return errUtils.Build(errUtils.ErrDeleteNotImplemented). + WithExplanation("Delete operation not implemented for backend type"). + WithContext("backend_type", backendType). + WithHint("Supported backend types for deletion: s3").Err() + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + return deleteFunc(ctx, params.AtmosConfig, backendConfig, params.AuthContext, params.Force) +} diff --git a/pkg/provisioner/provisioner_test.go b/pkg/provisioner/provisioner_test.go new file mode 100644 index 0000000000..085bcd870f --- /dev/null +++ b/pkg/provisioner/provisioner_test.go @@ -0,0 +1,530 @@ +package provisioner + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/provisioner/backend" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestProvisionWithParams_NilParams(t *testing.T) { + err := ProvisionWithParams(nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNilParam) + assert.Contains(t, err.Error(), "provision params") +} + +func TestProvisionWithParams_NilDescribeComponent(t *testing.T) { + params := &ProvisionParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + ProvisionerType: "backend", + Component: "vpc", + Stack: "dev", + DescribeComponent: nil, + AuthContext: nil, + } + + err := ProvisionWithParams(params) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNilParam) + assert.Contains(t, err.Error(), "DescribeComponent callback") +} + +func TestProvisionWithParams_UnsupportedProvisionerType(t *testing.T) { + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + }, nil + } + + params := &ProvisionParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + ProvisionerType: "unsupported", + Component: "vpc", + Stack: "dev", + DescribeComponent: mockDescribe, + AuthContext: nil, + } + + err := ProvisionWithParams(params) + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnsupportedProvisionerType) + assert.Contains(t, err.Error(), "unsupported") + assert.Contains(t, err.Error(), "supported: backend") +} + +func TestProvisionWithParams_DescribeComponentFailure(t *testing.T) { + mockDescribe := func(component string, stack string) (map[string]any, error) { + return nil, errors.New("component not found") + } + + params := &ProvisionParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + ProvisionerType: "backend", + Component: "vpc", + Stack: "dev", + DescribeComponent: mockDescribe, + AuthContext: nil, + } + + err := ProvisionWithParams(params) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to describe component") + assert.Contains(t, err.Error(), "component not found") +} + +func TestProvisionWithParams_BackendProvisioningSuccess(t *testing.T) { + // Clean up registry after test to ensure test isolation. + t.Cleanup(backend.ResetRegistryForTesting) + + // Register a mock backend provisioner for testing. + mockProvisionerCalled := false + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + mockProvisionerCalled = true + // Verify the backend config was passed correctly. + bucket, ok := backendConfig["bucket"].(string) + assert.True(t, ok) + assert.Equal(t, "test-bucket", bucket) + + region, ok := backendConfig["region"].(string) + assert.True(t, ok) + assert.Equal(t, "us-west-2", region) + + return nil + } + + // Temporarily register the mock provisioner. + backend.RegisterBackendCreate("s3", mockProvisioner) + + mockDescribe := func(component string, stack string) (map[string]any, error) { + assert.Equal(t, "vpc", component) + assert.Equal(t, "dev", stack) + + return map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + }, nil + } + + params := &ProvisionParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + ProvisionerType: "backend", + Component: "vpc", + Stack: "dev", + DescribeComponent: mockDescribe, + AuthContext: nil, + } + + err := ProvisionWithParams(params) + require.NoError(t, err) + assert.True(t, mockProvisionerCalled, "Backend provisioner should have been called") +} + +func TestProvisionWithParams_BackendProvisioningFailure(t *testing.T) { + // Clean up registry after test to ensure test isolation. + t.Cleanup(backend.ResetRegistryForTesting) + + // Register a mock backend provisioner that fails. + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return errors.New("provisioning failed: bucket already exists in another account") + } + + // Temporarily register the mock provisioner. + backend.RegisterBackendCreate("s3", mockProvisioner) + + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + }, nil + } + + params := &ProvisionParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + ProvisionerType: "backend", + Component: "vpc", + Stack: "dev", + DescribeComponent: mockDescribe, + AuthContext: nil, + } + + err := ProvisionWithParams(params) + require.Error(t, err) + assert.Contains(t, err.Error(), "backend provisioning failed") + assert.Contains(t, err.Error(), "bucket already exists in another account") +} + +func TestProvision_DelegatesToProvisionWithParams(t *testing.T) { + // Clean up registry after test to ensure test isolation. + t.Cleanup(backend.ResetRegistryForTesting) + + // This test verifies that the Provision wrapper function correctly creates + // a ProvisionParams struct and delegates to ProvisionWithParams. + + mockDescribe := func(component string, stack string) (map[string]any, error) { + assert.Equal(t, "vpc", component) + assert.Equal(t, "dev", stack) + + return map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + }, nil + } + + // Register a mock backend provisioner. + mockProvisionerCalled := false + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + mockProvisionerCalled = true + return nil + } + backend.RegisterBackendCreate("s3", mockProvisioner) + + atmosConfig := &schema.AtmosConfiguration{} + err := ProvisionWithParams(&ProvisionParams{ + AtmosConfig: atmosConfig, + ProvisionerType: "backend", + Component: "vpc", + Stack: "dev", + DescribeComponent: mockDescribe, + AuthContext: nil, + }) + + require.NoError(t, err) + assert.True(t, mockProvisionerCalled, "Backend provisioner should have been called") +} + +func TestProvisionWithParams_WithAuthContext(t *testing.T) { + // Clean up registry after test to ensure test isolation. + t.Cleanup(backend.ResetRegistryForTesting) + + // This test verifies that AuthContext is correctly passed through to the backend provisioner. + + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + "provision": map[string]any{ + "backend": map[string]any{ + "enabled": true, + }, + }, + }, nil + } + + // Register a mock backend provisioner that verifies authContext handling. + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + // AuthContext is passed through from params; nil here because test provides nil. + assert.Nil(t, authContext, "AuthContext should be nil when params.AuthContext is nil") + return nil + } + backend.RegisterBackendCreate("s3", mockProvisioner) + + params := &ProvisionParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + ProvisionerType: "backend", + Component: "vpc", + Stack: "dev", + DescribeComponent: mockDescribe, + AuthContext: nil, + } + + err := ProvisionWithParams(params) + require.NoError(t, err) +} + +func TestProvisionWithParams_BackendTypeValidation(t *testing.T) { + // Clean up registry after test to ensure test isolation. + t.Cleanup(backend.ResetRegistryForTesting) + + tests := []struct { + name string + provisionType string + wantErr bool + errContains string + }{ + { + name: "backend type is supported", + provisionType: "backend", + wantErr: false, + }, + { + name: "terraform type is not supported", + provisionType: "terraform", + wantErr: true, + errContains: "unsupported provisioner type", + }, + { + name: "helmfile type is not supported", + provisionType: "helmfile", + wantErr: true, + errContains: "unsupported provisioner type", + }, + { + name: "empty type is not supported", + provisionType: "", + wantErr: true, + errContains: "unsupported provisioner type", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + }, nil + } + + // Register a mock provisioner for backend type. + if tt.provisionType == "backend" { + mockProvisioner := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext) error { + return nil + } + backend.RegisterBackendCreate("s3", mockProvisioner) + } + + params := &ProvisionParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + ProvisionerType: tt.provisionType, + Component: "vpc", + Stack: "dev", + DescribeComponent: mockDescribe, + AuthContext: nil, + } + + err := ProvisionWithParams(params) + + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + if tt.provisionType != "" && tt.provisionType != "backend" { + assert.ErrorIs(t, err, ErrUnsupportedProvisionerType) + } + } else { + require.NoError(t, err) + } + }) + } +} + +func TestListBackends(t *testing.T) { + t.Run("returns ErrNotImplemented", func(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{} + opts := map[string]string{"format": "table"} + + err := ListBackends(atmosConfig, opts) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNotImplemented) + }) + + t.Run("returns ErrNotImplemented with nil opts", func(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{} + err := ListBackends(atmosConfig, nil) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNotImplemented) + }) +} + +func TestDescribeBackend(t *testing.T) { + t.Run("returns ErrNotImplemented", func(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{} + component := "vpc" + opts := map[string]string{"format": "yaml"} + + err := DescribeBackend(atmosConfig, component, opts) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNotImplemented) + }) + + t.Run("returns ErrNotImplemented with nil opts", func(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{} + err := DescribeBackend(atmosConfig, "vpc", nil) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNotImplemented) + }) + + t.Run("returns ErrNotImplemented with empty component", func(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{} + err := DescribeBackend(atmosConfig, "", map[string]string{"format": "json"}) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNotImplemented) + }) +} + +func TestDeleteBackend(t *testing.T) { + // Clean up registry after test to ensure test isolation. + t.Cleanup(backend.ResetRegistryForTesting) + + t.Run("returns error when params is nil", func(t *testing.T) { + err := DeleteBackendWithParams(nil) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNilParam) + }) + + t.Run("returns error when DescribeComponent is nil", func(t *testing.T) { + err := DeleteBackendWithParams(&DeleteBackendParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + Component: "vpc", + Stack: "dev", + Force: true, + DescribeComponent: nil, + AuthContext: nil, + }) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrNilParam) + }) + + t.Run("returns error when describe component fails", func(t *testing.T) { + mockDescribe := func(component string, stack string) (map[string]any, error) { + return nil, errors.New("component not found in stack") + } + + err := DeleteBackendWithParams(&DeleteBackendParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + Component: "vpc", + Stack: "dev", + Force: true, + DescribeComponent: mockDescribe, + AuthContext: nil, + }) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrDescribeComponent) + }) + + t.Run("returns error when backend_type not specified", func(t *testing.T) { + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend": map[string]any{ + "bucket": "test-bucket", + }, + // No backend_type + }, nil + } + + err := DeleteBackendWithParams(&DeleteBackendParams{ + AtmosConfig: &schema.AtmosConfiguration{}, + Component: "vpc", + Stack: "dev", + Force: true, + DescribeComponent: mockDescribe, + AuthContext: nil, + }) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBackendTypeRequired) + }) + + t.Run("deletes backend successfully", func(t *testing.T) { + // Register a mock delete function. + mockDeleter := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, backendConfig map[string]any, authContext *schema.AuthContext, force bool) error { + assert.True(t, force, "Force flag should be true") + return nil + } + backend.RegisterBackendDelete("s3", mockDeleter) + + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend_type": "s3", + "backend": map[string]any{ + "bucket": "test-bucket", + "region": "us-west-2", + }, + }, nil + } + + atmosConfig := &schema.AtmosConfiguration{} + err := DeleteBackendWithParams(&DeleteBackendParams{ + AtmosConfig: atmosConfig, + Component: "vpc", + Stack: "dev", + Force: true, + DescribeComponent: mockDescribe, + AuthContext: nil, + }) + assert.NoError(t, err, "DeleteBackend should not error") + }) + + t.Run("returns error when backend not found", func(t *testing.T) { + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend_type": "s3", + // No backend configuration + }, nil + } + + atmosConfig := &schema.AtmosConfiguration{} + err := DeleteBackendWithParams(&DeleteBackendParams{ + AtmosConfig: atmosConfig, + Component: "vpc", + Stack: "dev", + Force: true, + DescribeComponent: mockDescribe, + AuthContext: nil, + }) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBackendNotFound) + }) + + t.Run("returns error when delete function not implemented", func(t *testing.T) { + mockDescribe := func(component string, stack string) (map[string]any, error) { + return map[string]any{ + "backend_type": "unsupported", + "backend": map[string]any{ + "bucket": "test-bucket", + }, + }, nil + } + + atmosConfig := &schema.AtmosConfiguration{} + err := DeleteBackendWithParams(&DeleteBackendParams{ + AtmosConfig: atmosConfig, + Component: "vpc", + Stack: "dev", + Force: true, + DescribeComponent: mockDescribe, + AuthContext: nil, + }) + assert.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrDeleteNotImplemented) + }) +} diff --git a/pkg/provisioner/registry.go b/pkg/provisioner/registry.go new file mode 100644 index 0000000000..ed3294a4cc --- /dev/null +++ b/pkg/provisioner/registry.go @@ -0,0 +1,126 @@ +package provisioner + +import ( + "context" + "fmt" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// HookEvent represents when a provisioner should run. +// This is a string type alias compatible with pkg/hooks.HookEvent to avoid circular dependencies. +// Use pkg/hooks.HookEvent constants (e.g., hooks.BeforeTerraformInit) when registering provisioners. +type HookEvent string + +// ProvisionerFunc is a function that provisions infrastructure. +// It receives the Atmos configuration, component configuration, and auth context. +// Returns an error if provisioning fails. +type ProvisionerFunc func( + ctx context.Context, + atmosConfig *schema.AtmosConfiguration, + componentConfig map[string]any, + authContext *schema.AuthContext, +) error + +// Provisioner represents a self-registering provisioner. +// All fields are validated at registration time by RegisterProvisioner. +type Provisioner struct { + // Type is the provisioner type (e.g., "backend", "component"). + Type string + + // HookEvent declares when this provisioner should run. + // Must not be empty; use pkg/hooks.HookEvent constants. + HookEvent HookEvent + + // Func is the provisioning function to execute. + // Must not be nil. + Func ProvisionerFunc +} + +var ( + // ProvisionersByEvent stores provisioners indexed by hook event. + provisionersByEvent = make(map[HookEvent][]Provisioner) + registryMu sync.RWMutex +) + +// RegisterProvisioner registers a provisioner for a specific hook event. +// Provisioners self-declare when they should run by specifying a hook event. +// Returns an error if Func is nil or HookEvent is empty. +func RegisterProvisioner(p Provisioner) error { + defer perf.Track(nil, "provisioner.RegisterProvisioner")() + + // Validate provisioner at registration time to catch configuration errors early. + if p.Func == nil { + return fmt.Errorf("%w: provisioner %q has nil Func", errUtils.ErrNilParam, p.Type) + } + if p.HookEvent == "" { + return fmt.Errorf("%w: provisioner %q has empty HookEvent", errUtils.ErrInvalidConfig, p.Type) + } + + registryMu.Lock() + defer registryMu.Unlock() + + provisionersByEvent[p.HookEvent] = append(provisionersByEvent[p.HookEvent], p) + return nil +} + +// GetProvisionersForEvent returns all provisioners registered for a specific hook event. +func GetProvisionersForEvent(event HookEvent) []Provisioner { + defer perf.Track(nil, "provisioner.GetProvisionersForEvent")() + + registryMu.RLock() + defer registryMu.RUnlock() + + provisioners, ok := provisionersByEvent[event] + if !ok { + return nil + } + + // Return a copy to prevent external modification. + result := make([]Provisioner, len(provisioners)) + copy(result, provisioners) + return result +} + +// ExecuteProvisioners executes all provisioners registered for a specific hook event. +// Returns an error if any provisioner fails (fail-fast behavior). +func ExecuteProvisioners( + ctx context.Context, + event HookEvent, + atmosConfig *schema.AtmosConfiguration, + componentConfig map[string]any, + authContext *schema.AuthContext, +) error { + defer perf.Track(atmosConfig, "provisioner.ExecuteProvisioners")() + + provisioners := GetProvisionersForEvent(event) + if len(provisioners) == 0 { + return nil + } + + for _, p := range provisioners { + // Defensive check: validation should happen at registration time, + // but guard against invalid entries that may have been added directly to the registry. + if p.Func == nil { + return errUtils.Build(errUtils.ErrProvisionerFailed). + WithExplanation("provisioner has nil function"). + WithContext("provisioner_type", p.Type). + WithContext("event", string(event)). + WithHint("Ensure provisioners are registered using RegisterProvisioner"). + Err() + } + + if err := p.Func(ctx, atmosConfig, componentConfig, authContext); err != nil { + return errUtils.Build(errUtils.ErrProvisionerFailed). + WithCause(err). + WithContext("provisioner_type", p.Type). + WithContext("event", string(event)). + Err() + } + } + + return nil +} diff --git a/pkg/provisioner/registry_test.go b/pkg/provisioner/registry_test.go new file mode 100644 index 0000000000..5980dd0100 --- /dev/null +++ b/pkg/provisioner/registry_test.go @@ -0,0 +1,521 @@ +package provisioner + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// resetRegistry clears the provisioner registry for testing. +func resetRegistry() { + registryMu.Lock() + defer registryMu.Unlock() + provisionersByEvent = make(map[HookEvent][]Provisioner) +} + +func TestRegisterProvisioner(t *testing.T) { + // Reset registry before test. + resetRegistry() + + event := HookEvent("before.terraform.init") + + mockFunc := func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + return nil + } + + provisioner := Provisioner{ + Type: "backend", + HookEvent: event, + Func: mockFunc, + } + + // Register the provisioner. + err := RegisterProvisioner(provisioner) + require.NoError(t, err) + + // Verify it was registered. + provisioners := GetProvisionersForEvent(event) + require.Len(t, provisioners, 1) + assert.Equal(t, "backend", provisioners[0].Type) + assert.Equal(t, event, provisioners[0].HookEvent) +} + +func TestRegisterProvisioner_NilFuncReturnsError(t *testing.T) { + // Reset registry before test. + resetRegistry() + + provisioner := Provisioner{ + Type: "backend", + HookEvent: HookEvent("before.terraform.init"), + Func: nil, + } + + // Should return error when Func is nil. + err := RegisterProvisioner(provisioner) + require.Error(t, err) + assert.Contains(t, err.Error(), "nil Func") +} + +func TestRegisterProvisioner_EmptyHookEventReturnsError(t *testing.T) { + // Reset registry before test. + resetRegistry() + + provisioner := Provisioner{ + Type: "backend", + HookEvent: "", + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + return nil + }, + } + + // Should return error when HookEvent is empty. + err := RegisterProvisioner(provisioner) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty HookEvent") +} + +func TestRegisterProvisioner_MultipleForSameEvent(t *testing.T) { + // Reset registry before test. + resetRegistry() + + event := HookEvent("before.terraform.init") + + provisioner1 := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + return nil + }, + } + + provisioner2 := Provisioner{ + Type: "validation", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + return nil + }, + } + + // Register both provisioners. + err := RegisterProvisioner(provisioner1) + require.NoError(t, err) + err = RegisterProvisioner(provisioner2) + require.NoError(t, err) + + // Verify both were registered. + provisioners := GetProvisionersForEvent(event) + require.Len(t, provisioners, 2) + + types := []string{provisioners[0].Type, provisioners[1].Type} + assert.Contains(t, types, "backend") + assert.Contains(t, types, "validation") +} + +func TestGetProvisionersForEvent_NonExistentEvent(t *testing.T) { + // Reset registry before test. + resetRegistry() + + event := HookEvent("non.existent.event") + + provisioners := GetProvisionersForEvent(event) + assert.Nil(t, provisioners) +} + +func TestGetProvisionersForEvent_ReturnsCopy(t *testing.T) { + // Reset registry before test. + resetRegistry() + + event := HookEvent("before.terraform.init") + + provisioner := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + return nil + }, + } + + err := RegisterProvisioner(provisioner) + require.NoError(t, err) + + // Get provisioners twice. + provisioners1 := GetProvisionersForEvent(event) + provisioners2 := GetProvisionersForEvent(event) + + // Verify we got copies (different slices). + require.Len(t, provisioners1, 1) + require.Len(t, provisioners2, 1) + + // Modify one slice. + provisioners1[0].Type = "modified" + + // Verify the other slice is unchanged. + assert.Equal(t, "backend", provisioners2[0].Type) + + // Verify the registry is unchanged. + provisioners3 := GetProvisionersForEvent(event) + assert.Equal(t, "backend", provisioners3[0].Type) +} + +func TestExecuteProvisioners_NoProvisioners(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event := HookEvent("non.existent.event") + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + + err := ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, nil) + require.NoError(t, err) +} + +func TestExecuteProvisioners_SingleProvisionerSuccess(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event := HookEvent("before.terraform.init") + + provisionerCalled := false + provisioner := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + provisionerCalled = true + assert.NotNil(t, atmosConfig) + assert.NotNil(t, componentConfig) + return nil + }, + } + + err := RegisterProvisioner(provisioner) + require.NoError(t, err) + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{ + "backend_type": "s3", + } + + err = ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, nil) + require.NoError(t, err) + assert.True(t, provisionerCalled, "Provisioner should have been called") +} + +func TestExecuteProvisioners_MultipleProvisionersSuccess(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event := HookEvent("before.terraform.init") + + provisioner1Called := false + provisioner1 := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + provisioner1Called = true + return nil + }, + } + + provisioner2Called := false + provisioner2 := Provisioner{ + Type: "validation", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + provisioner2Called = true + return nil + }, + } + + err := RegisterProvisioner(provisioner1) + require.NoError(t, err) + err = RegisterProvisioner(provisioner2) + require.NoError(t, err) + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + + err = ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, nil) + require.NoError(t, err) + assert.True(t, provisioner1Called, "Provisioner 1 should have been called") + assert.True(t, provisioner2Called, "Provisioner 2 should have been called") +} + +func TestExecuteProvisioners_FailFast(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event := HookEvent("before.terraform.init") + + provisioner1Called := false + provisioner2Called := false + expectedErr := errors.New("provisioning failed") + provisioner1 := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + provisioner1Called = true + return expectedErr + }, + } + + provisioner2 := Provisioner{ + Type: "validation", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + provisioner2Called = true + return nil + }, + } + + // Register provisioner1 first - execution order is deterministic (slice append order). + err := RegisterProvisioner(provisioner1) + require.NoError(t, err) + err = RegisterProvisioner(provisioner2) + require.NoError(t, err) + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + + err = ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrProvisionerFailed, "Should return ErrProvisionerFailed sentinel") + assert.ErrorIs(t, err, expectedErr, "Should wrap the underlying error") + assert.True(t, provisioner1Called, "Provisioner 1 should have been called") + assert.False(t, provisioner2Called, "Provisioner 2 should not have been called due to fail-fast") +} + +func TestExecuteProvisioners_WithAuthContext(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event := HookEvent("before.terraform.init") + + var capturedAuthContext *schema.AuthContext + provisioner := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + capturedAuthContext = authContext + return nil + }, + } + + err := RegisterProvisioner(provisioner) + require.NoError(t, err) + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + authContext := &schema.AuthContext{ + AWS: &schema.AWSAuthContext{ + Profile: "test-profile", + Region: "us-west-2", + }, + } + + err = ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, authContext) + require.NoError(t, err) + require.NotNil(t, capturedAuthContext) + require.NotNil(t, capturedAuthContext.AWS) + assert.Equal(t, "test-profile", capturedAuthContext.AWS.Profile) + assert.Equal(t, "us-west-2", capturedAuthContext.AWS.Region) +} + +func TestExecuteProvisioners_DifferentEvents(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event1 := HookEvent("before.terraform.init") + event2 := HookEvent("after.terraform.apply") + + provisioner1Called := false + provisioner1 := Provisioner{ + Type: "backend", + HookEvent: event1, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + provisioner1Called = true + return nil + }, + } + + provisioner2Called := false + provisioner2 := Provisioner{ + Type: "cleanup", + HookEvent: event2, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + provisioner2Called = true + return nil + }, + } + + err := RegisterProvisioner(provisioner1) + require.NoError(t, err) + err = RegisterProvisioner(provisioner2) + require.NoError(t, err) + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + + // Execute event1 provisioners. + err = ExecuteProvisioners(ctx, event1, atmosConfig, componentConfig, nil) + require.NoError(t, err) + assert.True(t, provisioner1Called, "Event1 provisioner should have been called") + assert.False(t, provisioner2Called, "Event2 provisioner should not have been called") + + // Execute event2 provisioners. + provisioner1Called = false + provisioner2Called = false + err = ExecuteProvisioners(ctx, event2, atmosConfig, componentConfig, nil) + require.NoError(t, err) + assert.False(t, provisioner1Called, "Event1 provisioner should not have been called") + assert.True(t, provisioner2Called, "Event2 provisioner should have been called") +} + +func TestConcurrentRegistration(t *testing.T) { + // Reset registry before test. + resetRegistry() + + event := HookEvent("before.terraform.init") + var wg sync.WaitGroup + + // Register 100 provisioners concurrently. + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + provisioner := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + return nil + }, + } + _ = RegisterProvisioner(provisioner) + }() + } + + wg.Wait() + + // Verify all provisioners were registered. + provisioners := GetProvisionersForEvent(event) + assert.Len(t, provisioners, 100, "All provisioners should be registered") +} + +func TestExecuteProvisioners_ContextCancellation(t *testing.T) { + // Reset registry before test. + resetRegistry() + + event := HookEvent("before.terraform.init") + + provisioner := Provisioner{ + Type: "backend", + HookEvent: event, + Func: func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, componentConfig map[string]any, authContext *schema.AuthContext) error { + // Check if context is cancelled. + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } + }, + } + + err := RegisterProvisioner(provisioner) + require.NoError(t, err) + + // Create a cancelled context. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + + err = ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrProvisionerFailed, "Should return ErrProvisionerFailed sentinel") + assert.ErrorIs(t, err, context.Canceled, "Should wrap the context.Canceled error") +} + +func TestHookEventType(t *testing.T) { + // Test that HookEvent is a string type and can be used as map key. + event1 := HookEvent("before.terraform.init") + event2 := HookEvent("before.terraform.init") + event3 := HookEvent("after.terraform.apply") + + assert.Equal(t, event1, event2) + assert.NotEqual(t, event1, event3) + + // Test as map key. + eventMap := make(map[HookEvent]string) + eventMap[event1] = "init" + eventMap[event3] = "apply" + + assert.Equal(t, "init", eventMap[event2]) + assert.Equal(t, "apply", eventMap[event3]) +} + +func TestExecuteProvisioners_NilFuncDefensiveCheck(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event := HookEvent("before.terraform.init") + + // Directly inject a provisioner with nil Func into the registry. + // This bypasses RegisterProvisioner validation to test the defensive check. + registryMu.Lock() + provisionersByEvent[event] = []Provisioner{ + { + Type: "invalid-provisioner", + HookEvent: event, + Func: nil, // Invalid: nil function. + }, + } + registryMu.Unlock() + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + + err := ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrProvisionerFailed, "Should return ErrProvisionerFailed for nil Func") +} + +func TestExecuteProvisioners_NilFuncWithEmptyType(t *testing.T) { + // Reset registry before test. + resetRegistry() + + ctx := context.Background() + event := HookEvent("before.terraform.init") + + // Directly inject a provisioner with nil Func and empty Type into the registry. + registryMu.Lock() + provisionersByEvent[event] = []Provisioner{ + { + Type: "", // Empty type. + HookEvent: event, + Func: nil, + }, + } + registryMu.Unlock() + + atmosConfig := &schema.AtmosConfiguration{} + componentConfig := map[string]any{} + + err := ExecuteProvisioners(ctx, event, atmosConfig, componentConfig, nil) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrProvisionerFailed, "Should return ErrProvisionerFailed for nil Func with empty type") +} diff --git a/tests/snapshots/TestCLICommands_atmos_terraform.stderr.golden b/tests/snapshots/TestCLICommands_atmos_terraform.stderr.golden index 5d6826b189..70c6381fc8 100644 --- a/tests/snapshots/TestCLICommands_atmos_terraform.stderr.golden +++ b/tests/snapshots/TestCLICommands_atmos_terraform.stderr.golden @@ -6,6 +6,7 @@ Valid subcommands are: • apply +• backend • clean • console • deploy diff --git a/tests/snapshots/TestCLICommands_atmos_terraform_--help.stdout.golden b/tests/snapshots/TestCLICommands_atmos_terraform_--help.stdout.golden index e4b3f72648..ec7f7588d4 100644 --- a/tests/snapshots/TestCLICommands_atmos_terraform_--help.stdout.golden +++ b/tests/snapshots/TestCLICommands_atmos_terraform_--help.stdout.golden @@ -145,6 +145,7 @@ EXAMPLES AVAILABLE COMMANDS apply Apply changes to infrastructure + backend [command] Manage Terraform state backends clean Clean up Terraform state and artifacts. console Try Terraform expressions at an interactive command prompt deploy Deploy the specified infrastructure using Terraform diff --git a/tests/snapshots/TestCLICommands_atmos_terraform_--help_alias_subcommand_check.stdout.golden b/tests/snapshots/TestCLICommands_atmos_terraform_--help_alias_subcommand_check.stdout.golden index ef38a7adf5..98d58fff63 100644 --- a/tests/snapshots/TestCLICommands_atmos_terraform_--help_alias_subcommand_check.stdout.golden +++ b/tests/snapshots/TestCLICommands_atmos_terraform_--help_alias_subcommand_check.stdout.golden @@ -145,6 +145,7 @@ EXAMPLES AVAILABLE COMMANDS apply Apply changes to infrastructure + backend [command] Manage Terraform state backends clean Clean up Terraform state and artifacts. console Try Terraform expressions at an interactive command prompt deploy Deploy the specified infrastructure using Terraform diff --git a/tests/snapshots/TestCLICommands_atmos_terraform_help.stdout.golden b/tests/snapshots/TestCLICommands_atmos_terraform_help.stdout.golden index bff9f3e3f9..dbace276fa 100644 --- a/tests/snapshots/TestCLICommands_atmos_terraform_help.stdout.golden +++ b/tests/snapshots/TestCLICommands_atmos_terraform_help.stdout.golden @@ -145,6 +145,7 @@ EXAMPLES AVAILABLE COMMANDS apply Apply changes to infrastructure + backend [command] Manage Terraform state backends clean Clean up Terraform state and artifacts. console Try Terraform expressions at an interactive command prompt deploy Deploy the specified infrastructure using Terraform diff --git a/tests/snapshots/TestCLICommands_atmos_terraform_non-existent.stderr.golden b/tests/snapshots/TestCLICommands_atmos_terraform_non-existent.stderr.golden index f9146a24be..d4ca3c0164 100644 --- a/tests/snapshots/TestCLICommands_atmos_terraform_non-existent.stderr.golden +++ b/tests/snapshots/TestCLICommands_atmos_terraform_non-existent.stderr.golden @@ -6,6 +6,7 @@ Valid subcommands are: • apply +• backend • clean • console • deploy diff --git a/website/blog/2025-11-20-automatic-backend-provisioning.mdx b/website/blog/2025-11-20-automatic-backend-provisioning.mdx new file mode 100644 index 0000000000..94e341639b --- /dev/null +++ b/website/blog/2025-11-20-automatic-backend-provisioning.mdx @@ -0,0 +1,208 @@ +--- +slug: automatic-backend-provisioning +title: "Solving the Terraform Bootstrap Problem with Automatic Backend Provisioning" +authors: [osterman] +tags: [feature, dx] +--- + +We're excited to introduce **automatic backend provisioning** in Atmos, a feature that solves the Terraform bootstrap problem. No more manual S3 bucket creation, no more chicken-and-egg workarounds—Atmos provisions your state backend automatically with secure defaults, making it fully compatible with Terraform-managed infrastructure. + + + +## The Problem: State Backend Bootstrapping + +Every Terraform project faces the same bootstrapping challenge: before you can manage infrastructure, you need somewhere to store your state. The typical workflow looks like this: + +1. Manually create an S3 bucket via AWS Console or CLI +2. Configure bucket versioning, encryption, and public access blocking +3. Set up DynamoDB table for state locking (optional with Terraform 1.10+) +4. Finally, start using Terraform + +This creates friction for new projects, complicates CI/CD pipelines, and introduces manual steps that conflict with infrastructure-as-code principles. + +## The Solution: Automatic Provisioning + +Atmos now provisions backends automatically when needed. Just enable it in your stack configuration: + +```yaml +components: + terraform: + vpc: + backend_type: s3 # Must be at component level + + backend: + bucket: my-terraform-state + key: vpc/terraform.tfstate + region: us-east-1 + + provision: + backend: + enabled: true # That's it! +``` + +When you run `atmos terraform plan vpc -s dev`, Atmos: + +1. **Checks** if the backend exists +2. **Provisions** it if needed (with secure defaults) +3. **Initializes** Terraform +4. **Continues** with your command + +All automatically. No manual intervention required. + +## Configuration Flexibility + +The `provision.backend` configuration works with Atmos's inheritance system, allowing you to set defaults at any level (organization, environment, component) and override when needed. + +## Secure by Default + +The S3 backend provisioner applies hardcoded security best practices: + +- ✅ **Versioning enabled** - Protect against accidental deletions +- ✅ **AES-256 encryption** - AWS-managed keys, always enabled +- ✅ **Public access blocked** - All four block settings enabled +- ✅ **Native S3 locking** - Terraform 1.10+ support (no DynamoDB needed) +- ✅ **Resource tags** - Automatic tagging for cost allocation + +These settings aren't configurable—they're opinionated defaults that follow AWS security best practices. + +## Solves the Terraform Bootstrap Problem + +Automatic provisioning is **fully compatible with Terraform-managed backends**. In fact, it solves a classic chicken-and-egg problem: "How do I manage my state backend with Terraform when I need that backend to exist before Terraform can run?" + +**The traditional workaround:** +1. Use local state temporarily +2. Create S3 bucket with Terraform using local state +3. Switch backend configuration to S3 +4. Import the bucket into the S3-backed state +5. Delete local state files + +**With Atmos automatic provisioning:** +1. Enable `provision.backend.enabled: true` +2. Run `atmos terraform plan` - bucket auto-created with secure defaults +3. Import the bucket into Terraform (no local state dance needed) +4. Done - everything managed by Terraform + +**Import the provisioned backend:** + +```hcl +import { + to = aws_s3_bucket.terraform_state + id = "my-terraform-state" +} + +resource "aws_s3_bucket" "terraform_state" { + bucket = "my-terraform-state" +} + +# Add any additional configuration +resource "aws_s3_bucket_lifecycle_configuration" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + + rule { + id = "delete-old-versions" + status = "Enabled" + + noncurrent_version_expiration { + noncurrent_days = 90 + } + } +} +``` + +This **eliminates the bootstrap hack** and makes it easier to manage everything with Terraform, not harder. The provisioner creates standard AWS resources that Terraform can import and manage - no special handling required. + +**Note:** You can leave `provision.backend.enabled: true` even after importing to Terraform. The provisioner is idempotent - it will detect the bucket exists and skip creation, causing no conflicts with Terraform management. + +## Cross-Account Support + +Provisioners integrate with Atmos AuthManager for cross-account operations: + +```yaml +components: + terraform: + vpc: + backend_type: s3 # Must be at component level + + backend: + bucket: my-terraform-state + region: us-east-1 + assume_role: + role_arn: arn:aws:iam::999999999999:role/TerraformStateAdmin + + provision: + backend: + enabled: true +``` + +The provisioner automatically assumes the role to create the bucket in the target account. + +## CLI Command + +For manual provisioning or CI/CD pipelines, use the `atmos terraform backend` command: + +```bash +# Provision backend explicitly +atmos terraform backend create vpc --stack dev + +# Automatic in CI/CD +atmos terraform backend create vpc --stack dev +atmos terraform backend create eks --stack dev +atmos terraform apply vpc --stack dev # Only runs if provisioning succeeded +``` + +Provisioning failures return non-zero exit codes, ensuring CI/CD pipelines fail fast. + +## Extensible Architecture + +The provisioner system is built on a **self-registering architecture** that makes it easy to add support for additional backend types and provisioner types in the future + +Backend provisioners register themselves and declare when they should run via hook events: + +```go +// Register S3 backend provisioner +provisioner.RegisterProvisioner(provisioner.Provisioner{ + Type: "backend", + HookEvent: "before.terraform.init", + Func: ProvisionS3Backend, +}) +``` + +## Getting Started + +Enable automatic backend provisioning in your stack configuration: + +```yaml +# stacks/dev.yaml +components: + terraform: + vpc: + backend_type: s3 # Must be at component level + + backend: + bucket: acme-terraform-state-dev + key: vpc/terraform.tfstate + region: us-east-1 + + provision: + backend: + enabled: true +``` + +Then run your Terraform commands as usual: + +```bash +atmos terraform plan vpc -s dev +# Backend provisioned automatically if needed +``` + +For more information: +- [CLI Documentation](/cli/commands/terraform/terraform-backend) +- [Backend Configuration](/components/terraform/backends) + +## Community Feedback + +We'd love to hear how you're using automatic backend provisioning! Share your experience in [GitHub Discussions](https://github.com/cloudposse/atmos/discussions) or report any issues in our [issue tracker](https://github.com/cloudposse/atmos/issues). + +--- + +**Try it today** and simplify your Terraform state management workflow! diff --git a/website/docs/cli/commands/terraform/terraform-backend.mdx b/website/docs/cli/commands/terraform/terraform-backend.mdx new file mode 100644 index 0000000000..1d9a9bc531 --- /dev/null +++ b/website/docs/cli/commands/terraform/terraform-backend.mdx @@ -0,0 +1,547 @@ +--- +title: atmos terraform backend +sidebar_label: backend +sidebar_class_name: command +id: terraform-backend +description: Manage Terraform state backend infrastructure +--- +import Intro from '@site/src/components/Intro' +import Screengrab from '@site/src/components/Screengrab' + + +Use these commands to manage Terraform state backend infrastructure. This solves the Terraform bootstrap problem by automatically provisioning backend storage with secure defaults, making it compatible with any Terraform-managed backend. + + + + +## Usage + +```shell +atmos terraform backend [options] +``` + +## Available Subcommands + +
+
`create `
+
Provision backend infrastructure for a component
+ +
`list`
+
List all backends in a stack
+ +
`describe `
+
Show backend configuration from stack
+ +
`update `
+
Update backend configuration (idempotent)
+ +
`delete `
+
Delete backend infrastructure (requires --force)
+
+ +## Creating Backends + +Provision backend infrastructure for a component in a specific stack: + +```shell +atmos terraform backend create --stack +``` + +The backend must have `provision.backend.enabled: true` in its stack configuration. + +## Examples + +### Create Backend + +Provision an S3 bucket with secure defaults: + +```shell +atmos terraform backend create vpc --stack dev +``` + +This creates an S3 bucket (if it doesn't exist) with: +- Versioning enabled +- AES-256 encryption +- Public access blocked +- Resource tags applied + +### List Backends + +Show all backend configurations in a stack: + +```shell +atmos terraform backend list --stack dev +``` + +### Describe Backend Configuration + +View a component's backend configuration from the stack: + +```shell +atmos terraform backend describe vpc --stack dev +atmos terraform backend describe vpc --stack dev --format json +``` + +### Update Backend + +Apply configuration changes to existing backend (idempotent): + +```shell +atmos terraform backend update vpc --stack dev +``` + +### Delete Backend + +Remove backend infrastructure (requires --force for safety): + +```shell +atmos terraform backend delete vpc --stack dev --force +``` + +### Provision Multiple Backends + +```shell +atmos terraform backend create vpc --stack dev +atmos terraform backend create eks --stack dev +atmos terraform backend create rds --stack dev +``` + +## Arguments + +
+
`component`
+
The Atmos component name (required for create, describe, update, delete)
+
+ +## Flags + +
+
`--stack` / `-s`
+
Atmos stack name (required). Can also be set via `ATMOS_STACK` environment variable
+ +
`--identity` / `-i`
+
Identity to use for authentication. Overrides component's default identity. Use without value to select interactively
+ +
`--format` / `-f`
+
Output format for list and describe commands: `table`, `yaml`, `json` (default: varies by command)
+ +
`--force`
+
Force deletion without confirmation (delete command only)
+
+ +## Authentication + +Backend commands support both global and component-level authentication, consistent with regular Terraform commands. + +### Component-Level Identity + +Components can specify their own authentication identity with a default: + +```yaml +# stacks/dev.yaml +components: + terraform: + vpc: + # Component-level auth configuration + auth: + identities: + vpc-deployer: + default: true # Used automatically for this component + kind: aws/permission-set + via: + provider: aws-sso + principal: + name: "NetworkAdmin" + account: + name: "network-dev" +``` + +With this configuration, backend commands automatically use the `vpc-deployer` identity: + +```shell +# Automatically uses vpc-deployer identity +atmos terraform backend create vpc --stack dev +atmos terraform backend delete vpc --stack dev --force +``` + +### Identity Flag Override + +The `--identity` flag overrides the component's default identity: + +```shell +# Override component default with admin identity +atmos terraform backend create vpc --stack dev --identity admin +``` + +### Global Identity + +Without component-level auth, backend commands use global auth configuration from `atmos.yaml` or the `--identity` flag. + +See [Authentication and Identity](/cli/commands/auth/usage) for complete auth configuration details. + +## How It Works + +### Manual Provisioning + +When you run `atmos terraform backend create`: + +1. **Load Configuration** - Atmos loads the component's stack configuration +2. **Check Provisioning** - Verifies `provision.backend.enabled: true` is set +3. **Select Provisioner** - Chooses provisioner based on `backend_type` (s3, gcs, azurerm) +4. **Check Existence** - Verifies if backend already exists (idempotent) +5. **Provision** - Creates backend with hardcoded security defaults if needed +6. **Apply Settings** - Configures versioning, encryption, access controls, and tags + +### Automatic Provisioning + +Backends are also provisioned **automatically** when running Terraform commands if `provision.backend.enabled: true`: + +```shell +# Backend provisioned automatically before terraform init +atmos terraform plan vpc --stack dev +atmos terraform apply vpc --stack dev +``` + +The automatic flow: + +``` +Auth Setup (TerraformPreHook) + ↓ +Backend Provisioning (if enabled) + ↓ +Terraform Init + ↓ +Terraform Command +``` + +### Solving the Terraform Bootstrap Problem + +Terraform has a chicken-and-egg problem: you need infrastructure to store state, but you need state to manage infrastructure. Atmos solves this by: + +1. **Automatic Detection** - Reads backend configuration from stack manifests +2. **Secure Defaults** - Creates backend with hardcoded security settings +3. **Idempotent Operations** - Safe to run multiple times +4. **Terraform Compatible** - Works with any Terraform-managed backend + +This eliminates manual backend setup and makes Terraform backends work like any other infrastructure resource. + +## Configuration + +Enable backend provisioning in your stack manifest: + +```yaml +# stacks/dev.yaml +components: + terraform: + vpc: + backend_type: s3 # Must be at component level + + backend: + bucket: acme-ue1-dev-tfstate + key: vpc/terraform.tfstate + region: us-east-1 + use_lockfile: true # Enable native S3 locking (Terraform 1.10+) + + provision: + backend: + enabled: true # Enable automatic provisioning +``` + +## Configuration Hierarchy + +The `provision.backend` configuration supports Atmos's deep-merge system and can be specified at multiple levels in the stack hierarchy. This provides flexibility to set defaults at high levels and override at component level. + +### Global Default (Organization-Level) + +Enable provisioning for all components in an organization: + +```yaml +# stacks/orgs/acme/_defaults.yaml +terraform: + provision: + backend: + enabled: true +``` + +All components in this organization will inherit `provision.backend.enabled: true` unless explicitly overridden. + +### Environment-Level Configuration + +Set different provisioning policies per environment: + +```yaml +# stacks/orgs/acme/plat/dev/_defaults.yaml +terraform: + provision: + backend: + enabled: true # Auto-provision in dev + +# stacks/orgs/acme/plat/prod/_defaults.yaml +terraform: + provision: + backend: + enabled: false # Pre-provisioned backends in prod +``` + +### Component Inheritance + +Use `metadata.inherits` to share provision configuration: + +```yaml +# stacks/catalog/vpc/defaults.yaml +components: + terraform: + vpc/defaults: + provision: + backend: + enabled: true + +# stacks/dev.yaml +components: + terraform: + vpc: + metadata: + inherits: [vpc/defaults] + # Inherits provision.backend.enabled: true +``` + +### Component-Level Override + +Override inherited settings per component: + +```yaml +components: + terraform: + vpc: + provision: + backend: + enabled: false # Disable for this specific component +``` + +**Deep-Merge Behavior:** Atmos combines configurations from all levels, giving you maximum flexibility: +- Set defaults at organization or environment level +- Override per component when needed +- Use catalog inheritance for reusable patterns +- Component-level configuration has highest precedence + +## Supported Backend Types + +### S3 (AWS) + +**Hardcoded Defaults:** +- Versioning: Enabled +- Encryption: AES-256 (AWS-managed keys) +- Public Access: Blocked (all 4 settings) +- Locking: Native S3 locking (Terraform 1.10+) +- Tags: `Name`, `ManagedBy=Atmos` + +**Required Configuration:** +```yaml +backend_type: s3 # Must be at component level + +backend: + bucket: my-terraform-state # Required + key: component/terraform.tfstate + region: us-east-1 # Required + use_lockfile: true # Enable native S3 locking (Terraform 1.10+) +``` + +**Cross-Account Support:** +```yaml +backend: + bucket: my-terraform-state + region: us-east-1 + use_lockfile: true # Enable native S3 locking (Terraform 1.10+) + assume_role: + role_arn: arn:aws:iam::999999999999:role/TerraformStateAdmin +``` + +The provisioner assumes the role to create the bucket in the target account. + +## Error Handling + +### Exit Codes + +| Exit Code | Error Type | Action | +|-----------|------------|--------| +| 0 | Success | Backend created or already exists | +| 1 | General error | Check error message for details | +| 2 | Configuration error | Fix `provision.backend` configuration | +| 3 | Permission error | Grant required IAM permissions | +| 4 | Resource conflict | Change bucket name (globally unique) | +| 5 | Network error | Check network connectivity to cloud provider | + +### Example Errors + +**Missing Configuration:** +``` +Error: backend.bucket is required in backend configuration + +Hint: Add bucket name to stack manifest +Example: + backend: + bucket: my-terraform-state + region: us-east-1 +``` + +**Permission Denied:** +``` +Error: failed to create bucket: AccessDenied + +Hint: Verify AWS credentials have s3:CreateBucket permission +Required IAM permissions: + - s3:CreateBucket + - s3:HeadBucket + - s3:PutBucketVersioning + - s3:PutBucketEncryption + - s3:PutBucketPublicAccessBlock + - s3:PutBucketTagging +``` + +**Bucket Name Conflict:** +``` +Error: failed to create bucket: BucketAlreadyExists + +Hint: S3 bucket names are globally unique across all AWS accounts +Try a different bucket name: + - acme-ue1-dev-tfstate-123456789012 (add account ID) + - acme-ue1-dev-tfstate-randomsuffix (add random suffix) +``` + +## Required IAM Permissions + +### S3 Backend + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:HeadBucket", + "s3:PutBucketVersioning", + "s3:PutBucketEncryption", + "s3:PutBucketPublicAccessBlock", + "s3:PutBucketTagging" + ], + "Resource": "arn:aws:s3:::my-terraform-state*" + } + ] +} +``` + +For cross-account provisioning, also add: + +```json +{ + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::999999999999:role/TerraformStateAdmin" +} +``` + +## Migrating to Terraform-Managed Backends + +Once your backend is provisioned, you can import it into Terraform for advanced management: + +### Step 1: Provision the Backend + +Use Atmos to create the backend with secure defaults: + +```shell +atmos terraform backend create vpc --stack prod +``` + +### Step 2: Import into Terraform + +Add the backend to your Terraform configuration and import it: + +```hcl +# Import the provisioned backend +import { + to = aws_s3_bucket.terraform_state + id = "acme-ue1-prod-tfstate" +} + +resource "aws_s3_bucket" "terraform_state" { + bucket = "acme-ue1-prod-tfstate" +} +``` + +### Step 3: Add Advanced Features + +Extend the backend with production-specific features: + +```hcl +# Add lifecycle rules +resource "aws_s3_bucket_lifecycle_configuration" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + + rule { + id = "delete-old-versions" + status = "Enabled" + + noncurrent_version_expiration { + noncurrent_days = 90 + } + } +} + +# Add replication +resource "aws_s3_bucket_replication_configuration" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + role = aws_iam_role.replication.arn + + rule { + id = "replicate-state" + status = "Enabled" + + destination { + bucket = aws_s3_bucket.terraform_state_replica.arn + storage_class = "STANDARD_IA" + } + } +} +``` + +### Step 4: Disable Automatic Provisioning + +Once Terraform manages the backend, disable automatic provisioning: + +```yaml +provision: + backend: + enabled: false # Backend now managed by Terraform +``` + +See the [Automatic Backend Provisioning](/components/terraform/backend-provisioning) docs for more migration patterns. + +## Idempotent Operations + +The provision command is **idempotent**—running it multiple times is safe: + +```shell +$ atmos terraform backend create vpc --stack dev +Running backend provisioner... +Creating S3 bucket 'acme-ue1-dev-tfstate'... +✓ Successfully provisioned backend + +$ atmos terraform backend create vpc --stack dev +Running backend provisioner... +S3 bucket 'acme-ue1-dev-tfstate' already exists (idempotent) +✓ Backend provisioning completed +``` + +## Related Commands + +- [`atmos terraform init`](/cli/commands/terraform/usage) - Initialize Terraform (auto-provisions if enabled) +- [`atmos terraform plan`](/cli/commands/terraform/usage) - Plan Terraform changes (auto-provisions if enabled) +- [`atmos terraform apply`](/cli/commands/terraform/usage) - Apply Terraform changes (auto-provisions if enabled) + +## Related Concepts + +- [Stack Configuration](/learn/stacks) +- [Backend Configuration](/components/terraform/backends) +- [Authentication and Identity](/cli/commands/auth/usage) diff --git a/website/docs/components/terraform/backend-provisioning.mdx b/website/docs/components/terraform/backend-provisioning.mdx new file mode 100644 index 0000000000..724900c616 --- /dev/null +++ b/website/docs/components/terraform/backend-provisioning.mdx @@ -0,0 +1,349 @@ +--- +title: Automatic Backend Provisioning +sidebar_position: 3 +sidebar_label: Backend Provisioning +description: Automatically provision Terraform backend infrastructure with Atmos. +id: backend-provisioning +--- +import Terminal from '@site/src/components/Terminal' +import Intro from '@site/src/components/Intro' + + +Atmos can automatically provision S3 backend infrastructure before running Terraform commands. +This eliminates the manual bootstrapping step of creating state storage. + + +:::tip Related Documentation +- [Terraform Backends](/components/terraform/backends) - Configure where state is stored +- [Remote State](/components/terraform/remote-state) - Read other components' state +- [`atmos terraform backend`](/cli/commands/terraform/terraform-backend) - CLI commands for backend management +::: + +## Configuration + +Enable automatic provisioning in your stack manifests using the `provision.backend.enabled` setting: + + +```yaml +components: + terraform: + vpc: + backend_type: s3 + backend: + bucket: acme-ue1-dev-tfstate + key: vpc/terraform.tfstate + region: us-east-1 + use_lockfile: true # Enable native S3 locking (Terraform 1.10+) + + provision: + backend: + enabled: true # Enable automatic provisioning +``` + + +When enabled, Atmos will: + +1. Check if the backend exists before running Terraform commands +2. Provision the backend if it doesn't exist (with secure defaults) +3. Continue with Terraform initialization and execution + +## Configuration Hierarchy + +The `provision.backend` configuration leverages Atmos's deep-merge system, allowing you to set defaults at high levels and override per component. + +### Organization-Level Defaults + +Enable provisioning for all components in development environments: + + +```yaml +terraform: + provision: + backend: + enabled: true # All dev components inherit this +``` + + +### Environment-Specific Overrides + +Configure different provisioning policies per environment: + + +```yaml +terraform: + provision: + backend: + enabled: false # Override for specific environment +``` + + +### Component Inheritance + +Share provision configuration through catalog components: + + +```yaml +components: + terraform: + vpc/defaults: + provision: + backend: + enabled: true # Catalog default + +# stacks/dev.yaml +components: + terraform: + vpc: + metadata: + inherits: [vpc/defaults] + # Inherits provision.backend.enabled: true +``` + + +### Component-Level Override + +Override for specific components: + + +```yaml +components: + terraform: + vpc: + provision: + backend: + enabled: false # Disable for this component +``` + + +**Deep-Merge Behavior:** Atmos combines configurations from all levels, giving you maximum flexibility: +- Set defaults at organization or environment level +- Override per component when needed +- Use catalog inheritance for reusable patterns +- Component-level configuration has highest precedence + +## Supported Backend Types + +### S3 (AWS) + +The S3 backend provisioner creates buckets with hardcoded security best practices: + +- **Versioning**: Enabled (protects against accidental deletions) +- **Encryption**: AES-256 with AWS-managed keys (always enabled) +- **Public Access**: Blocked (all 4 block settings enabled) +- **Locking**: Native S3 locking (Terraform 1.10+, no DynamoDB required) +- **Tags**: Automatic resource tags (`Name`, `ManagedBy=Atmos`) + +**Required Configuration:** + + +```yaml +backend_type: s3 +backend: + bucket: my-terraform-state # Required + key: vpc/terraform.tfstate + region: us-east-1 # Required + use_lockfile: true # Enable native S3 locking (Terraform 1.10+) + +provision: + backend: + enabled: true +``` + + +**Cross-Account Provisioning:** + + +```yaml +backend: + bucket: my-terraform-state + region: us-east-1 + use_lockfile: true # Enable native S3 locking (Terraform 1.10+) + assume_role: + role_arn: arn:aws:iam::999999999999:role/TerraformStateAdmin + +provision: + backend: + enabled: true +``` + + +The provisioner will assume the specified role to create the bucket in the target account. + +## Manual Provisioning + +You can also provision backends explicitly using the CLI: + + +```shell +# Provision backend before Terraform execution +atmos terraform backend create vpc --stack dev + +# Then run Terraform +atmos terraform apply vpc --stack dev +``` + + +This is useful for: + +- CI/CD pipelines with separate provisioning stages +- Troubleshooting provisioning issues +- Batch provisioning for multiple components +- Pre-provisioning before large-scale deployments + +See [`atmos terraform backend`](/cli/commands/terraform/terraform-backend) for complete CLI documentation. + +## Required IAM Permissions + +For S3 backend provisioning, the identity needs these permissions: + + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:HeadBucket", + "s3:PutBucketVersioning", + "s3:PutBucketEncryption", + "s3:PutBucketPublicAccessBlock", + "s3:PutBucketTagging" + ], + "Resource": "arn:aws:s3:::my-terraform-state*" + } + ] +} +``` + + +For cross-account provisioning, also add: + + +```json +{ + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::999999999999:role/TerraformStateAdmin" +} +``` + + +## Solving the Terraform Bootstrap Problem + +Automatic provisioning is **fully compatible with Terraform-managed backends**. It solves a classic chicken-and-egg problem: "How do I manage my state backend with Terraform when I need that backend to exist before Terraform can run?" + +**Traditional Workaround:** +1. Use local state temporarily +2. Create S3 bucket with Terraform using local state +3. Switch backend configuration to S3 +4. Import the bucket into the S3-backed state +5. Delete local state files + +**With Atmos Automatic Provisioning:** +1. Enable `provision.backend.enabled: true` +2. Run `atmos terraform plan` - backend auto-created with secure defaults +3. Import the bucket into Terraform (no local state dance needed) +4. Done - everything managed by Terraform + +## Migrating to Terraform-Managed Backends + +Once your backend is provisioned, you can import it into Terraform for advanced management: + +**Step 1: Provision the Backend** + +Use Atmos to create the backend with secure defaults: + + +```shell +atmos terraform backend create vpc --stack prod +``` + + +**Step 2: Import into Terraform** + +Add the backend to your Terraform configuration and import it: + + +```hcl +# Import the provisioned backend +import { + to = aws_s3_bucket.terraform_state + id = "acme-ue1-prod-tfstate" +} + +resource "aws_s3_bucket" "terraform_state" { + bucket = "acme-ue1-prod-tfstate" +} + +# Add lifecycle rules +resource "aws_s3_bucket_lifecycle_configuration" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + + rule { + id = "delete-old-versions" + status = "Enabled" + + noncurrent_version_expiration { + noncurrent_days = 90 + } + } +} + +# Add replication for disaster recovery +resource "aws_s3_bucket_replication_configuration" "terraform_state" { + bucket = aws_s3_bucket.terraform_state.id + role = aws_iam_role.replication.arn + + rule { + id = "replicate-state" + status = "Enabled" + + destination { + bucket = aws_s3_bucket.terraform_state_replica.arn + storage_class = "STANDARD_IA" + } + } +} +``` + + +**Step 3: Optionally Disable Automatic Provisioning** + +Once Terraform manages the backend, you can optionally disable automatic provisioning: + + +```yaml +provision: + backend: + enabled: false # Backend now managed by Terraform +``` + + +**Note:** You can leave `provision.backend.enabled: true` even after importing to Terraform. The provisioner is idempotent - it will detect the bucket exists and skip creation, causing no conflicts with Terraform management. + +Alternatively, use the [`terraform-aws-tfstate-backend`](https://github.com/cloudposse/terraform-aws-tfstate-backend) module for backends with advanced features like cross-region replication, lifecycle policies, and custom KMS keys. + +## Idempotent Operations + +Backend provisioning is idempotent—running it multiple times is safe: + + +```shell +$ atmos terraform backend create vpc --stack dev +✓ Created S3 bucket 'acme-ue1-dev-tfstate' + +$ atmos terraform backend create vpc --stack dev +S3 bucket 'acme-ue1-dev-tfstate' already exists (idempotent) +✓ Backend provisioning completed +``` + + +## References + +- [Terraform Backends](/components/terraform/backends) - Configure backend storage +- [Remote State](/components/terraform/remote-state) - Read other components' state +- [`atmos terraform backend`](/cli/commands/terraform/terraform-backend) - CLI commands +- [Terraform Backend Configuration](https://developer.hashicorp.com/terraform/language/settings/backends/configuration) +- [`terraform-aws-tfstate-backend`](https://github.com/cloudposse/terraform-aws-tfstate-backend) - Advanced backend module diff --git a/website/docs/components/terraform/backends.mdx b/website/docs/components/terraform/backends.mdx index 8c1967463e..84840d2a57 100644 --- a/website/docs/components/terraform/backends.mdx +++ b/website/docs/components/terraform/backends.mdx @@ -11,10 +11,15 @@ import Tabs from '@theme/Tabs' import TabItem from '@theme/TabItem' -Backends define where [Terraform](https://opentofu.org/docs/language/state/) and -[OpenTofu](https://opentofu.org/docs/language/state/) store its state. +Backends define where [Terraform](https://developer.hashicorp.com/terraform/language/state) and +[OpenTofu](https://opentofu.org/docs/language/state/) store their state. +:::tip Configuration Reference +For detailed backend configuration syntax and all supported backend types, +see [Backend Configuration](/stacks/components/terraform/backend). +::: + ## Supported Backends @@ -51,401 +56,42 @@ Backends define where [Terraform](https://opentofu.org/docs/language/state/) and -## Local Backend - -By default, Terraform will use a backend called [local](https://developer.hashicorp.com/terraform/language/settings/backends/local), which stores -Terraform state on the local filesystem, locks that state using system APIs, and performs operations locally. - -Terraform's local backend is designed for development and testing purposes and is generally not recommended for production use. There are several reasons why using the local backend in a production environment may not be suitable: - -- **Not Suitable for Collaboration**: Local backend doesn't support easy state sharing. -- **No Concurrency and Locking**: Local backend lacks locking, leading to race conditions when multiple users modify the state. -- **Lacks Durability and Backup**: Local backend has no durability or backup. Machine failures can lead to data loss. -- **Unsuitable for CI/CD**: Local backend isn't ideal for CI/CD pipelines. - -To address these concerns, it's recommended to use one of the supported remote backends, such as Amazon S3, Azure Storage, Google Cloud Storage, HashiCorp Consul, or Terraform Cloud, for production environments. Remote backends provide better scalability, collaboration support, and durability, making them more suitable for managing infrastructure at scale in production environments. - -## AWS S3 Backend - -Terraform's [S3](https://developer.hashicorp.com/terraform/language/settings/backends/s3) backend is a popular remote -backend for storing Terraform state files in an Amazon Simple Storage Service (S3) bucket. Using S3 as a backend offers -many advantages, particularly in production environments. - -To configure Terraform to use an S3 backend, you typically provide the S3 bucket name and an optional key prefix in your Terraform configuration. -Here's a simplified example: - - -```hcl -terraform { - backend "s3" { - acl = "bucket-owner-full-control" - bucket = "your-s3-bucket-name" - key = "path/to/terraform.tfstate" - region = "your-aws-region" - encrypt = true - dynamodb_table = "terraform_locks" - } -} -``` - - -In the example, `terraform_locks` is a DynamoDB table used for state locking. DynamoDB is recommended for locking when using the S3 backend to ensure -safe concurrent access. - -Once the S3 bucket and DynamoDB table are provisioned, you can start using them to store Terraform state for the Terraform components. -There are two ways of doing this: - -- Manually create `backend.tf` file in each component's folder with the following content: - - -```hcl -terraform { - backend "s3" { - acl = "bucket-owner-full-control" - bucket = "your-s3-bucket-name" - dynamodb_table = "your-dynamodb-table-name" - encrypt = true - key = "terraform.tfstate" - region = "your-aws-region" - role_arn = "arn:aws:iam::xxxxxxxx:role/IAM Role with permissions to access the Terraform backend" - workspace_key_prefix = "vpc" # Stable identifier for this component - } -} -``` - - -- Configure Terraform S3 backend with Atmos to automatically generate a backend file for each Atmos component. This is the recommended way -of configuring Terraform state backend since it offers many advantages and will save you from manually creating a backend configuration file for -each component - -Configuring Terraform S3 backend with Atmos consists of three steps: - -- Set `auto_generate_backend_file` to `true` in the `atmos.yaml` CLI config file in the `components.terraform` section: - - -```yaml -components: - terraform: - # Can also be set using 'ATMOS_COMPONENTS_TERRAFORM_AUTO_GENERATE_BACKEND_FILE' ENV var, or '--auto-generate-backend-file' command-line argument - auto_generate_backend_file: true -``` - - -- Configure the S3 backend in one of the `_defaults.yaml` manifests. You can configure it for the entire Organization, or per OU/tenant, or per -region, or per account. - -:::note -The `_defaults.yaml` stack manifests contain the default settings for Organizations, Organizational Units, and accounts. -::: - -:::info -The `_defaults.yaml` stack manifests are not imported into other Atmos manifests automatically. -You need to explicitly import them using [imports](/stacks/imports). -::: - -To configure the S3 backend for the entire Organization, add the following config in `stacks/orgs/acme/_defaults.yaml`: - - -```yaml -terraform: - backend_type: s3 - backend: - s3: - acl: "bucket-owner-full-control" - encrypt: true - bucket: "your-s3-bucket-name" - dynamodb_table: "your-dynamodb-table-name" - key: "terraform.tfstate" - region: "your-aws-region" - role_arn: "arn:aws:iam::xxxxxxxx:role/IAM Role with permissions to access the Terraform backend" -``` - - -- (This step is optional) For each component, you can add `workspace_key_prefix` similar to the following: - - -```yaml -components: - terraform: - # `vpc` is the Atmos component name - vpc: - # Optional backend configuration for the component - backend: - s3: - workspace_key_prefix: vpc - metadata: - # Point to the Terraform component - component: vpc - settings: {} - vars: {} - env: {} -``` - - -Note that this is optional. If you don't add `backend.s3.workspace_key_prefix` to the component manifest, Atmos auto-generates it using this priority: - -1. `metadata.name` (logical component name) -2. `metadata.component` (physical path to Terraform code) -3. Atmos component name (YAML key) - -In all cases, `/` (slash) is replaced with `-` (dash). - -:::tip Using metadata.name for versioned components -When using [versioned component folders](/design-patterns/version-management/folder-based-versioning), set `metadata.name` to the logical component name (without version). This prevents version changes from affecting state paths: - -```yaml -vpc/defaults: - metadata: - name: vpc # Logical name → workspace_key_prefix: vpc - component: vpc/v2 # Physical path (version can change) -``` - -See [Terraform Workspaces](/components/terraform/workspaces#using-metadataname-for-stable-state-paths) for details. -::: - -Once all the above is configured, when you run the commands `atmos terraform plan vpc -s ` -or `atmos terraform apply vpc -s `, before executing the Terraform commands, Atmos will [deep-merge](#backend-inheritance) -the backend configurations from the `_defaults.yaml` manifest and from the component itself, and will generate a backend -config JSON file `backend.tf.json` in the component's folder, similar to the following example: - - -```json -{ - "terraform": { - "backend": { - "s3": { - "acl": "bucket-owner-full-control", - "bucket": "your-s3-bucket-name", - "dynamodb_table": "your-dynamodb-table-name", - "encrypt": true, - "key": "terraform.tfstate", - "region": "your-aws-region", - "role_arn": "arn:aws:iam::xxxxxxxx:role/IAM Role with permissions to access the Terraform backend", - "workspace_key_prefix": "vpc" - } - } - } -} -``` - - -You can also generate the backend configuration file for a component in a stack by executing the -command [atmos terraform generate backend](/cli/commands/terraform/generate-backend). Or generate the backend configuration files for all components -by executing the command [atmos terraform generate backends](/cli/commands/terraform/generate-backends). - -## Azure Blob Storage Backend - -[`azurerm`](https://developer.hashicorp.com/terraform/language/settings/backends/azurerm) backend stores the state as a -Blob with the given Key within the Blob Container within the Blob Storage Account. This backend supports state locking -and consistency checking with Azure Blob Storage native capabilities. - -To configure the [Azure Blob Storage backend](https://developer.hashicorp.com/terraform/language/settings/backends/azurerm) -in Atmos, add the following config to an Atmos manifest in `_defaults.yaml`: - - -```yaml -terraform: - backend_type: azurerm - backend: - azurerm: - resource_group_name: "StorageAccount-ResourceGroup" - storage_account_name: "abcd1234" - container_name: "tfstate" - # Other parameters -``` - - -For each component, you can optionally add the `key` parameter similar to the following: - - -```yaml -components: - terraform: - my-component: - # Optional backend configuration for the component - backend: - azurerm: - key: "my-component" -``` - - -If the `key` is not specified for a component, Atmos will use the component name (`my-component` in the example above) -to auto-generate the `key` parameter in the format `.terraform.tfstate` replacing `` -with the Atmos component name. In ``, all occurrences of `/` (slash) will be replaced with `-` (dash). - -If `auto_generate_backend_file` is set to `true` in the `atmos.yaml` CLI config file in the `components.terraform` section, -Atmos will [deep-merge](#backend-inheritance) the backend configurations from the `_defaults.yaml` manifests and -from the component itself, and will generate a backend config JSON file `backend.tf.json` in the component's folder, -similar to the following example: - - -```json -{ - "terraform": { - "backend": { - "azurerm": { - "resource_group_name": "StorageAccount-ResourceGroup", - "storage_account_name": "abcd1234", - "container_name": "tfstate", - "key": "my-component.terraform.tfstate" - } - } - } -} -``` - - -## Google Cloud Storage Backend - -[`gcs`](https://developer.hashicorp.com/terraform/language/settings/backends/gcs) backend stores the state as an object -in a configurable `prefix` in a pre-existing bucket on Google Cloud Storage (GCS). -The bucket must exist prior to configuring the backend. The backend supports state locking. +## Why Use Atmos for Backend Configuration? -To configure the [Google Cloud Storage backend](https://developer.hashicorp.com/terraform/language/settings/backends/gcs) -in Atmos, add the following config to an Atmos manifest in `_defaults.yaml`: +Atmos provides **one consistent way to manage backends**—both configuration AND provisioning. - -```yaml -terraform: - backend_type: gcs - backend: - gcs: - bucket: "tf-state" - # Other parameters -``` - - -For each component, you can optionally add the `prefix` parameter similar to the following: - - -```yaml -components: - terraform: - my-component: - # Optional backend configuration for the component - backend: - gcp: - prefix: "my-component" -``` - - -If the `prefix` is not specified for a component, Atmos will use the component name (`my-component` in the example above) -to auto-generate the `prefix`. In the component name, all occurrences of `/` (slash) will be replaced with `-` (dash). - -If `auto_generate_backend_file` is set to `true` in the `atmos.yaml` CLI config file in the `components.terraform` section, -Atmos will [deep-merge](#backend-inheritance) the backend configurations from the `_defaults.yaml` manifests and -from the component itself, and will generate a backend config JSON file `backend.tf.json` in the component's folder, -similar to the following example: - - -```json -{ - "terraform": { - "backend": { - "gcp": { - "bucket": "tf-state", - "prefix": "my-component" - } - } - } -} -``` - - -## Terraform Cloud Backend - -[Terraform Cloud](https://developer.hashicorp.com/terraform/cli/cloud/settings) backend uses a `cloud` block to specify -which organization and workspace(s) to use. - -To configure the [Terraform Cloud backend](https://developer.hashicorp.com/terraform/cli/cloud/settings) -in Atmos, add the following config to an Atmos manifest in `_defaults.yaml`: +### Configure Backends - -```yaml -terraform: - backend_type: cloud - backend: - cloud: - organization: "my-org" - hostname: "app.terraform.io" - workspaces: - # Parameters for workspaces -``` - +Atmos generates `backend.tf.json` dynamically from your stack manifests: -For each component, you can optionally specify the `workspaces.name` parameter similar to the following: +- **DRY Configuration**: Define backend defaults once at the org level, override per environment using [inheritance](#backend-inheritance) +- **Reuse Root Modules Across Environments**: Deploy the same module to dev, staging, and prod with different backend configurations +- **Use ANY Module as a Root Module**: Your Terraform modules don't need `backend.tf` files—Atmos generates them +- **Works with Terraform AND OpenTofu**: Same stack configuration works with either runtime - -```yaml -components: - terraform: - my-component: - # Optional backend configuration for the component - backend: - cloud: - workspaces: - name: "my-component-workspace" -``` - +### Provision Backends -If `auto_generate_backend_file` is set to `true` in the `atmos.yaml` CLI config file in the `components.terraform` section, -Atmos will [deep-merge](#backend-inheritance) the backend configurations from the `_defaults.yaml` manifests and -from the component itself, and will generate a backend config JSON file `backend.tf.json` in the component's folder, -similar to the following example: +Atmos can create backend infrastructure automatically. Currently supports S3 (AWS), with more backends planned: - -```json -{ - "terraform": { - "cloud": { - "hostname": "app.terraform.io", - "organization": "my-org", - "workspaces": { - "name": "my-component-workspace" - } - } - } -} -``` - - -Instead of specifying the `workspaces.name` parameter for each component in the component manifests, you can use -the `{terraform_workspace}` token in the `cloud` backend config in the `_defaults.yaml` manifest. -The token `{terraform_workspace}` will be automatically replaced by Atmos with the Terraform workspace for each component. -This will make the entire configuration DRY. +- **Solves the Bootstrap Problem**: No chicken-and-egg with state storage +- **Secure Defaults**: Versioning, encryption, public access blocked +- **Cross-Account Support**: Use nested `assume_role.role_arn` to provision backends in different AWS accounts - -```yaml -terraform: - backend_type: cloud - backend: - cloud: - organization: "my-org" - hostname: "app.terraform.io" - workspaces: - # The token `{terraform_workspace}` will be automatically replaced with the - # Terraform workspace for each Atmos component - name: "{terraform_workspace}" -``` - +See [Backend Provisioning](/components/terraform/backend-provisioning) for details. -:::tip -Refer to [Terraform Workspaces in Atmos](/components/terraform/workspaces) for more information on how -Atmos calculates Terraform workspaces for components, and how workspaces can be overridden for each component. +:::note Backend Configuration vs. Provisioning +The examples below show **backend configuration** using root-level `role_arn`—this is the role Terraform assumes to *access* state. For **backend provisioning** (creating the bucket), use the nested `assume_role.role_arn` format shown in [Backend Provisioning](/components/terraform/backend-provisioning#cross-account-provisioning). ::: ## Backend Inheritance -Suppose that for security and audit reasons, you want to use different Terraform backends for `dev`, `staging` and `prod`. -Each account needs to have a separate S3 bucket, DynamoDB table, and IAM role with different permissions -(for example, the `development` Team should be able to access the Terraform backend only in the `dev` account, but not in `staging` and `prod`). +Atmos supports deep-merging of backend configuration across stack manifests, enabling you to define defaults at higher levels and override per environment. -Atmos supports this use-case by using deep-merging of stack manifests, [Imports](/stacks/imports) -and [Inheritance](/howto/inheritance), which makes the backend configuration reusable and DRY. +Suppose you want different backends for `dev`, `staging`, and `prod`—each with separate S3 buckets and IAM roles for security and audit purposes. -We'll split the backend config between the Organization and the accounts. +### Organization-Level Defaults -Add the following config to the Organization stack manifest in `stacks/orgs/acme/_defaults.yaml`: +Define common settings at the organization level: ```yaml @@ -456,69 +102,53 @@ terraform: acl: "bucket-owner-full-control" encrypt: true key: "terraform.tfstate" - region: "your-aws-region" + region: us-east-1 ``` -Add the following config to the `dev` stack manifest in `stacks/orgs/acme/plat/dev/_defaults.yaml`: +### Environment-Level Overrides - -```yaml -terraform: - backend_type: s3 - backend: - s3: - bucket: "your-dev-s3-bucket-name" - dynamodb_table: "your-dev-dynamodb-table-name" - role_arn: "IAM Role with permissions to access the 'dev' Terraform backend" -``` - +Override bucket and credentials per environment: -Add the following config to the `staging` stack manifest in `stacks/orgs/acme/plat/staging/_defaults.yaml`: - - + ```yaml terraform: - backend_type: s3 backend: s3: - bucket: "your-staging-s3-bucket-name" - dynamodb_table: "your-staging-dynamodb-table-name" - role_arn: "IAM Role with permissions to access the 'staging' Terraform backend" + bucket: acme-ue1-dev-tfstate + use_lockfile: true + role_arn: "arn:aws:iam::111111111111:role/TerraformStateDev" ``` -Add the following config to the `prod` stack manifest in `stacks/orgs/acme/plat/prod/_defaults.yaml`: - ```yaml terraform: - backend_type: s3 backend: s3: - bucket: "your-prod-s3-bucket-name" - dynamodb_table: "your-prod-dynamodb-table-name" - role_arn: "IAM Role with permissions to access the 'prod' Terraform backend" + bucket: acme-ue1-prod-tfstate + use_lockfile: true + role_arn: "arn:aws:iam::222222222222:role/TerraformStateProd" ``` -When you provision the `vpc` component into the `dev` account (by executing the command `atmos terraform apply vpc -s plat-ue2-dev`), Atmos will -deep-merge the backend configuration from the Organization-level manifest with the configuration from the `dev` manifest, and will automatically -add `workspace_key_prefix` for the component, generating the following final deep-merged backend config for the `vpc` component in the `dev` account: +### Deep-Merged Result - +When you run `atmos terraform apply vpc -s plat-ue1-dev`, Atmos deep-merges all configurations: + + ```json { "terraform": { "backend": { "s3": { "acl": "bucket-owner-full-control", - "bucket": "your-dev-s3-bucket-name", - "dynamodb_table": "your-dev-dynamodb-table-name", + "bucket": "acme-ue1-dev-tfstate", "encrypt": true, "key": "terraform.tfstate", - "region": "your-aws-region", - "role_arn": "", + "region": "us-east-1", + "role_arn": "arn:aws:iam::111111111111:role/TerraformStateDev", + "use_lockfile": true, "workspace_key_prefix": "vpc" } } @@ -527,139 +157,44 @@ add `workspace_key_prefix` for the component, generating the following final dee ``` -In the same way, you can create different Terraform backends per Organizational Unit, per region, per account (or a group of accounts, e.g. `prod` -and `non-prod`), or even per component or a set of components (e.g. root-level components like `account` and IAM roles can have a separate backend), -and then configure parts of the backend config in the corresponding Atmos stack manifests. Atmos will deep-merge all the parts from the -different scopes and generate the final backend config for the components in the stacks. - -## Terraform/OpenTofu Backend with Multiple Component Instances +You can create different backends per Organizational Unit, region, account group (e.g., `prod` vs `non-prod`), or even per component. Atmos deep-merges all parts from different scopes into the final backend config. -We mentioned before that you can configure the Terraform backend for the components manually (by creating a file `backend.tf` in each Terraform -component's folder), or you can set up Atmos to generate the backend configuration for each component in the stacks automatically. While -auto-generating the backend config file is helpful and saves you from creating the backend files for each component, it becomes a requirement -when you provision multiple instances of a Terraform component into the same environment (same account and region). +## Multiple Component Instances -You can provision more than one instance of the same Terraform component (with the same or different settings) into the same environment by defining -many Atmos components that provide configuration for the Terraform component. - -:::tip -For more information on configuring and provision multiple instances of a Terraform component, -refer to [Multiple Component Instances Atmos Design Patterns](/design-patterns/inheritance-patterns/multiple-component-instances) -::: - -For example, the following config shows how to define two Atmos -components, `vpc/1` and `vpc/2`, which both point to the same Terraform component `vpc`: +When you deploy multiple instances of the same Terraform component to the same environment, each instance needs its own state. Atmos handles this automatically by using the Atmos component name as the `workspace_key_prefix`. ```yaml -import: - # Import the defaults for all VPC components - - catalog/vpc/defaults - components: terraform: - # Atmos component `vpc/1` + # First VPC instance vpc/1: metadata: - # Point to the Terraform component in `components/terraform/vpc` component: vpc - # Inherit the defaults for all VPC components - inherits: - - vpc/defaults - # Define variables specific to this `vpc/1` component vars: name: vpc-1 ipv4_primary_cidr_block: 10.9.0.0/18 - # Optional backend configuration for the component - # If not specified, the Atmos component name `vpc/1` will be used (`/` will be replaced with `-`) - backend: - s3: - workspace_key_prefix: vpc-1 - # Atmos component `vpc/2` + # Second VPC instance vpc/2: metadata: - # Point to the Terraform component in `components/terraform/vpc` component: vpc - # Inherit the defaults for all VPC components - inherits: - - vpc/defaults - # Define variables specific to this `vpc/2` component vars: name: vpc-2 ipv4_primary_cidr_block: 10.10.0.0/18 - # Optional backend configuration for the component - # If not specified, the Atmos component name `vpc/2` will be used (`/` will be replaced with `-`) - backend: - s3: - workspace_key_prefix: vpc-2 ``` -If we manually create a `backend.tf` file for the `vpc` Terraform component in the `components/terraform/vpc` folder -using `workspace_key_prefix: "vpc"`, then both `vpc/1` and `vpc/2` Atmos components will use the same `workspace_key_prefix`, and they will -not function correctly. +Atmos generates separate `workspace_key_prefix` values (`vpc-1` and `vpc-2`), ensuring each instance has its own state file. -On the other hand, if we configure Atmos to auto-generate the backend config file, then each component will have a different `workspace_key_prefix` -auto-generated by Atmos by using the Atmos component name (or you can override this behavior by specifying `workspace_key_prefix` for each component -in the component manifest in the `backend.s3.workspace_key_prefix` section). - -For example, when the command `atmos terraform apply vpc/1 -s plat-ue2-dev` is executed, the following `backend.tf.json` file is generated in the -`components/terraform/vpc` folder: - - -```json -{ - "terraform": { - "backend": { - "s3": { - "acl": "bucket-owner-full-control", - "bucket": "your-dev-s3-bucket-name", - "dynamodb_table": "your-dev-dynamodb-table-name", - "encrypt": true, - "key": "terraform.tfstate", - "region": "your-aws-region", - "role_arn": "", - "workspace_key_prefix": "vpc-1" - } - } - } -} -``` - - -Similarly, when the command `atmos terraform apply vpc/2 -s plat-ue2-dev` is executed, the following `backend.tf.json` file is generated in the -`components/terraform/vpc` folder: - - -```json -{ - "terraform": { - "backend": { - "s3": { - "acl": "bucket-owner-full-control", - "bucket": "your-dev-s3-bucket-name", - "dynamodb_table": "your-dev-dynamodb-table-name", - "encrypt": true, - "key": "terraform.tfstate", - "region": "your-aws-region", - "role_arn": "", - "workspace_key_prefix": "vpc-2" - } - } - } -} -``` - - -The generated files will have different `workspace_key_prefix` attribute auto-generated by Atmos. - -For this reason, configuring Atmos to auto-generate the backend configuration for the components in the stacks is recommended -for all supported backend types. +:::tip +For more patterns on multiple component instances, see [Multiple Component Instances](/design-patterns/inheritance-patterns/multiple-component-instances). +::: -## References +## Related -- [Terraform Backend Configuration](https://developer.hashicorp.com/terraform/language/settings/backends/configuration) -- [OpenTofu Backend Configuration](https://opentofu.org/docs/language/settings/backends/configuration) -- [Terraform Cloud Settings](https://developer.hashicorp.com/terraform/cli/cloud/settings) -- [Multiple Component Instances Atmos Design Patterns](/design-patterns/inheritance-patterns/multiple-component-instances) +- [Backend Configuration](/stacks/backend) - Detailed configuration reference +- [Terraform Backend Configuration](/stacks/components/terraform/backend) - Component-level backend defaults +- [Backend Provisioning](/components/terraform/backend-provisioning) - Automatic backend creation +- [Remote State](/components/terraform/remote-state) - Read other components' state +- [Terraform Workspaces](/components/terraform/workspaces) diff --git a/website/docs/components/terraform/state-backend.mdx b/website/docs/components/terraform/remote-state.mdx similarity index 75% rename from website/docs/components/terraform/state-backend.mdx rename to website/docs/components/terraform/remote-state.mdx index 00d76cbfd9..bc5d959a6e 100644 --- a/website/docs/components/terraform/state-backend.mdx +++ b/website/docs/components/terraform/remote-state.mdx @@ -1,8 +1,9 @@ --- -title: State Backend Configuration -sidebar_position: 3 -sidebar_label: Backend Configuration -id: state-backend +title: Terraform Remote State +sidebar_position: 4 +sidebar_label: Remote State +description: Configure remote state access for reading other components' outputs. +id: remote-state --- import Intro from '@site/src/components/Intro' @@ -13,11 +14,16 @@ and [Remote State](/stacks/remote-state) to get the outputs of a [Terraform/Open provisioned in the same or a different [Atmos stack](/learn/stacks), and use the outputs as inputs to another Atmos component. -Bear in mind that Atmos is simply managing the configuration of the Backend; -provisioning the backend resources themselves is the responsibility of a Terraform/OpenTofu component. +:::tip Related Documentation +- [Terraform Backends](/components/terraform/backends) - Configure where state is stored +- [Backend Provisioning](/components/terraform/backend-provisioning) - Auto-create backends +- [Remote State Data Sharing](/stacks/remote-state) - Using remote state in components +::: -Atmos also supports Remote State Backends (in the `remote_state_backend` section), which can be used to configure the -following: +## Remote State Backend Configuration + +Atmos supports the `remote_state_backend` section which can be used to configure how components access the remote state +of other components. This is useful for: - Override [Terraform Backend](/components/terraform/backends) configuration to access the remote state of a component (e.g. override the IAM role to assume, which in this case can be a read-only role) @@ -25,10 +31,7 @@ following: - Configure a remote state of type `static` which can be used to provide configurations for [Brownfield development](https://en.wikipedia.org/wiki/Brownfield_(software_development)) -## Override Terraform Backend Configuration to Access Remote State - -Atmos supports the `remote_state_backend` section which can be used to provide configuration to access the remote state -of components. +## Override Backend Configuration for Remote State Access To access the remote state of components, you can override any [Terraform Backend](/components/terraform/backends) @@ -37,7 +40,7 @@ is a first-class section, and it can be defined globally at any scope (organizat component, and then deep-merged using [Atmos Component Inheritance](/howto/inheritance). For example, let's suppose we have the following S3 backend configuration for the entire organization -(refer to [AWS S3 Backend](/components/terraform/backends#aws-s3-backend) for more details): +(refer to [Backend Configuration](/stacks/backend#s3-backend) for more details): ```yaml title="stacks/orgs/acme/_defaults.yaml" terraform: @@ -47,10 +50,10 @@ terraform: acl: "bucket-owner-full-control" encrypt: true bucket: "your-s3-bucket-name" - dynamodb_table: "your-dynamodb-table-name" key: "terraform.tfstate" region: "your-aws-region" role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-write" + use_lockfile: true ``` Let's say we also have a read-only IAM role, and we want to use it to access the remote state instead of the read-write @@ -68,10 +71,10 @@ terraform: acl: "bucket-owner-full-control" encrypt: true bucket: "your-s3-bucket-name" - dynamodb_table: "your-dynamodb-table-name" key: "terraform.tfstate" region: "your-aws-region" role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-write" + use_lockfile: true remote_state_backend_type: s3 # s3, remote, vault, azurerm, gcs, cloud, static remote_state_backend: @@ -86,3 +89,10 @@ deep-merges the `remote_state_backend` section with the `backend` section). When working with Terraform backends and writing/updating the state, the `terraform-backend-read-write` role will be used. But when reading the remote state of components, the `terraform-backend-read-only` role will be used. + +## References + +- [Terraform Backends](/components/terraform/backends) - Configure backend storage +- [Backend Provisioning](/components/terraform/backend-provisioning) - Auto-create backends +- [Remote State Data Sharing](/stacks/remote-state) - Using remote state in components +- [Terraform Remote State](https://developer.hashicorp.com/terraform/language/state/remote-state-data) diff --git a/website/docs/design-patterns/stack-organization/defaults-pattern.mdx b/website/docs/design-patterns/stack-organization/defaults-pattern.mdx index 2886767778..5bdced744a 100644 --- a/website/docs/design-patterns/stack-organization/defaults-pattern.mdx +++ b/website/docs/design-patterns/stack-organization/defaults-pattern.mdx @@ -201,8 +201,8 @@ vars: terraform: backend: s3: - bucket: "acme-terraform-state" - dynamodb_table: "acme-terraform-state-lock" + bucket: "acme-gbl-root-tfstate" + use_lockfile: true ``` ```yaml title="stacks/orgs/acme/plat/_defaults.yaml" diff --git a/website/docs/quick-start/advanced/advanced.mdx b/website/docs/quick-start/advanced/advanced.mdx index 22627448d9..8aa86c7f25 100644 --- a/website/docs/quick-start/advanced/advanced.mdx +++ b/website/docs/quick-start/advanced/advanced.mdx @@ -23,8 +23,8 @@ You can clone it and configure to your own needs. The repository should be a goo ::: -In this advanced tutorial, we’ll delve into concepts like [inheritance](/howto/inheritance) -and [state management](/components/terraform/state-backend). +In this advanced tutorial, we'll delve into concepts like [inheritance](/howto/inheritance) +and [state management](/components/terraform/remote-state). Additionally, we’ll cover how to read the remote state from other components using native Terraform. This example focuses on AWS, and while Atmos isn’t AWS-specific, this tutorial will be. diff --git a/website/docs/quick-start/advanced/configure-terraform-backend.mdx b/website/docs/quick-start/advanced/configure-terraform-backend.mdx index 900ed8dcb5..1eb19a1b4c 100644 --- a/website/docs/quick-start/advanced/configure-terraform-backend.mdx +++ b/website/docs/quick-start/advanced/configure-terraform-backend.mdx @@ -78,28 +78,35 @@ Here's a simplified example: ```hcl terraform { backend "s3" { - acl = "bucket-owner-full-control" - bucket = "your-s3-bucket-name" - key = "path/to/terraform.tfstate" - region = "your-aws-region" - encrypt = true - dynamodb_table = "terraform_locks" + acl = "bucket-owner-full-control" + bucket = "your-s3-bucket-name" + key = "path/to/terraform.tfstate" + region = "your-aws-region" + encrypt = true + use_lockfile = true # Native S3 locking (Terraform 1.10+) } } ``` -In the example, `terraform_locks` is a DynamoDB table used for state locking. DynamoDB is recommended for locking when using the S3 backend to ensure -safe concurrent access. +:::note Native S3 Locking +Terraform 1.10+ and OpenTofu 1.8+ support native S3 locking via `use_lockfile: true`, eliminating the need for a +DynamoDB table. For older versions, see [legacy DynamoDB locking](https://developer.hashicorp.com/terraform/language/settings/backends/s3#dynamodb-state-locking). +::: ## Provision Terraform S3 Backend -Before using Terraform S3 backend, a backend S3 bucket and DynamoDB table need to be provisioned. +Before using Terraform S3 backend, a backend S3 bucket needs to be provisioned. + +:::tip Automatic Backend Provisioning +Atmos can automatically provision S3 backends with secure defaults. See [Backend Provisioning](/components/terraform/backend-provisioning) +for the fastest way to get started. +::: -You can provision them using the [tfstate-backend](https://github.com/cloudposse/terraform-aws-tfstate-backend) Terraform module and +Alternatively, you can provision the S3 bucket using the [tfstate-backend](https://github.com/cloudposse/terraform-aws-tfstate-backend) Terraform module and [tfstate-backend](https://github.com/cloudposse/terraform-aws-components/tree/main/modules/tfstate-backend) Terraform component (root module). -Note that the [tfstate-backend](https://github.com/cloudposse/terraform-aws-components/tree/main/modules/tfstate-backend) Terraform component +The [tfstate-backend](https://github.com/cloudposse/terraform-aws-components/tree/main/modules/tfstate-backend) Terraform component can be added to the `components/terraform` folder, the configuration for the component can be added to the `stacks`, and the component itself can be provisioned with Atmos. @@ -121,7 +128,7 @@ components: ## Configure Terraform S3 Backend -Once the S3 bucket and DynamoDB table are provisioned, you can start using them to store Terraform state for the Terraform components. +Once the S3 bucket is provisioned, you can start using it to store Terraform state for the Terraform components. There are two ways of doing this: - Manually create `backend.tf` file in each component's folder with the following content: @@ -131,12 +138,12 @@ There are two ways of doing this: backend "s3" { acl = "bucket-owner-full-control" bucket = "your-s3-bucket-name" - dynamodb_table = "your-dynamodb-table-name" encrypt = true key = "terraform.tfstate" region = "your-aws-region" role_arn = "arn:aws:iam:::role/" - workspace_key_prefix = "vpc" # Or any stable identifier for your component + workspace_key_prefix = "" + use_lockfile = true } } ``` @@ -185,10 +192,10 @@ Configuring Terraform S3 backend with Atmos consists of the three steps: acl: "bucket-owner-full-control" encrypt: true bucket: "your-s3-bucket-name" - dynamodb_table: "your-dynamodb-table-name" key: "terraform.tfstate" region: "your-aws-region" role_arn: "arn:aws:iam:::role/" + use_lockfile: true ``` @@ -230,11 +237,11 @@ similar to the following example: "s3": { "acl": "bucket-owner-full-control", "bucket": "your-s3-bucket-name", - "dynamodb_table": "your-dynamodb-table-name", "encrypt": true, "key": "terraform.tfstate", "region": "your-aws-region", "role_arn": "arn:aws:iam:::role/", + "use_lockfile": true, "workspace_key_prefix": "vpc" } } @@ -250,11 +257,11 @@ by executing the command [atmos terraform generate backends](/cli/commands/terra ## Terraform Backend Inheritance In the previous section, we configured the S3 backend for the entire Organization by adding the `terraform.backend.s3` section to -the `stacks/orgs/acme/_defaults.yaml` stack manifest. The same backend configuration (S3 bucket, DynamoDB table, and IAM role) will be used for all +the `stacks/orgs/acme/_defaults.yaml` stack manifest. The same backend configuration (S3 bucket and IAM role) will be used for all OUs, accounts and regions. Suppose that for security and audit reasons, you want to use different Terraform backends for the `dev`, `staging` and `prod` accounts. Each account -needs to have a separate S3 bucket, DynamoDB table, and IAM role with different permissions (for example, the `development` Team should be able to +needs to have a separate S3 bucket and IAM role with different permissions (for example, the `development` Team should be able to access the Terraform backend only in the `dev` account, but not in `staging` and `prod`). Atmos supports this use-case by using deep-merging of stack manifests, [Imports](/stacks/imports) @@ -273,6 +280,7 @@ Add the following config to the Organization stack manifest in `stacks/orgs/acme encrypt: true key: "terraform.tfstate" region: "your-aws-region" + use_lockfile: true ``` @@ -285,7 +293,6 @@ Add the following config to the `dev` stack manifest in `stacks/orgs/acme/plat/d backend: s3: bucket: "your-dev-s3-bucket-name" - dynamodb_table: "your-dev-dynamodb-table-name" role_arn: "" ``` @@ -298,7 +305,6 @@ Add the following config to the `staging` stack manifest in `stacks/orgs/acme/pl backend: s3: bucket: "your-staging-s3-bucket-name" - dynamodb_table: "your-staging-dynamodb-table-name" role_arn: "" ``` @@ -312,7 +318,6 @@ Add the following config to the `prod` stack manifest in `stacks/orgs/acme/plat/ backend: s3: bucket: "your-prod-s3-bucket-name" - dynamodb_table: "your-prod-dynamodb-table-name" role_arn: "" ``` @@ -329,11 +334,11 @@ add `workspace_key_prefix` for the component, generating the following final dee "s3": { "acl": "bucket-owner-full-control", "bucket": "your-dev-s3-bucket-name", - "dynamodb_table": "your-dev-dynamodb-table-name", "encrypt": true, "key": "terraform.tfstate", "region": "your-aws-region", "role_arn": "", + "use_lockfile": true, "workspace_key_prefix": "vpc" } } @@ -430,11 +435,11 @@ For example, when the command `atmos terraform apply vpc/1 -s plat-ue2-dev` is e "s3": { "acl": "bucket-owner-full-control", "bucket": "your-dev-s3-bucket-name", - "dynamodb_table": "your-dev-dynamodb-table-name", "encrypt": true, "key": "terraform.tfstate", "region": "your-aws-region", "role_arn": "", + "use_lockfile": true, "workspace_key_prefix": "vpc-1" } } @@ -454,11 +459,11 @@ Similarly, when the command `atmos terraform apply vpc/2 -s plat-ue2-dev` is exe "s3": { "acl": "bucket-owner-full-control", "bucket": "your-dev-s3-bucket-name", - "dynamodb_table": "your-dev-dynamodb-table-name", "encrypt": true, "key": "terraform.tfstate", "region": "your-aws-region", "role_arn": "", + "use_lockfile": true, "workspace_key_prefix": "vpc-2" } } diff --git a/website/docs/stacks/_partials/_backend-config.mdx b/website/docs/stacks/_partials/_backend-config.mdx new file mode 100644 index 0000000000..a6c54e7294 --- /dev/null +++ b/website/docs/stacks/_partials/_backend-config.mdx @@ -0,0 +1,437 @@ +import File from '@site/src/components/File' +import Tabs from '@theme/Tabs' +import TabItem from '@theme/TabItem' + +## Backend Types + +Atmos supports all Terraform backend types. Configure using `backend_type` and the corresponding configuration under `backend`. + +### S3 Backend + +The most common backend for AWS environments. We recommend using `use_lockfile: true` for native S3 state locking (Terraform 1.10+, OpenTofu 1.8+) instead of DynamoDB: + + + + ```yaml + terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-root-tfstate + region: us-east-1 + key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" + encrypt: true + use_lockfile: true # Native S3 locking (Terraform 1.10+) + ``` + + + ```json title="components/terraform/vpc/backend.tf.json" + { + "terraform": { + "backend": { + "s3": { + "bucket": "acme-ue1-root-tfstate", + "region": "us-east-1", + "key": "ue1/prod/vpc/terraform.tfstate", + "encrypt": true, + "use_lockfile": true + } + } + } + } + ``` + + + +#### S3 Backend with Assume Role + +For cross-account access, configure the backend to assume a role: + + + + ```yaml + terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-root-tfstate + region: us-east-1 + key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" + encrypt: true + use_lockfile: true + assume_role: + role_arn: "arn:aws:iam::{{ .vars.state_account_id }}:role/TerraformStateAccess" + session_name: "atmos-{{ .component }}" + ``` + + + ```json title="components/terraform/vpc/backend.tf.json" + { + "terraform": { + "backend": { + "s3": { + "bucket": "acme-ue1-root-tfstate", + "region": "us-east-1", + "key": "ue1/prod/vpc/terraform.tfstate", + "encrypt": true, + "use_lockfile": true, + "assume_role": { + "role_arn": "arn:aws:iam::123456789012:role/TerraformStateAccess", + "session_name": "atmos-vpc" + } + } + } + } + } + ``` + + + +#### S3 Backend with DynamoDB Locking (Legacy) + +For Terraform versions before 1.10, use DynamoDB for state locking: + +```yaml +terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-root-tfstate + region: us-east-1 + key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" + dynamodb_table: acme-ue1-root-tfstate-lock + encrypt: true +``` + +### Azure Blob Backend + +For Azure environments: + + + + ```yaml + terraform: + backend_type: azurerm + backend: + azurerm: + resource_group_name: terraform-state-rg + storage_account_name: tfstateaccount + container_name: tfstate + key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" + ``` + + + ```json title="components/terraform/vpc/backend.tf.json" + { + "terraform": { + "backend": { + "azurerm": { + "resource_group_name": "terraform-state-rg", + "storage_account_name": "tfstateaccount", + "container_name": "tfstate", + "key": "ue1/prod/vpc/terraform.tfstate" + } + } + } + } + ``` + + + +### GCS Backend + +For Google Cloud environments: + + + + ```yaml + terraform: + backend_type: gcs + backend: + gcs: + bucket: acme-terraform-state + prefix: "{{ .environment }}/{{ .stage }}/{{ .component }}" + ``` + + + ```json title="components/terraform/vpc/backend.tf.json" + { + "terraform": { + "backend": { + "gcs": { + "bucket": "acme-terraform-state", + "prefix": "ue1/prod/vpc" + } + } + } + } + ``` + + + +### Terraform Cloud / Enterprise + +For Terraform Cloud or Enterprise: + + + + ```yaml + terraform: + backend_type: remote + backend: + remote: + hostname: app.terraform.io + organization: acme + workspaces: + name: "{{ .namespace }}-{{ .environment }}-{{ .stage }}-{{ .component }}" + ``` + + Or using the newer `cloud` backend: + + ```yaml + terraform: + backend_type: cloud + backend: + cloud: + organization: acme + workspaces: + name: "{{ .namespace }}-{{ .environment }}-{{ .stage }}-{{ .component }}" + ``` + + + ```json title="components/terraform/vpc/backend.tf.json" + { + "terraform": { + "backend": { + "remote": { + "hostname": "app.terraform.io", + "organization": "acme", + "workspaces": { + "name": "acme-ue1-prod-vpc" + } + } + } + } + } + ``` + + + +### Local Backend + +For development or single-user scenarios: + + + + ```yaml + terraform: + backend_type: local + backend: + local: + path: "{{ .component }}/terraform.tfstate" + ``` + + + ```json title="components/terraform/vpc/backend.tf.json" + { + "terraform": { + "backend": { + "local": { + "path": "vpc/terraform.tfstate" + } + } + } + } + ``` + + + +## Remote State Backend + +The `remote_state_backend` configuration is separate from `backend` and controls how other components can read this component's state: + +```yaml +terraform: + # Where this component stores its state + backend_type: s3 + backend: + s3: + bucket: acme-ue1-root-tfstate + region: us-east-1 + + # How other components read this component's state + remote_state_backend_type: s3 + remote_state_backend: + s3: + bucket: acme-ue1-root-tfstate + region: us-east-1 +``` + +This separation allows for scenarios like: +- Using Terraform Cloud for operations but S3 for remote state access +- Different credentials for writing vs. reading state +- Custom remote state configurations + +## Backend Provisioning + +Atmos can automatically provision S3 backend infrastructure before running Terraform commands, solving the bootstrap problem of "how do I create state storage before I can use Terraform?" + +### Enable Automatic Provisioning + +```yaml +terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-dev-tfstate + region: us-east-1 + use_lockfile: true + + provision: + backend: + enabled: true # Automatically create backend if it doesn't exist +``` + +When `provision.backend.enabled` is `true`, Atmos will: +1. Check if the backend exists before running Terraform +2. Create it with secure defaults (versioning, encryption, public access blocked) +3. Proceed with normal Terraform operations + +See [Backend Provisioning](/components/terraform/backend-provisioning) for complete documentation. + +## Examples + +### Environment-Specific State Buckets + + +```yaml +terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-dev-tfstate + region: us-east-1 + encrypt: true + use_lockfile: true +``` + + + +```yaml +terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-prod-tfstate + region: us-east-1 + encrypt: true + use_lockfile: true +``` + + +### Dynamic State Keys with Templates + + +```yaml +terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-root-tfstate + region: "{{ .vars.region }}" + key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" + encrypt: true + use_lockfile: true +``` + + +### Component-Specific Backend Override + + +```yaml +import: + - orgs/acme/_defaults + +components: + terraform: + # Uses default backend + vpc: + vars: + vpc_cidr: "10.0.0.0/16" + + # Uses custom backend for sensitive data + secrets-manager: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-prod-secrets-tfstate + region: us-east-1 + key: "secrets/terraform.tfstate" + encrypt: true + use_lockfile: true + kms_key_id: alias/terraform-state-key + vars: + # ... +``` + + +### Multi-Cloud Configuration + + +```yaml +terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-root-tfstate + region: us-east-1 + use_lockfile: true +``` + + + +```yaml +terraform: + backend_type: azurerm + backend: + azurerm: + resource_group_name: terraform-state-rg + storage_account_name: acmetfstate + container_name: tfstate +``` + + +## Workspace Configuration + +For backends that support workspaces, you can configure workspace patterns: + +```yaml +terraform: + backend_type: remote + backend: + remote: + organization: acme + workspaces: + prefix: "acme-" + +components: + terraform: + vpc: + terraform_workspace: "{{ .environment }}-{{ .stage }}-vpc" +``` + +See [Terraform Workspaces](/components/terraform/workspaces) for detailed workspace configuration. + +## Best Practices + +1. **Use Remote State:** Always use a remote backend for team environments to enable collaboration and state locking. + +2. **Enable Encryption:** Always enable encryption for backends that support it (S3, Azure Blob, GCS). + +3. **Configure State Locking:** Use `use_lockfile: true` for S3 (Terraform 1.10+), blob leases for Azure, or built-in locking for other backends. + +4. **Organize State Keys:** Use a consistent key structure that includes environment, stage, and component information. + +5. **Separate Sensitive State:** Consider using separate state buckets or additional encryption for components managing secrets. + +6. **Use Templates:** Leverage Go templates for dynamic backend configuration based on context variables. diff --git a/website/docs/stacks/backend.mdx b/website/docs/stacks/backend.mdx index 00c181931b..76d88b8eaa 100644 --- a/website/docs/stacks/backend.mdx +++ b/website/docs/stacks/backend.mdx @@ -10,19 +10,24 @@ import File from '@site/src/components/File' import Intro from '@site/src/components/Intro' import Tabs from '@theme/Tabs' import TabItem from '@theme/TabItem' +import BackendConfig from './_partials/_backend-config.mdx' -The `backend` section generates Terraform backend configuration files (`backend.tf.json`) for your components. This allows you to manage state storage declaratively in your stack configuration, with Atmos automatically generating the backend files when you run Terraform commands. This section is **Terraform-specific**. +The `backend` section generates Terraform backend configuration files (`backend.tf.json`) +for your components. Define backend settings once in your stacks and Atmos generates +the appropriate files for each environment. ## How It Works When you run any `atmos terraform` command, Atmos: 1. Reads your `backend` and `backend_type` configuration from the stack -2. Generates a `backend.tf.json` file in the component directory -3. Terraform uses this file to configure state storage without modifying your source code +2. Deep-merges settings from all inherited stack manifests +3. Generates a `backend.tf.json` file in the component directory +4. Terraform uses this file to configure state storage -This approach lets you define backend configuration once in your stacks and have Atmos generate the appropriate files for each environment, with dynamic values like state keys computed from context variables. +This separation means your Terraform modules stay clean—no hardcoded backend +configuration in your source code. :::tip Generated Files Add `backend.tf.json` to your `.gitignore` since these files are generated automatically by Atmos and should not be committed to version control: @@ -31,18 +36,29 @@ Add `backend.tf.json` to your `.gitignore` since these files are generated autom # Atmos generated files backend.tf.json ``` + +Some automation systems may require the generated file to be committed—in those cases, +committing `backend.tf.json` is acceptable. ::: ## Use Cases - **State Management:** Configure S3, Azure Blob, GCS, or other backends for remote state storage. - **Environment Isolation:** Use different state storage per environment or account. -- **State Locking:** Configure DynamoDB, Azure Blob leases, or other locking mechanisms. +- **State Locking:** Configure native locking (`use_lockfile`) or DynamoDB for legacy setups. - **Remote State Access:** Configure `remote_state_backend` for cross-component state references. ## Configuration Scopes -The backend configuration can be defined at two levels: +Backend settings can be defined at multiple levels, with more specific scopes +overriding broader ones: + +| Scope | Example File | Effect | +|-------|-------------|--------| +| Organization | `stacks/orgs/acme/_defaults.yaml` | All components inherit | +| Account/Stage | `stacks/orgs/acme/plat/prod/_defaults.yaml` | Override for prod | +| Component-type | Under `terraform:` in any stack | All Terraform components | +| Component | Under `components.terraform.:` | Single component | ### Component-Type Level @@ -55,10 +71,10 @@ Backend settings defined under `terraform` apply to all Terraform components: backend_type: s3 backend: s3: - bucket: acme-terraform-state + bucket: acme-ue1-root-tfstate region: us-east-1 - dynamodb_table: terraform-locks encrypt: true + use_lockfile: true ``` @@ -67,10 +83,10 @@ Backend settings defined under `terraform` apply to all Terraform components: "terraform": { "backend": { "s3": { - "bucket": "acme-terraform-state", + "bucket": "acme-ue1-root-tfstate", "region": "us-east-1", - "dynamodb_table": "terraform-locks", "encrypt": true, + "use_lockfile": true, "key": "ue1/prod/vpc/terraform.tfstate" } } @@ -93,7 +109,7 @@ Backend settings within a component override the defaults: backend_type: s3 backend: s3: - bucket: acme-special-state + bucket: acme-ue1-prod-special-tfstate key: "special/terraform.tfstate" ``` @@ -103,7 +119,7 @@ Backend settings within a component override the defaults: "terraform": { "backend": { "s3": { - "bucket": "acme-special-state", + "bucket": "acme-ue1-prod-special-tfstate", "key": "special/terraform.tfstate" } } @@ -113,414 +129,12 @@ Backend settings within a component override the defaults: -## Backend Types - -Atmos supports all Terraform backend types. Configure using `backend_type` and the corresponding configuration under `backend`. - -### S3 Backend - -The most common backend for AWS environments. We recommend using `use_lockfile: true` for native S3 state locking (Terraform 1.10+) instead of DynamoDB: - - - - ```yaml - terraform: - backend_type: s3 - backend: - s3: - bucket: "{{ .namespace }}-terraform-state" - region: us-east-1 - key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" - encrypt: true - use_lockfile: true # Native S3 locking (Terraform 1.10+) - ``` - - - ```json title="components/terraform/vpc/backend.tf.json" - { - "terraform": { - "backend": { - "s3": { - "bucket": "acme-terraform-state", - "region": "us-east-1", - "key": "ue1/prod/vpc/terraform.tfstate", - "encrypt": true, - "use_lockfile": true - } - } - } - } - ``` - - - -#### S3 Backend with Assume Role - -For cross-account access, configure the backend to assume a role: - - - - ```yaml - terraform: - backend_type: s3 - backend: - s3: - bucket: "{{ .namespace }}-terraform-state" - region: us-east-1 - key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" - encrypt: true - use_lockfile: true - assume_role: - role_arn: "arn:aws:iam::{{ .vars.state_account_id }}:role/TerraformStateAccess" - session_name: "atmos-{{ .component }}" - ``` - - - ```json title="components/terraform/vpc/backend.tf.json" - { - "terraform": { - "backend": { - "s3": { - "bucket": "acme-terraform-state", - "region": "us-east-1", - "key": "ue1/prod/vpc/terraform.tfstate", - "encrypt": true, - "use_lockfile": true, - "assume_role": { - "role_arn": "arn:aws:iam::123456789012:role/TerraformStateAccess", - "session_name": "atmos-vpc" - } - } - } - } - } - ``` - - - -#### S3 Backend with DynamoDB Locking (Legacy) - -For Terraform versions before 1.10, use DynamoDB for state locking: - -```yaml -terraform: - backend_type: s3 - backend: - s3: - bucket: "{{ .namespace }}-terraform-state" - region: us-east-1 - key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" - dynamodb_table: terraform-locks - encrypt: true -``` - -### Azure Blob Backend - -For Azure environments: - - - - ```yaml - terraform: - backend_type: azurerm - backend: - azurerm: - resource_group_name: terraform-state-rg - storage_account_name: tfstateaccount - container_name: tfstate - key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" - ``` - - - ```json title="components/terraform/vpc/backend.tf.json" - { - "terraform": { - "backend": { - "azurerm": { - "resource_group_name": "terraform-state-rg", - "storage_account_name": "tfstateaccount", - "container_name": "tfstate", - "key": "ue1/prod/vpc/terraform.tfstate" - } - } - } - } - ``` - - - -### GCS Backend - -For Google Cloud environments: - - - - ```yaml - terraform: - backend_type: gcs - backend: - gcs: - bucket: "{{ .namespace }}-terraform-state" - prefix: "{{ .environment }}/{{ .stage }}/{{ .component }}" - ``` - - - ```json title="components/terraform/vpc/backend.tf.json" - { - "terraform": { - "backend": { - "gcs": { - "bucket": "acme-terraform-state", - "prefix": "ue1/prod/vpc" - } - } - } - } - ``` - - - -### Terraform Cloud / Enterprise - -For Terraform Cloud or Enterprise: - - - - ```yaml - terraform: - backend_type: remote - backend: - remote: - hostname: app.terraform.io - organization: acme - workspaces: - name: "{{ .namespace }}-{{ .environment }}-{{ .stage }}-{{ .component }}" - ``` - - Or using the newer `cloud` backend: - - ```yaml - terraform: - backend_type: cloud - backend: - cloud: - organization: acme - workspaces: - name: "{{ .namespace }}-{{ .environment }}-{{ .stage }}-{{ .component }}" - ``` - - - ```json title="components/terraform/vpc/backend.tf.json" - { - "terraform": { - "backend": { - "remote": { - "hostname": "app.terraform.io", - "organization": "acme", - "workspaces": { - "name": "acme-ue1-prod-vpc" - } - } - } - } - } - ``` - - - -### Local Backend - -For development or single-user scenarios: - - - - ```yaml - terraform: - backend_type: local - backend: - local: - path: "{{ .component }}/terraform.tfstate" - ``` - - - ```json title="components/terraform/vpc/backend.tf.json" - { - "terraform": { - "backend": { - "local": { - "path": "vpc/terraform.tfstate" - } - } - } - } - ``` - - - -## Remote State Backend - -The `remote_state_backend` configuration is separate from `backend` and controls how other components can read this component's state: - -```yaml -terraform: - # Where this component stores its state - backend_type: s3 - backend: - s3: - bucket: acme-terraform-state - region: us-east-1 - - # How other components read this component's state - remote_state_backend_type: s3 - remote_state_backend: - s3: - bucket: acme-terraform-state - region: us-east-1 -``` - -This separation allows for scenarios like: -- Using Terraform Cloud for operations but S3 for remote state access -- Different credentials for writing vs. reading state -- Custom remote state configurations - -## Examples - -### Environment-Specific State Buckets - - -```yaml -terraform: - backend_type: s3 - backend: - s3: - bucket: acme-dev-terraform-state - region: us-east-1 - dynamodb_table: terraform-locks-dev - encrypt: true -``` - - - -```yaml -terraform: - backend_type: s3 - backend: - s3: - bucket: acme-prod-terraform-state - region: us-east-1 - dynamodb_table: terraform-locks-prod - encrypt: true -``` - - -### Dynamic State Keys with Templates - - -```yaml -terraform: - backend_type: s3 - backend: - s3: - bucket: "{{ .namespace }}-terraform-state" - region: "{{ .vars.region }}" - key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" - dynamodb_table: terraform-locks - encrypt: true -``` - - -### Component-Specific Backend Override - - -```yaml -import: - - orgs/acme/_defaults - -components: - terraform: - # Uses default backend - vpc: - vars: - vpc_cidr: "10.0.0.0/16" - - # Uses custom backend for sensitive data - secrets-manager: - backend_type: s3 - backend: - s3: - bucket: acme-sensitive-state - region: us-east-1 - key: "secrets/terraform.tfstate" - encrypt: true - kms_key_id: alias/terraform-state-key - vars: - # ... -``` - - -### Multi-Cloud Configuration - - -```yaml -terraform: - backend_type: s3 - backend: - s3: - bucket: acme-aws-terraform-state - region: us-east-1 -``` - - - -```yaml -terraform: - backend_type: azurerm - backend: - azurerm: - resource_group_name: terraform-state-rg - storage_account_name: acmetfstate - container_name: tfstate -``` - - -## Workspace Configuration - -For backends that support workspaces, you can configure workspace patterns: - -```yaml -terraform: - backend_type: remote - backend: - remote: - organization: acme - workspaces: - prefix: "acme-" - -components: - terraform: - vpc: - terraform_workspace: "{{ .environment }}-{{ .stage }}-vpc" -``` - -See [Terraform Workspaces](/components/terraform/workspaces) for detailed workspace configuration. - -## Best Practices - -1. **Use Remote State:** Always use a remote backend for team environments to enable collaboration and state locking. - -2. **Enable Encryption:** Always enable encryption for backends that support it (S3, Azure Blob, GCS). - -3. **Configure State Locking:** Use DynamoDB for S3, blob leases for Azure, or built-in locking for other backends. - -4. **Organize State Keys:** Use a consistent key structure that includes environment, stage, and component information. - -5. **Separate Sensitive State:** Consider using separate state buckets or additional encryption for components managing secrets. - -6. **Use Templates:** Leverage Go templates for dynamic backend configuration based on context variables. + ## Related -- [Terraform Backends](/components/terraform/backends) - Detailed backend configuration reference -- [Remote State](/stacks/remote-state) - Accessing state from other components -- [Terraform Providers](/stacks/providers) +- [Terraform Backend Configuration](/stacks/components/terraform/backend) - Component-level backend defaults +- [Terraform Backends](/components/terraform/backends) - Conceptual overview and inheritance +- [Backend Provisioning](/components/terraform/backend-provisioning) - Automatic backend creation +- [Remote State](/stacks/remote-state) - Reading state from other components - [Terraform Workspaces](/components/terraform/workspaces) diff --git a/website/docs/stacks/components/terraform.mdx b/website/docs/stacks/components/terraform.mdx index 9932c76dda..add72f4761 100644 --- a/website/docs/stacks/components/terraform.mdx +++ b/website/docs/stacks/components/terraform.mdx @@ -36,16 +36,16 @@ Terraform components support all common sections plus Terraform-specific options
[`hooks`](/stacks/hooks)
Lifecycle event handlers.
-
[`backend`](/stacks/backend)
+
[`backend`](/stacks/components/terraform/backend)
State storage configuration.
-
[`backend_type`](/stacks/backend)
+
[`backend_type`](/stacks/components/terraform/backend)
Backend type (s3, azurerm, etc.).
-
[`remote_state_backend`](/stacks/backend)
+
[`remote_state_backend`](/stacks/components/terraform/backend)
Remote state access configuration.
-
[`remote_state_backend_type`](/stacks/backend)
+
[`remote_state_backend_type`](/stacks/components/terraform/backend)
Remote state backend type.
[`providers`](/stacks/providers)
@@ -173,7 +173,7 @@ terraform: s3: bucket: "{{ .namespace }}-terraform-state" region: us-east-1 - dynamodb_table: terraform-locks + use_lockfile: true encrypt: true providers: aws: @@ -208,10 +208,10 @@ terraform: backend_type: s3 backend: s3: - bucket: acme-prod-terraform-state + bucket: acme-ue1-prod-tfstate region: us-east-1 key: "{{ .environment }}/{{ .stage }}/{{ .component }}/terraform.tfstate" - dynamodb_table: terraform-locks + use_lockfile: true encrypt: true providers: aws: @@ -330,9 +330,11 @@ components/terraform/ ## Related -- [Configure Backend](/stacks/backend) +- [Terraform Backend Configuration](/stacks/components/terraform/backend) +- [Generate Terraform Backend](/stacks/backend) - [Configure Providers](/stacks/providers) -- [Terraform Backends Reference](/components/terraform/backends) +- [Terraform Backends Overview](/components/terraform/backends) +- [Backend Provisioning](/components/terraform/backend-provisioning) - [Terraform Providers Reference](/components/terraform/providers) - [Terraform Workspaces](/components/terraform/workspaces) - [Remote State](/stacks/remote-state) diff --git a/website/docs/stacks/components/terraform/backend.mdx b/website/docs/stacks/components/terraform/backend.mdx new file mode 100644 index 0000000000..08443e7a91 --- /dev/null +++ b/website/docs/stacks/components/terraform/backend.mdx @@ -0,0 +1,66 @@ +--- +title: Terraform Backend Configuration +sidebar_position: 5 +sidebar_label: backend +slug: /stacks/components/terraform/backend +description: Configure Terraform state backends for components. +id: terraform-backend +--- +import Intro from '@site/src/components/Intro' +import BackendConfig from '../../_partials/_backend-config.mdx' + + +The `backend` section under `terraform:` or `components.terraform.:` configures +where Terraform stores state for your components. Settings defined here become defaults +for all Terraform components in the stack. + + +## Setting Defaults for Terraform Components + +When you define backend configuration under the `terraform:` section, it applies to +**all Terraform components** in that stack manifest: + +```yaml title="stacks/orgs/acme/_defaults.yaml" +terraform: + backend_type: s3 + backend: + s3: + bucket: acme-ue1-root-tfstate + region: us-east-1 + encrypt: true + use_lockfile: true +``` + +Every Terraform component that imports this manifest inherits these backend settings. +Individual components can override specific values as needed. + +## Component-Level Overrides + +Override backend settings for a specific component: + +```yaml title="stacks/orgs/acme/plat/prod/us-east-1.yaml" +components: + terraform: + vpc: + # Inherits org defaults, overrides key prefix + backend: + s3: + workspace_key_prefix: networking/vpc + + secrets-manager: + # Uses a separate, more restricted bucket + backend: + s3: + bucket: acme-ue1-prod-secrets-tfstate + kms_key_id: alias/secrets-state-key +``` + + + +## Related + +- [Generate Terraform Backend](/stacks/backend) - Stack-level backend configuration +- [Terraform Backends](/components/terraform/backends) - Conceptual overview +- [Backend Provisioning](/components/terraform/backend-provisioning) - Automatic backend creation +- [Remote State](/stacks/remote-state) - Reading state from other components +- [Terraform Workspaces](/components/terraform/workspaces) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index a595bb000f..43140f32d5 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -66,6 +66,15 @@ const config = { from: '/reference/terraform-limitations', to: '/intro/why-atmos' }, + // Backend documentation reorganization + { + from: '/core-concepts/components/terraform/state-backend', + to: '/components/terraform/remote-state' + }, + { + from: '/core-concepts/components/terraform/remote-state', + to: '/components/terraform/remote-state' + }, // Component Catalog redirects for reorganization { from: '/design-patterns/component-catalog-with-mixins',