From 665b85e7e9be7606a27002fad93cb0d348cd7091 Mon Sep 17 00:00:00 2001 From: Thomas Gummerer Date: Tue, 9 Apr 2024 18:11:57 +0200 Subject: [PATCH 1/2] use test environments for codegen tests Currently these tests always just use the directory in the repository tree to run. This isn't great for a couple of reasons: - The tests write files into the repo tree (these are .gitignored, but still large and aren't cleaned up after the test finished) - Tests running in parallel can potentially use the same directory. We have a nice helper that creates new test environments and cleans them up at the end of the tests. Use that here for more test isolation. Also remove the t.Run() in the inner loop. For a lot of these tests it's not possible to run only one bit at a time (which is also why there is an ordering enforced). It's better to just make them run in the outer test loop to avoid confusion when trying to rerun only part of the test. --- pkg/codegen/testing/test/sdk_driver.go | 37 +++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/pkg/codegen/testing/test/sdk_driver.go b/pkg/codegen/testing/test/sdk_driver.go index 4939a5a51335..753d72dad2a5 100644 --- a/pkg/codegen/testing/test/sdk_driver.go +++ b/pkg/codegen/testing/test/sdk_driver.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/pulumi/pulumi/pkg/v3/codegen" + ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing" ) // Defines an extra check logic that accepts the directory with the @@ -578,11 +579,21 @@ func TestSDKCodegen(t *testing.T, opts *SDKCodegenOptions) { // revive:disable-l t.Log(tt.Description) + e := ptesting.NewEnvironment(t) + defer e.DeleteIfNotFailed() + + // Some tests need the directory to have the right name. Create a subdirectory + // under the test environment to ensure that + err := os.Mkdir(filepath.Join(e.RootPath, filepath.FromSlash(tt.Directory)), 0o755) + require.NoError(t, err) + e.CWD = filepath.Join(e.RootPath, filepath.FromSlash(tt.Directory)) + dirPath := filepath.Join(testDir, filepath.FromSlash(tt.Directory)) + e.ImportDirectory(dirPath) - schemaPath := filepath.Join(dirPath, "schema.json") + schemaPath := filepath.Join(e.CWD, "schema.json") if _, err := os.Stat(schemaPath); err != nil && os.IsNotExist(err) { - schemaPath = filepath.Join(dirPath, "schema.yaml") + schemaPath = filepath.Join(e.CWD, "schema.yaml") } if tt.ShouldSkipCodegen(opts.Language) { @@ -593,8 +604,8 @@ func TestSDKCodegen(t *testing.T, opts *SDKCodegenOptions) { // revive:disable-l files, err := GeneratePackageFilesFromSchema(schemaPath, opts.GenPackage) require.NoError(t, err) - if !RewriteFilesWhenPulumiAccept(t, dirPath, opts.Language, files) { - expectedFiles, err := LoadBaseline(dirPath, opts.Language) + if !RewriteFilesWhenPulumiAccept(t, e.CWD, opts.Language, files) { + expectedFiles, err := LoadBaseline(e.CWD, opts.Language) require.NoError(t, err) if !ValidateFileEquality(t, files, expectedFiles) { @@ -606,7 +617,7 @@ func TestSDKCodegen(t *testing.T, opts *SDKCodegenOptions) { // revive:disable-l return } - CopyExtraFiles(t, dirPath, opts.Language) + CopyExtraFiles(t, e.CWD, opts.Language) // Merge language-specific global and // test-specific checks, with test-specific @@ -626,19 +637,15 @@ func TestSDKCodegen(t *testing.T, opts *SDKCodegenOptions) { // revive:disable-l } sort.Strings(checkOrder) - codeDir := filepath.Join(dirPath, opts.Language) + codeDir := filepath.Join(e.CWD, opts.Language) // Perform the checks. - //nolint:paralleltest // test functions are ordered for _, check := range checkOrder { - check := check - t.Run(check, func(t *testing.T) { - if tt.ShouldSkipTest(opts.Language, check) { - t.Skip() - } - checkFun := allChecks[check] - checkFun(t, codeDir) - }) + if tt.ShouldSkipTest(opts.Language, check) { + t.Skip() + } + checkFun := allChecks[check] + checkFun(t, codeDir) } }) } From c5c41fb83e39945c706dd6507ce3120825ae530a Mon Sep 17 00:00:00 2001 From: Thomas Gummerer Date: Mon, 15 Apr 2024 13:19:45 +0200 Subject: [PATCH 2/2] export directory if PULUMI_ACCEPT was set --- pkg/codegen/testing/test/sdk_driver.go | 2 ++ sdk/go/common/testing/environment.go | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/pkg/codegen/testing/test/sdk_driver.go b/pkg/codegen/testing/test/sdk_driver.go index 753d72dad2a5..37bf6de1df64 100644 --- a/pkg/codegen/testing/test/sdk_driver.go +++ b/pkg/codegen/testing/test/sdk_driver.go @@ -611,6 +611,8 @@ func TestSDKCodegen(t *testing.T, opts *SDKCodegenOptions) { // revive:disable-l if !ValidateFileEquality(t, files, expectedFiles) { t.Fail() } + } else { + e.ExportDirectory(dirPath) } if genSDKOnly { diff --git a/sdk/go/common/testing/environment.go b/sdk/go/common/testing/environment.go index 75b84dc4b6dd..c8289c784857 100644 --- a/sdk/go/common/testing/environment.go +++ b/sdk/go/common/testing/environment.go @@ -133,6 +133,15 @@ func (e *Environment) ImportDirectory(path string) { } } +// ExportDirectory copies a folder from the test environment to the given path. This is useful for generated +// files via PULUMI_ACCEPT. +func (e *Environment) ExportDirectory(path string) { + err := fsutil.CopyFile(path, e.CWD, nil) + if err != nil { + e.T.Fatalf("error exporting directory: %v", err) + } +} + // DeleteEnvironment deletes the environment's RootPath, and everything underneath it. func (e *Environment) DeleteEnvironment() { e.Helper()