Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix folder archives in the engine #16119

Merged
merged 1 commit into from
May 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
changes:
- type: fix
scope: engine
description: Fix folder archive outside of cwd.
23 changes: 22 additions & 1 deletion cmd/pulumi-test-language/l2resourceasset_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ func (h *L2ResourceAssetArchiveLanguageHost) Run(
Path: "../archive.tar",
}, plugin.MarshalOptions{})
if err != nil {
return nil, fmt.Errorf("could not marshal asset: %w", err)
return nil, fmt.Errorf("could not marshal archive: %w", err)
}

_, err = monitor.RegisterResource(ctx, &pulumirpc.RegisterResourceRequest{
Expand All @@ -255,6 +255,27 @@ func (h *L2ResourceAssetArchiveLanguageHost) Run(
return nil, fmt.Errorf("could not register resource: %w", err)
}

folder, err := plugin.MarshalArchive(&resource.Archive{
Path: "../folder",
}, plugin.MarshalOptions{})
if err != nil {
return nil, fmt.Errorf("could not marshal folder: %w", err)
}

_, err = monitor.RegisterResource(ctx, &pulumirpc.RegisterResourceRequest{
Type: "asset-archive:index:ArchiveResource",
Custom: true,
Name: "dir",
Object: &structpb.Struct{
Fields: map[string]*structpb.Value{
"value": folder,
},
},
})
if err != nil {
return nil, fmt.Errorf("could not register resource: %w", err)
}

return &pulumirpc.RunResponse{}, nil
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
data in a folder
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,8 @@

resource "arc" "asset-archive:index:ArchiveResource" {
value = fileArchive("../archive.tar")
}

resource "dir" "asset-archive:index:ArchiveResource" {
value = fileArchive("../folder")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
data in a folder
36 changes: 26 additions & 10 deletions cmd/pulumi-test-language/tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,25 +307,30 @@ var languageTests = map[string]languageTest{
) {
requireStackResource(l, res, changes)

// Check we have the the asset and archive resources in the snapshot, the provider and the stack.
require.Len(l, snap.Resources, 4, "expected 4 resources in snapshot")
// Check we have the the asset, archive, and folder resources in the snapshot, the provider and the stack.
require.Len(l, snap.Resources, 5, "expected 5 resources in snapshot")

provider := snap.Resources[1]
assert.Equal(l, "pulumi:providers:asset-archive", provider.Type.String(), "expected asset-archive provider")

// We don't know what order the asset and archive resources will be in, so check both.
var asset, archive *resource.State
if snap.Resources[2].Type == "asset-archive:index:AssetResource" {
asset = snap.Resources[2]
archive = snap.Resources[3]
} else {
asset = snap.Resources[3]
archive = snap.Resources[2]
// We don't know what order the resources will be in so we map by name
resources := map[string]*resource.State{}
for _, r := range snap.Resources[2:] {
resources[r.URN.Name()] = r
}

asset, ok := resources["ass"]
require.True(l, ok, "expected asset resource")
assert.Equal(l, "asset-archive:index:AssetResource", asset.Type.String(), "expected asset resource")

archive, ok := resources["arc"]
require.True(l, ok, "expected archive resource")
assert.Equal(l, "asset-archive:index:ArchiveResource", archive.Type.String(), "expected archive resource")

folder, ok := resources["dir"]
require.True(l, ok, "expected folder resource")
assert.Equal(l, "asset-archive:index:ArchiveResource", folder.Type.String(), "expected archive resource")

main := filepath.Join(projectDirectory, "subdir")

assetValue, err := resource.NewPathAssetWithWD("../test.txt", main)
Expand All @@ -349,6 +354,17 @@ var languageTests = map[string]languageTest{

assert.Equal(l, want, archive.Inputs, "expected inputs to be {value: %v}", archiveValue)
assert.Equal(l, archive.Inputs, archive.Outputs, "expected inputs and outputs to match")

folderValue, err := resource.NewPathArchiveWithWD("../folder", main)
require.NoError(l, err)
assert.Equal(l, "25df47ed6b3c8e07479e5d9c908eff93d624ec693b6aa7559a9bcb084db70774", folderValue.Hash)

want = resource.NewPropertyMapFromMap(map[string]any{
"value": folderValue,
})

assert.Equal(l, want, folder.Inputs, "expected inputs to be {value: %v}", folderValue)
assert.Equal(l, folder.Inputs, folder.Outputs, "expected inputs and outputs to match")
},
},
},
Expand Down
49 changes: 36 additions & 13 deletions sdk/go/common/resource/archive/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,11 +321,20 @@ type Reader interface {

// Open returns an ArchiveReader that can be used to iterate over the named blobs that comprise the archive.
func (a *Archive) Open() (Reader, error) {
wd, err := os.Getwd()
if err != nil {
return nil, err
}
return a.OpenWithWD(wd)
}

// Open returns an ArchiveReader that can be used to iterate over the named blobs that comprise the archive.
func (a *Archive) OpenWithWD(wd string) (Reader, error) {
contract.Assertf(a.HasContents(), "cannot read an archive that has no contents")
if a.IsAssets() {
return a.readAssets()
} else if a.IsPath() {
return a.readPath()
return a.readPath(wd)
} else if a.IsURI() {
return a.readURI()
}
Expand Down Expand Up @@ -452,11 +461,15 @@ func (r *directoryArchiveReader) Close() error {
return nil
}

func (a *Archive) readPath() (Reader, error) {
func (a *Archive) readPath(wd string) (Reader, error) {
// To read a path-based archive, read that file and use its extension to ascertain what format to use.
path, ispath := a.GetPath()
contract.Assertf(ispath, "Expected a path-based asset")

if !filepath.IsAbs(path) {
path = filepath.Join(wd, path)
}

format := detectArchiveFormat(path)

if format == NotArchive {
Expand Down Expand Up @@ -580,8 +593,18 @@ func (a *Archive) Bytes(format Format) ([]byte, error) {
// Archive produces a single archive stream in the desired format. It prefers to return the archive with as little
// copying as is feasible, however if the desired format is different from the source, it will need to translate.
func (a *Archive) Archive(format Format, w io.Writer) error {
wd, err := os.Getwd()
if err != nil {
return err
}
return a.ArchiveWithWD(format, w, wd)
}

// Archive produces a single archive stream in the desired format. It prefers to return the archive with as little
// copying as is feasible, however if the desired format is different from the source, it will need to translate.
func (a *Archive) ArchiveWithWD(format Format, w io.Writer, wd string) error {
// If the source format is the same, just return that.
if sf, ss, err := a.ReadSourceArchive(); sf != NotArchive && sf == format {
if sf, ss, err := a.ReadSourceArchiveWithWD(wd); sf != NotArchive && sf == format {
if err != nil {
return err
}
Expand All @@ -591,11 +614,11 @@ func (a *Archive) Archive(format Format, w io.Writer) error {

switch format {
case TarArchive:
return a.archiveTar(w)
return a.archiveTar(w, wd)
case TarGZIPArchive:
return a.archiveTarGZIP(w)
return a.archiveTarGZIP(w, wd)
case ZIPArchive:
return a.archiveZIP(w)
return a.archiveZIP(w, wd)
default:
contract.Failf("Illegal archive type: %v", format)
return nil
Expand Down Expand Up @@ -639,9 +662,9 @@ func addNextFileToTar(r Reader, tw *tar.Writer, seenFiles map[string]bool) error
return err
}

func (a *Archive) archiveTar(w io.Writer) error {
func (a *Archive) archiveTar(w io.Writer, wd string) error {
// Open the archive.
reader, err := a.Open()
reader, err := a.OpenWithWD(wd)
if err != nil {
return err
}
Expand All @@ -660,9 +683,9 @@ func (a *Archive) archiveTar(w io.Writer) error {
return tw.Close()
}

func (a *Archive) archiveTarGZIP(w io.Writer) error {
func (a *Archive) archiveTarGZIP(w io.Writer, wd string) error {
z := gzip.NewWriter(w)
return a.archiveTar(z)
return a.archiveTar(z, wd)
}

// addNextFileToZIP adds the next file in the given archive to the given ZIP file. Returns io.EOF if the archive
Expand Down Expand Up @@ -708,9 +731,9 @@ func addNextFileToZIP(r Reader, zw *zip.Writer, seenFiles map[string]bool) error
return nil
}

func (a *Archive) archiveZIP(w io.Writer) error {
func (a *Archive) archiveZIP(w io.Writer, wd string) error {
// Open the archive.
reader, err := a.Open()
reader, err := a.OpenWithWD(wd)
if err != nil {
return err
}
Expand Down Expand Up @@ -788,7 +811,7 @@ func (a *Archive) EnsureHashWithWD(wd string) error {
} else {
// Otherwise, it's not an archive; we'll need to transform it into one. Pick tar since it avoids
// any superfluous compression which doesn't actually help us in this situation.
err := a.Archive(TarArchive, hash)
err := a.ArchiveWithWD(TarArchive, hash, wd)
if err != nil {
return err
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
data in a folder
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ import * as asset_archive from "@pulumi/asset-archive";

const ass = new asset_archive.AssetResource("ass", {value: new pulumi.asset.FileAsset("../test.txt")});
const arc = new asset_archive.ArchiveResource("arc", {value: new pulumi.asset.FileArchive("../archive.tar")});
const dir = new asset_archive.ArchiveResource("dir", {value: new pulumi.asset.FileArchive("../folder")});
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
data in a folder
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ import * as asset_archive from "@pulumi/asset-archive";

const ass = new asset_archive.AssetResource("ass", {value: new pulumi.asset.FileAsset("../test.txt")});
const arc = new asset_archive.ArchiveResource("arc", {value: new pulumi.asset.FileArchive("../archive.tar")});
const dir = new asset_archive.ArchiveResource("dir", {value: new pulumi.asset.FileArchive("../folder")});
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
data in a folder
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@

ass = asset_archive.AssetResource("ass", value=pulumi.FileAsset("../test.txt"))
arc = asset_archive.ArchiveResource("arc", value=pulumi.FileArchive("../archive.tar"))
dir = asset_archive.ArchiveResource("dir", value=pulumi.FileArchive("../folder"))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
data in a folder
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@

ass = asset_archive.AssetResource("ass", value=pulumi.FileAsset("../test.txt"))
arc = asset_archive.ArchiveResource("arc", value=pulumi.FileArchive("../archive.tar"))
dir = asset_archive.ArchiveResource("dir", value=pulumi.FileArchive("../folder"))