AWS SDK for Go v2: cannot list/get objects

[kokes]

Expected Behavior

I wanted to try the new SDK that went GA quite recently. I wanted to list and get objects in my Minio server.

(As a side note - it might be handy to include the v2 SDK example code in minio's docs.)

Current Behavior

All I got was api error NoSuchBucket: The specified bucket does not exist. I would understand an issue with auth or anything else, but here the SDK is telling me the bucket doesn't exist, even if I plug in the bucket name from ListBuckets (the only method I found to work so far).

Possible Solution

Downgrading to AWS SDK v1 or Minio's client library works. Not really a solution.

Looking at the trace, it seems like MinIO is looking for a bucket named like the object. You can see the in the XML at the very end of the trace that it didn't find bucket data.txt. That is something to go on.

Steps to Reproduce (for bugs)

1. Installed the following

• github.com/aws/aws-sdk-go v1.38.16
• github.com/aws/aws-sdk-go-v2 v1.3.1
• github.com/minio/minio-go/v7 v7.0.10
• minio version RELEASE.2021-04-06T23-11-00Z

2. Set up the server and uploaded data

$ minio server tmp
$ ~/go/bin/mc alias set myminio/ http://localhost:9000 minioadmin minioadmin
$ ~/go/bin/mc admin trace --verbose myminio
$ aws s3 --endpoint-url http://localhost:9000 mb s3://storage
# checks the minio ui
$ echo "hi" > data.txt
$ aws s3 --endpoint-url http://localhost:9000 cp data.txt s3://storage/
# verify it's there
$ aws s3 --endpoint-url http://localhost:9000 cp s3://storage/data.txt -
hi

3. Try to retrieve that data with MinIO's Go library

(Code largely taken from minio docs)

$ go run minio.go
hi

package main

import (
	"context"
	"io"
	"log"
	"os"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
)

func main() {
	if err := run(); err != nil {
		log.Fatal(err)
	}
}

func run() error {
	endpoint := "localhost:9000"
	accessKeyID := "minioadmin"
	secretAccessKey := "minioadmin"
	useSSL := false

	minioClient, err := minio.New(endpoint, &minio.Options{
		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
		Secure: useSSL,
	})
	if err != nil {
		return err
	}

	opts := minio.GetObjectOptions{}
	ret, err := minioClient.GetObject(context.Background(), "storage", "data.txt", opts)
	if err != nil {
		return err
	}

	if _, err := io.Copy(os.Stdout, ret); err != nil {
		return err
	}

	return nil
}

4. Try to retrieve that data with AWS SDK for Go v1

(Code largely taken from minio docs)

$ go run minio-aws.go
hi

package main

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

func main() {
	bucket := aws.String("storage")
	key := aws.String("data.txt")

	// Configure to use MinIO Server
	s3Config := &aws.Config{
		Credentials:      credentials.NewStaticCredentials("minioadmin", "minioadmin", ""),
		Endpoint:         aws.String("http://localhost:9000"),
		Region:           aws.String("us-east-1"),
		DisableSSL:       aws.Bool(true),
		S3ForcePathStyle: aws.Bool(true),
	}
	newSession := session.New(s3Config)

	s3Client := s3.New(newSession)

	// the following works as well
	// listing, err := s3Client.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: bucket})
	// if err != nil {
	// 	log.Fatal(err)
	// }
	// fmt.Printf("I see these objects: %v\n", listing.Contents)

	resp, err := s3Client.GetObject(&s3.GetObjectInput{
		Bucket: bucket,
		Key:    key,
	})
	if err != nil {
		fmt.Printf("Failed to download data from %s/%s, %s\n", *bucket, *key, err.Error())
		return
	}
	defer resp.Body.Close()
	if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
		log.Fatal(err)
	}
}

trace

localhost [REQUEST s3.GetObject] [08:01:53.256] [Client IP: ::1]
localhost GET /storage/data.txt
localhost Proto: HTTP/1.1
localhost Host: localhost:9000
localhost Accept-Encoding: gzip
localhost Authorization: AWS4-HMAC-SHA256 Credential=minioadmin/20210409/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=11da3bb52f95507fba4a507460260bbb253fa05c362c6f71914e7678cf9b3986
localhost Content-Length: 0
localhost User-Agent: aws-sdk-go/1.38.16 (go1.16; darwin; amd64)
localhost X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
localhost X-Amz-Date: 20210409T080153Z
localhost <BODY>
localhost [RESPONSE] [08:01:53.257] [ Duration 1.113ms  ↑ 93 B  ↓ 303 B ]
localhost 200 OK
localhost Vary: Origin
localhost X-Amz-Request-Id: 167421C8FD141FC8
localhost Accept-Ranges: bytes
localhost ETag: "764efa883dda1e11db47671c4a3bbd9e"
localhost Content-Type: text/plain
localhost Last-Modified: Fri, 09 Apr 2021 06:50:33 GMT
localhost Server: MinIO
localhost X-Xss-Protection: 1; mode=block
localhost Content-Length: 3
localhost Content-Security-Policy: block-all-mixed-content
localhost <BODY>
localhost 

5. Try to download the data using AWS SDK for Go v2

$ go run minio-aws2.go
got this region: us-east-2
I see bucket "storage"
Objects in that bucket: []
2021/04/09 10:16:41 operation error S3: GetObject, https response error StatusCode: 404, RequestID: 16742297C6E938F0, HostID: 3af60b7b-edf7-4fa9-b8c4-6875fb49302a, api error NoSuchBucket: The specified bucket does not exist
exit status 1

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/credentials"
	"github.com/aws/aws-sdk-go-v2/service/s3"
)

func main() {
	if err := run(); err != nil {
		log.Fatal(err)
	}
}

func run() error {
	resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
		return aws.Endpoint{
			URL: "http://localhost:9000",
		}, nil
	})

	creds := credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "")
	cfg, err := config.LoadDefaultConfig(context.TODO(),
		config.WithCredentialsProvider(creds),
		config.WithEndpointResolver(resolver),
	)
	if err != nil {
		return err
	}
	fmt.Printf("got this region: %v\n", cfg.Region)
	svc := s3.NewFromConfig(cfg)
	ctx := context.Background()

	buckets, err := svc.ListBuckets(ctx, nil)
	if err != nil {
		return err
	}

	for _, bucket := range buckets.Buckets {
		fmt.Printf("I see bucket \"%s\"\n", *bucket.Name)
	}

	objs, err := svc.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String("storage")})
	if err != nil {
		return err
	}
	fmt.Printf("Objects in that bucket: %+v\n", objs.Contents)

	input := &s3.GetObjectInput{
		Bucket: aws.String("storage"),
		// Bucket: buckets.Buckets[0].Name, // this doesn't work either
		Key: aws.String("data.txt"),
	}
	output, err := svc.GetObject(ctx, input)
	if err != nil {
		return err
	}
	defer output.Body.Close()
	if _, err := io.Copy(os.Stdout, output.Body); err != nil {
		return err
	}

	return nil
}

trace

localhost [REQUEST s3.ListBuckets] [08:23:54.245] [Client IP: ::1]
localhost GET /
localhost Proto: HTTP/1.1
localhost Host: localhost:9000
localhost Accept-Encoding: identity
localhost Amz-Sdk-Request: attempt=1; max=3
localhost User-Agent: aws-sdk-go-v2/1.3.1
localhost X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
localhost X-Amz-Date: 20210409T082354Z
localhost X-Amz-User-Agent: aws-sdk-go-v2/1.3.1 os/other lang/go/1.16 md/GOOS/darwin md/GOARCH/amd64
localhost Amz-Sdk-Invocation-Id: 078f5d87-6c2c-4863-9051-d24ff83ed426
localhost Authorization: AWS4-HMAC-SHA256 Credential=minioadmin/20210409//s3/aws4_request, SignedHeaders=accept-encoding;amz-sdk-invocation-id;amz-sdk-request;host;x-amz-content-sha256;x-amz-date;x-amz-user-agent, Signature=21b12c4a93967e4efc54b0c8a1d01878b57a4d21e3f446f2b3d7ca36709f378b
localhost Content-Length: 0
localhost 
localhost [RESPONSE] [08:23:54.245] [ Duration 436µs  ↑ 148 B  ↓ 586 B ]
localhost 200 OK
localhost Server: MinIO
localhost Vary: Origin
localhost X-Amz-Request-Id: 167422FC8E2BCD88
localhost X-Xss-Protection: 1; mode=block
localhost Accept-Ranges: bytes
localhost Content-Length: 365
localhost Content-Security-Policy: block-all-mixed-content
localhost Content-Type: application/xml
localhost <?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4</ID><DisplayName>minio</DisplayName></Owner><Buckets><Bucket><Name>storage</Name><CreationDate>2021-04-09T06:49:53.661Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>
localhost 
storage.localhost [REQUEST s3.ListBuckets] [08:23:54.247] [Client IP: ::1]
storage.localhost GET /?list-type=2
storage.localhost Proto: HTTP/1.1
storage.localhost Host: storage.localhost:9000
storage.localhost Accept-Encoding: identity
storage.localhost Amz-Sdk-Invocation-Id: 95804937-d419-4ac1-83eb-d8788779b9cc
storage.localhost User-Agent: aws-sdk-go-v2/1.3.1
storage.localhost X-Amz-Date: 20210409T082354Z
storage.localhost X-Amz-User-Agent: aws-sdk-go-v2/1.3.1 os/other lang/go/1.16 md/GOOS/darwin md/GOARCH/amd64
storage.localhost Amz-Sdk-Request: attempt=1; max=3
storage.localhost Authorization: AWS4-HMAC-SHA256 Credential=minioadmin/20210409//s3/aws4_request, SignedHeaders=accept-encoding;amz-sdk-invocation-id;amz-sdk-request;host;x-amz-content-sha256;x-amz-date;x-amz-user-agent, Signature=a8298629810990a672ad275946f5f35e18e7221fad139f93fd2acd55374df16a
storage.localhost Content-Length: 0
storage.localhost X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
storage.localhost 
storage.localhost [RESPONSE] [08:23:54.247] [ Duration 342µs  ↑ 148 B  ↓ 586 B ]
storage.localhost 200 OK
storage.localhost Accept-Ranges: bytes
storage.localhost Content-Length: 365
storage.localhost Content-Security-Policy: block-all-mixed-content
storage.localhost Content-Type: application/xml
storage.localhost Server: MinIO
storage.localhost Vary: Origin
storage.localhost X-Amz-Request-Id: 167422FC8E4D0188
storage.localhost X-Xss-Protection: 1; mode=block
storage.localhost <?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4</ID><DisplayName>minio</DisplayName></Owner><Buckets><Bucket><Name>storage</Name><CreationDate>2021-04-09T06:49:53.661Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>
storage.localhost 
storage.localhost [REQUEST s3.ListObjectsV1] [08:23:54.248] [Client IP: ::1]
storage.localhost GET /data.txt?x-id=GetObject
storage.localhost Proto: HTTP/1.1
storage.localhost Host: storage.localhost:9000
storage.localhost X-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
storage.localhost X-Amz-Date: 20210409T082354Z
storage.localhost Amz-Sdk-Request: attempt=1; max=3
storage.localhost Content-Length: 0
storage.localhost User-Agent: aws-sdk-go-v2/1.3.1
storage.localhost X-Amz-User-Agent: aws-sdk-go-v2/1.3.1 os/other lang/go/1.16 md/GOOS/darwin md/GOARCH/amd64
storage.localhost Accept-Encoding: identity
storage.localhost Amz-Sdk-Invocation-Id: 7016a36c-7833-4595-bb48-1cdf0f9cdf60
storage.localhost Authorization: AWS4-HMAC-SHA256 Credential=minioadmin/20210409//s3/aws4_request, SignedHeaders=accept-encoding;amz-sdk-invocation-id;amz-sdk-request;host;x-amz-content-sha256;x-amz-date;x-amz-user-agent, Signature=f7f104145836992f0749319170090d70b3b1a5b0dcc87ffcb60efbee97eb5a84
storage.localhost 
storage.localhost [RESPONSE] [08:23:54.248] [ Duration 196µs  ↑ 148 B  ↓ 516 B ]
storage.localhost 404 Not Found
storage.localhost Server: MinIO
storage.localhost Vary: Origin
storage.localhost X-Amz-Request-Id: 167422FC8E581190
storage.localhost X-Xss-Protection: 1; mode=block
storage.localhost Accept-Ranges: bytes
storage.localhost Content-Length: 288
storage.localhost Content-Security-Policy: block-all-mixed-content
storage.localhost Content-Type: application/xml
storage.localhost <?xml version="1.0" encoding="UTF-8"?>
<Error><Code>NoSuchBucket</Code><Message>The specified bucket does not exist</Message><BucketName>data.txt</BucketName><Resource>/data.txt</Resource><RequestId>167422FC8E581190</RequestId><HostId>3af60b7b-edf7-4fa9-b8c4-6875fb49302a</HostId></Error>
storage.localhost 

Context

I wanted to integrate MinIO for local integration tests to mock my S3 buckets - I used to grab the v1 SDK to do this, but went with v2 this time around and hit this issue.

Regression

Not having worked with MinIO for long, I can't tell at this point.

Your Environment

$ minio --version
minio version RELEASE.2021-04-06T23-11-00Z
$ uname -a
Darwin prg-mp84w 19.6.0 Darwin Kernel Version 19.6.0: Tue Jan 12 22:13:05 PST 2021; root:xnu-6153.141.16~1/RELEASE_X86_64 x86_64

[klauspost]

Likely Host: storage.localhost:9000 is the problem. In the old SDK, this is handled by S3ForcePathStyle: aws.Bool(true), which changes to use the localhost:9000/storage/object.txt style.

Check if there is a way to do that, or see domain section here on how to configure virtual-host-style requests.

[kokes]

@klauspost you are correct.

The config is quite clunky, it's now set up like this:

	svc := s3.NewFromConfig(cfg, func(o *s3.Options) {
		o.UsePathStyle = true
	})

Fixing that, I get the output as expected:

$ go run minio-aws2.go 
got this region: us-east-2
I see bucket "storage"
Objects in that bucket: [{ETag:0xc0000200d0 Key:0xc0000200c0 LastModified:2021-04-09 06:50:33.843 +0000 UTC Owner:0xc0000200e0 Size:3 StorageClass:STANDARD}]
hi

Full code for reference:

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/credentials"
	"github.com/aws/aws-sdk-go-v2/service/s3"
)

func main() {
	if err := run(); err != nil {
		log.Fatal(err)
	}
}

func run() error {
	resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
		return aws.Endpoint{
			URL: "http://localhost:9000",
		}, nil
	})

	creds := credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "")
	cfg, err := config.LoadDefaultConfig(context.TODO(),
		config.WithCredentialsProvider(creds),
		config.WithEndpointResolver(resolver),
	)
	if err != nil {
		return err
	}
	fmt.Printf("got this region: %v\n", cfg.Region)
	svc := s3.NewFromConfig(cfg, func(o *s3.Options) {
		o.UsePathStyle = true
	})
	ctx := context.Background()

	buckets, err := svc.ListBuckets(ctx, nil)
	if err != nil {
		return err
	}

	for _, bucket := range buckets.Buckets {
		fmt.Printf("I see bucket \"%s\"\n", *bucket.Name)
	}

	objs, err := svc.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String("storage")})
	if err != nil {
		return err
	}
	fmt.Printf("Objects in that bucket: %+v\n", objs.Contents)

	input := &s3.GetObjectInput{
		Bucket: aws.String("storage"),
		// Bucket: buckets.Buckets[0].Name, // this doesn't work either
		Key: aws.String("data.txt"),
	}
	output, err := svc.GetObject(ctx, input)
	if err != nil {
		return err
	}
	defer output.Body.Close()
	if _, err := io.Copy(os.Stdout, output.Body); err != nil {
		return err
	}

	return nil
}

---

This could form a basis for a documentation update as I expect more and more people will start using the new SDK and guidance online is quite sparse. For the time being, I guess people can at least find this issue and the resolution here.

Thanks!