kubetest2-kops: automatically create buckets

When working with boskos, this means we don't need to have a shared
bucket with dynamic permissions.

tests/e2e/go.mod

@@ -8,9 +8,12 @@ replace k8s.io/kops => ../../.
 replace k8s.io/client-go => k8s.io/client-go v0.24.2
 
 require (
+ github.com/aws/aws-sdk-go v1.44.283
  github.com/blang/semver/v4 v4.0.0
  github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+ github.com/google/uuid v1.3.0
  github.com/octago/sflags v0.2.0
+ github.com/spf13/cobra v1.7.0
  github.com/spf13/pflag v1.0.5
  k8s.io/api v0.27.3
  k8s.io/apimachinery v0.27.3
@@ -35,7 +38,6 @@ require (
  github.com/StackExchange/wmi v1.2.1 // indirect
  github.com/acomagu/bufpipe v1.0.3 // indirect
  github.com/apparentlymart/go-cidr v1.1.0 // indirect
- github.com/aws/aws-sdk-go v1.44.283 // indirect
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/blang/semver v3.5.1+incompatible // indirect
  github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -70,7 +72,6 @@ require (
  github.com/google/go-querystring v1.1.0 // indirect
  github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect
  github.com/google/s2a-go v0.1.4 // indirect
- github.com/google/uuid v1.3.0 // indirect
  github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
  github.com/googleapis/gax-go/v2 v2.10.0 // indirect
  github.com/gophercloud/gophercloud v1.4.0 // indirect
@@ -106,7 +107,6 @@ require (
  github.com/sergi/go-diff v1.3.1 // indirect
  github.com/shirou/gopsutil/v3 v3.21.10 // indirect
  github.com/sirupsen/logrus v1.9.0 // indirect
- github.com/spf13/cobra v1.7.0 // indirect
  github.com/ulikunitz/xz v0.5.10 // indirect
  github.com/vbatts/tar-split v0.11.3 // indirect
  github.com/xanzy/ssh-agent v0.3.1 // indirect

tests/e2e/kubetest2-kops/deployer/common.go

@@ -25,7 +25,9 @@ import (
  "path/filepath"
  "strings"
 
+ "github.com/aws/aws-sdk-go/aws/credentials"
  "k8s.io/klog/v2"
+ "k8s.io/kops/tests/e2e/kubetest2-kops/aws"
  "k8s.io/kops/tests/e2e/kubetest2-kops/gce"
  "k8s.io/kops/tests/e2e/pkg/kops"
  "k8s.io/kops/tests/e2e/pkg/target"
@@ -83,10 +85,8 @@ func (d *deployer) initialize(ctx context.Context) error {
  if !ok {
  return fmt.Errorf("secret-access-key not found in boskos resource %q", resource.Name)
  }
- d.awsStaticCredentials = &awsStaticCredentials{
- AccessKeyID: accessKeyIDObj.(string),
- SecretAccessKey: secretAccessKeyObj.(string),
- }
+ d.awsCredentials = credentials.NewStaticCredentials(accessKeyIDObj.(string), secretAccessKeyObj.(string), "")
+ d.createBucket = true
  }
 
  if d.SSHPrivateKeyPath == "" || d.SSHPublicKeyPath == "" {
@@ -213,9 +213,13 @@ func (d *deployer) env() []string {
  // https://github.com/kubernetes/kubernetes/blob/a750d8054a6cb3167f495829ce3e77ab0ccca48e/test/e2e/framework/ssh/ssh.go#L59-L62
  vars = append(vars, fmt.Sprintf("KUBE_SSH_KEY_PATH=%v", d.SSHPrivateKeyPath))
 
- if d.awsStaticCredentials != nil {
- vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", d.awsStaticCredentials.AccessKeyID))
- vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", d.awsStaticCredentials.SecretAccessKey))
+ if d.awsCredentials != nil {
+ credentials, err := d.awsCredentials.Get()
+ if err != nil {
+ klog.Fatalf("error getting aws credentials: %v", err)
+ }
+ vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", credentials.AccessKeyID))
+ vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", credentials.SecretAccessKey))
  }
  } else if d.CloudProvider == "digitalocean" {
  // Pass through some env vars if set
@@ -277,11 +281,18 @@ func defaultClusterName(cloudProvider string) (string, error) {
 // stateStore returns the kops state store to use
 // defaulting to values used in prow jobs
 func (d *deployer) stateStore() string {
+ ctx := context.TODO()
+
  ss := os.Getenv("KOPS_STATE_STORE")
  if ss == "" {
  switch d.CloudProvider {
  case "aws":
- ss = "s3://k8s-kops-prow"
+ bucketName, err := aws.AWSBucketName(ctx, d.awsCredentials)
+ if err != nil {
+ klog.Fatalf("error building aws bucket name: %v", err)
+ }
+ ss = "s3://" + bucketName
+ d.createBucket = true
  case "gce":
  d.createBucket = true
  ss = "gs://" + gce.GCSBucketName(d.GCPProject)

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -22,6 +22,7 @@ import (
  "sync"
  "time"
 
+ "github.com/aws/aws-sdk-go/aws/credentials"
  "github.com/octago/sflags/gen/gpflag"
  "github.com/spf13/pflag"
  "k8s.io/klog/v2"
@@ -85,13 +86,8 @@ type deployer struct {
 
  boskos boskosHelper
 
- // awsStaticCredentials holds credentials for AWS loaded from boskos
- awsStaticCredentials *awsStaticCredentials
-}
-
-type awsStaticCredentials struct {
- AccessKeyID string
- SecretAccessKey string
+ // awsCredentials holds credentials for AWS loaded from boskos
+ awsCredentials *credentials.Credentials
 }
 
 // assert that New implements types.NewDeployer

tests/e2e/kubetest2-kops/deployer/down.go

@@ -18,9 +18,11 @@ package deployer
 
 import (
  "context"
+ "fmt"
  "strings"
 
  "k8s.io/klog/v2"
+ "k8s.io/kops/tests/e2e/kubetest2-kops/aws"
  "k8s.io/kops/tests/e2e/kubetest2-kops/gce"
  "sigs.k8s.io/kubetest2/pkg/exec"
 )
@@ -55,8 +57,22 @@ func (d *deployer) Down() error {
  return err
  }
 
- if d.CloudProvider == "gce" && d.createBucket {
- gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
+ if d.createBucket {
+ switch d.CloudProvider {
+ case "gce":
+ gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
+ case "aws":
+ bucketName, err := aws.AWSBucketName(ctx, d.awsCredentials)
+ if err != nil {
+ return fmt.Errorf("error building aws bucket name: %w", err)
+ }
+
+ if err := aws.DeleteAWSBucket(ctx, d.awsCredentials, bucketName); err != nil {
+ klog.Warningf("error deleting AWS bucket: %w", err)
+ }
+ default:
+ return fmt.Errorf("bucket cleanup not implemented for cloud %q", d.CloudProvider)
+ }
  }
 
  if err := d.boskos.Cleanup(ctx); err != nil {

tests/e2e/kubetest2-kops/deployer/up.go

@@ -17,6 +17,7 @@ limitations under the License.
 package deployer
 
 import (
+ "context"
  "errors"
  "fmt"
  osexec "os/exec"
@@ -36,6 +37,8 @@ import (
 )
 
 func (d *deployer) Up() error {
+ ctx := context.TODO()
+
  if err := d.init(); err != nil {
  return err
  }
@@ -47,9 +50,18 @@ func (d *deployer) Up() error {
  _ = d.Down()
  }
 
- if d.CloudProvider == "gce" && d.createBucket {
- if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
- return err
+ if d.createBucket {
+ switch d.CloudProvider {
+ case "gce":
+ if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
+ return err
+ }
+ case "aws":
+ if err := aws.EnsureAWSBucket(ctx, d.awsCredentials, d.stateStore()); err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf("bucket creation not implemented for cloud %q", d.CloudProvider)
  }
  }