kubetest2-kops: add support for boskos-resource-type flag

This will cause AWS tests to acquire an aws-account from boskos (when
set to aws-account), and run the tests with those credentials.

tests/e2e/kubetest2-kops/deployer/boskos.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployer
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "k8s.io/klog/v2"
+ "sigs.k8s.io/boskos/client"
+ "sigs.k8s.io/boskos/common"
+ "sigs.k8s.io/kubetest2/pkg/boskos"
+)
+
+type boskosHelper struct {
+ // boskos holds the client we use for boskos communication.
+ boskos *client.Client
+
+ // this channel serves as a signal channel for the boskos heartbeat goroutine
+ // so that it can be explicitly closed
+ boskosHeartbeatClose chan struct{}
+
+ // resources tracks acquired resources so they can be freed in Cleanup.
+ resources []*common.Resource
+}
+
+func (h *boskosHelper) Acquire(ctx context.Context, resourceType string) (*common.Resource, error) {
+ if h.boskos == nil {
+ h.boskosHeartbeatClose = make(chan struct{})
+
+ boskosURL := os.Getenv("BOSKOS_HOST")
+ if boskosURL == "" {
+ boskosURL = "http://boskos.test-pods.svc.cluster.local."
+ }
+ boskosClient, err := boskos.NewClient(boskosURL)
+ if err != nil {
+ return nil, fmt.Errorf("failed to make boskos client for %q: %w", boskosURL, err)
+ }
+ h.boskos = boskosClient
+ }
+
+ resource, err := boskos.Acquire(
+ h.boskos,
+ resourceType,
+ 5*time.Minute,
+ 5*time.Minute,
+ h.boskosHeartbeatClose,
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to get %q resource from boskos: %w", resourceType, err)
+ }
+ h.resources = append(h.resources, resource)
+
+ return resource, nil
+}
+
+// Cleanup releases any resources acquired from boskos
+func (h *boskosHelper) Cleanup(ctx context.Context) error {
+ if h.boskos != nil {
+ var resourceNames []string
+ for _, resource := range h.resources {
+ klog.V(2).Info("releasing boskos resource %v %q", resource.Type, resource.Name)
+ resourceNames = append(resourceNames, resource.Name)
+ }
+ err := boskos.Release(
+ h.boskos,
+ resourceNames,
+ h.boskosHeartbeatClose,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to release boskos resources %v: %w", resourceNames, err)
+ }
+ }
+
+ return nil
+}

tests/e2e/kubetest2-kops/deployer/common.go

@@ -17,30 +17,29 @@ limitations under the License.
 package deployer
 
 import (
+ "context"
  "errors"
  "fmt"
  "os"
  "path"
  "path/filepath"
  "strings"
- "time"
 
  "k8s.io/klog/v2"
  "k8s.io/kops/tests/e2e/kubetest2-kops/gce"
  "k8s.io/kops/tests/e2e/pkg/kops"
  "k8s.io/kops/tests/e2e/pkg/target"
  "k8s.io/kops/tests/e2e/pkg/util"
- "sigs.k8s.io/kubetest2/pkg/boskos"
 )
 
 func (d *deployer) init() error {
  var err error
- d.doInit.Do(func() { err = d.initialize() })
+ d.doInit.Do(func() { err = d.initialize(context.TODO()) })
  return err
 }
 
 // initialize should only be called by init(), behind a sync.Once
-func (d *deployer) initialize() error {
+func (d *deployer) initialize(ctx context.Context) error {
  if d.commonOptions.ShouldBuild() {
  if err := d.verifyBuildFlags(); err != nil {
  return fmt.Errorf("init failed to check build flags: %v", err)
@@ -67,6 +66,29 @@ func (d *deployer) initialize() error {
 
  switch d.CloudProvider {
  case "aws":
+ if d.BoskosResourceType != "" {
+ klog.V(1).Info("acquiring AWS credentials from Boskos")
+
+ resource, err := d.boskos.Acquire(ctx, d.BoskosResourceType)
+ if err != nil {
+ return fmt.Errorf("init failed to get resource %q from boskos: %w", d.BoskosResourceType, err)
+ }
+ klog.V(1).Infof("Got AWS account %s from boskos", resource.Name)
+
+ accessKeyIDObj, ok := resource.UserData.Load("access-key-id")
+ if !ok {
+ return fmt.Errorf("access-key-id not found in boskos resource %q", resource.Name)
+ }
+ secretAccessKeyObj, ok := resource.UserData.Load("secret-access-key")
+ if !ok {
+ return fmt.Errorf("secret-access-key not found in boskos resource %q", resource.Name)
+ }
+ d.awsStaticCredentials = &awsStaticCredentials{
+ AccessKeyID: accessKeyIDObj.(string),
+ SecretAccessKey: secretAccessKeyObj.(string),
+ }
+ }
+
  if d.SSHPrivateKeyPath == "" || d.SSHPublicKeyPath == "" {
  publicKeyPath, privateKeyPath, err := util.CreateSSHKeyPair(d.ClusterName)
  if err != nil {
@@ -87,21 +109,10 @@ func (d *deployer) initialize() error {
  if d.GCPProject == "" {
  klog.V(1).Info("No GCP project provided, acquiring from Boskos")
 
- boskosClient, err := boskos.NewClient("http://boskos.test-pods.svc.cluster.local.")
+ resourceType := "gce-project"
+ resource, err := d.boskos.Acquire(ctx, resourceType)
  if err != nil {
- return fmt.Errorf("failed to make boskos client: %s", err)
- }
- d.boskos = boskosClient
-
- resource, err := boskos.Acquire(
- d.boskos,
- "gce-project",
- 5*time.Minute,
- 5*time.Minute,
- d.boskosHeartbeatClose,
- )
- if err != nil {
- return fmt.Errorf("init failed to get project from boskos: %s", err)
+ return fmt.Errorf("init failed to get %q resource from boskos: %w", resourceType, err)
  }
  d.GCPProject = resource.Name
  klog.V(1).Infof("Got project %s from boskos", d.GCPProject)
@@ -201,6 +212,11 @@ func (d *deployer) env() []string {
  // Recognized by the e2e framework
  // https://github.com/kubernetes/kubernetes/blob/a750d8054a6cb3167f495829ce3e77ab0ccca48e/test/e2e/framework/ssh/ssh.go#L59-L62
  vars = append(vars, fmt.Sprintf("KUBE_SSH_KEY_PATH=%v", d.SSHPrivateKeyPath))
+
+ if d.awsStaticCredentials != nil {
+ vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", d.awsStaticCredentials.AccessKeyID))
+ vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", d.awsStaticCredentials.SecretAccessKey))
+ }
  } else if d.CloudProvider == "digitalocean" {
  // Pass through some env vars if set
  for _, k := range []string{"DIGITALOCEAN_ACCESS_TOKEN", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"} {

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -28,7 +28,6 @@ import (
  "k8s.io/kops/tests/e2e/kubetest2-kops/builder"
  "k8s.io/kops/tests/e2e/pkg/target"
 
- "sigs.k8s.io/boskos/client"
  "sigs.k8s.io/kubetest2/pkg/types"
 )
 
@@ -82,13 +81,17 @@ type deployer struct {
  manifestPath string
  terraform *target.Terraform
 
- // boskos struct field will be non-nil when the deployer is
- // using boskos to acquire a GCP project
- boskos *client.Client
+ BoskosResourceType string `flag:"boskos-resource-type" desc:"Resource type to acquire from boskos, for credentials"`
 
- // this channel serves as a signal channel for the hearbeat goroutine
- // so that it can be explicitly closed
- boskosHeartbeatClose chan struct{}
+ boskos boskosHelper
+
+ // awsStaticCredentials holds credentials for AWS loaded from boskos
+ awsStaticCredentials *awsStaticCredentials
+}
+
+type awsStaticCredentials struct {
+ AccessKeyID string
+ SecretAccessKey string
 }
 
 // assert that New implements types.NewDeployer
@@ -106,9 +109,8 @@ func (d *deployer) Provider() string {
 func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
  // create a deployer object and set fields that are not flag controlled
  d := &deployer{
- commonOptions: opts,
- BuildOptions: &builder.BuildOptions{},
- boskosHeartbeatClose: make(chan struct{}),
+ commonOptions: opts,
+ BuildOptions: &builder.BuildOptions{},
  }
 
  dir, err := defaultArtifactsDir()

tests/e2e/kubetest2-kops/deployer/down.go

@@ -17,16 +17,17 @@ limitations under the License.
 package deployer
 
 import (
- "fmt"
+ "context"
  "strings"
 
  "k8s.io/klog/v2"
  "k8s.io/kops/tests/e2e/kubetest2-kops/gce"
- "sigs.k8s.io/kubetest2/pkg/boskos"
  "sigs.k8s.io/kubetest2/pkg/exec"
 )
 
 func (d *deployer) Down() error {
+ ctx := context.TODO()
+
  if err := d.init(); err != nil {
  return err
  }
@@ -58,16 +59,8 @@ func (d *deployer) Down() error {
  gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
  }
 
- if d.boskos != nil {
- klog.V(2).Info("releasing boskos project")
- err := boskos.Release(
- d.boskos,
- []string{d.GCPProject},
- d.boskosHeartbeatClose,
- )
- if err != nil {
- return fmt.Errorf("down failed to release boskos project: %s", err)
- }
+ if err := d.boskos.Cleanup(ctx); err != nil {
+ return err
  }
  return nil
 }