Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

provider: added k3d provider and node lifecycle handlers #441

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+785 −34
Open

provider: added k3d provider and node lifecycle handlers #441

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions examples/k3d/k3d_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k3d

import (
"context"
"fmt"
"testing"
"time"

"sigs.k8s.io/e2e-framework/pkg/stepfuncs"
"sigs.k8s.io/e2e-framework/support"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/e2e-framework/klient/wait"
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/pkg/features"
)

func newDeployment(namespace string, name string, replicaCount int32) *appsv1.Deployment {
podSpec := corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "my-container",
Image: "nginx",
},
},
}
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app": "test-app"}},
Spec: appsv1.DeploymentSpec{
Replicas: &replicaCount,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "test-app"},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "test-app"}},
Spec: podSpec,
},
},
}
}

func TestK3DCluster(t *testing.T) {
deploymentFeature := features.New("Should be able to create a new deployment in the k3d cluster").
Assess("Create a new deployment", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
deployment := newDeployment(c.Namespace(), "test-deployment", 1)
if err := c.Client().Resources().Create(ctx, deployment); err != nil {
t.Fatal(err)
}
var dep appsv1.Deployment
if err := c.Client().Resources().Get(ctx, "test-deployment", c.Namespace(), &dep); err != nil {
t.Fatal(err)
}
err := wait.For(conditions.New(c.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, corev1.ConditionTrue), wait.WithTimeout(time.Minute*3))
if err != nil {
t.Fatal(err)
}
return context.WithValue(ctx, "test-deployment", &dep)
}).
Feature()

nodeAddFeature := features.New("Should be able to add a new node to the k3d cluster").
Setup(stepfuncs.PerformNodeOperation(support.AddNode, &support.Node{
Name: fmt.Sprintf("%s-agent", clusterName),
Cluster: clusterName,
Role: "agent",
})).
Assess("Check if the node is added to the cluster", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
var node corev1.Node
if err := c.Client().Resources().Get(ctx, fmt.Sprintf("k3d-%s-agent-0", clusterName), c.Namespace(), &node); err != nil {
t.Fatal(err)
}
return ctx
}).Feature()

testEnv.Test(t, deploymentFeature, nodeAddFeature)
}
51 changes: 51 additions & 0 deletions examples/k3d/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k3d

import (
"os"
"testing"

"sigs.k8s.io/e2e-framework/pkg/env"
"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/pkg/envfuncs"
"sigs.k8s.io/e2e-framework/support/k3d"
)

var (
testEnv env.Environment
clusterName string
)

func TestMain(m *testing.M) {
testEnv = env.New()
clusterName = envconf.RandomName("test", 16)
namespace := envconf.RandomName("k3d-ns", 16)

testEnv.Setup(
envfuncs.CreateClusterWithOpts(k3d.NewProvider(), clusterName, k3d.WithImage("rancher/k3s:v1.29.6-k3s1")),
harshanarayana marked this conversation as resolved.
Show resolved Hide resolved
envfuncs.CreateNamespace(namespace),
envfuncs.LoadImageToCluster(clusterName, "rancher/k3s:v1.29.6-k3s1", "--verbose", "--mode", "direct"),
)

testEnv.Finish(
envfuncs.DeleteNamespace(namespace),
envfuncs.DestroyCluster(clusterName),
)

os.Exit(testEnv.Run(m))
}
48 changes: 34 additions & 14 deletions pkg/envfuncs/provider_funcs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,19 @@ import (
"context"
"fmt"

"sigs.k8s.io/e2e-framework/pkg/utils"

"sigs.k8s.io/e2e-framework/pkg/env"
"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/support"
)

type clusterNameContextKey string

var LoadDockerImageToCluster = LoadImageToCluster

// GetClusterFromContext helps extract the E2EClusterProvider object from the context.
// This can be used to setup and run tests of multi cluster e2e Prioviders.
func GetClusterFromContext(ctx context.Context, clusterName string) (support.E2EClusterProvider, bool) {
c := ctx.Value(clusterNameContextKey(clusterName))
c := ctx.Value(support.ClusterNameContextKey(clusterName))
if c == nil {
return nil, false
}
Expand All @@ -47,8 +47,19 @@ func GetClusterFromContext(ctx context.Context, clusterName string) (support.E2E
// NOTE: the returned function will update its env config with the
// kubeconfig file for the config client.
func CreateCluster(p support.E2EClusterProvider, clusterName string) env.Func {
return CreateClusterWithOpts(p, clusterName)
}

// CreateClusterWithOpts returns an env.Func that is used to
// create an E2E provider cluster that is then injected in the context
// using the name as a key. This can be provided with additional opts to extend the create
// workflow of the cluster.
//
// NOTE: the returned function will update its env config with the
// kubeconfig file for the config client.
func CreateClusterWithOpts(p support.E2EClusterProvider, clusterName string, opts ...support.ClusterOpts) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
k := p.SetDefaults().WithName(clusterName)
k := p.SetDefaults().WithName(clusterName).WithOpts(opts...)
kubecfg, err := k.Create(ctx)
if err != nil {
return ctx, err
Expand All @@ -63,7 +74,7 @@ func CreateCluster(p support.E2EClusterProvider, clusterName string) env.Func {
}

// store entire cluster value in ctx for future access using the cluster name
return context.WithValue(ctx, clusterNameContextKey(clusterName), k), nil
return context.WithValue(ctx, support.ClusterNameContextKey(clusterName), k), nil
}
}

Expand All @@ -90,7 +101,7 @@ func CreateClusterWithConfig(p support.E2EClusterProvider, clusterName, configFi
}

// store entire cluster value in ctx for future access using the cluster name
return context.WithValue(ctx, clusterNameContextKey(clusterName), k), nil
return context.WithValue(ctx, support.ClusterNameContextKey(clusterName), k), nil
}
}

Expand All @@ -100,7 +111,7 @@ func CreateClusterWithConfig(p support.E2EClusterProvider, clusterName, configFi
// NOTE: this should be used in a Environment.Finish step.
func DestroyCluster(name string) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
clusterVal := ctx.Value(clusterNameContextKey(name))
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
if clusterVal == nil {
return ctx, fmt.Errorf("destroy e2e provider cluster func: context cluster is nil")
}
Expand All @@ -121,9 +132,9 @@ func DestroyCluster(name string) env.Func {
// LoadImageToCluster returns an EnvFunc that
// retrieves a previously saved e2e provider Cluster in the context (using the name), and then loads a container image
// from the host into the cluster.
func LoadImageToCluster(name, image string) env.Func {
func LoadImageToCluster(name, image string, args ...string) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
clusterVal := ctx.Value(clusterNameContextKey(name))
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
if clusterVal == nil {
return ctx, fmt.Errorf("load image func: context cluster is nil")
}
Expand All @@ -133,7 +144,7 @@ func LoadImageToCluster(name, image string) env.Func {
return ctx, fmt.Errorf("load image archive func: cluster provider does not support LoadImage helper")
}

if err := cluster.LoadImage(ctx, image); err != nil {
if err := cluster.LoadImage(ctx, image, args...); err != nil {
return ctx, fmt.Errorf("load image: %w", err)
}

Expand All @@ -144,9 +155,9 @@ func LoadImageToCluster(name, image string) env.Func {
// LoadImageArchiveToCluster returns an EnvFunc that
// retrieves a previously saved e2e provider Cluster in the context (using the name), and then loads a container image TAR archive
// from the host into the cluster.
func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
func LoadImageArchiveToCluster(name, imageArchive string, args ...string) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
clusterVal := ctx.Value(clusterNameContextKey(name))
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
if clusterVal == nil {
return ctx, fmt.Errorf("load image archive func: context cluster is nil")
}
Expand All @@ -156,7 +167,7 @@ func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
return ctx, fmt.Errorf("load image archive func: cluster provider does not support LoadImageArchive helper")
}

if err := cluster.LoadImageArchive(ctx, imageArchive); err != nil {
if err := cluster.LoadImageArchive(ctx, imageArchive, args...); err != nil {
return ctx, fmt.Errorf("load image archive: %w", err)
}

Expand All @@ -169,7 +180,7 @@ func LoadImageArchiveToCluster(name, imageArchive string) env.Func {
// in the provided destination.
func ExportClusterLogs(name, dest string) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
clusterVal := ctx.Value(clusterNameContextKey(name))
clusterVal := ctx.Value(support.ClusterNameContextKey(name))
if clusterVal == nil {
return ctx, fmt.Errorf("export e2e provider cluster logs: context cluster is nil")
}
Expand All @@ -186,3 +197,12 @@ func ExportClusterLogs(name, dest string) env.Func {
return ctx, nil
}
}

// PerformNodeOperation returns an EnvFunc that can be used to perform some node lifecycle operations.
// This can be used to add/remove/start/stop nodes in the cluster.
func PerformNodeOperation(clusterName string, action support.NodeOperation, node *support.Node, args ...string) env.Func {
return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
err := utils.PerformNodeLifecycleOperation(ctx, action, node, args...)
return ctx, err
}
}
43 changes: 43 additions & 0 deletions pkg/stepfuncs/nodelifecycle_funcs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package stepfuncs

import (
"context"
"testing"

"sigs.k8s.io/e2e-framework/pkg/utils"

"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/pkg/types"
"sigs.k8s.io/e2e-framework/support"
)

// PerformNodeOperation returns a step function that performs a node operation on a cluster.
// This can be integrated as a setup function for a feature in question before the feature
// is tested.
func PerformNodeOperation(action support.NodeOperation, node *support.Node, args ...string) types.StepFunc {
return func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
t.Helper()

err := utils.PerformNodeLifecycleOperation(ctx, action, node, args...)
if err != nil {
t.Fatalf("failed to perform node operation: %v", err)
}
return ctx
}
}
51 changes: 51 additions & 0 deletions pkg/utils/nodelifecycle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
"context"
"fmt"

"sigs.k8s.io/e2e-framework/support"
)

// PerformNodeLifecycleOperation performs a node operation on a cluster. These operations can range from Add/Remove/Start/Stop.
// This helper is re-used in both node lifecycle handler used as types.StepFunc or env.Func
func PerformNodeLifecycleOperation(ctx context.Context, action support.NodeOperation, node *support.Node, args ...string) error {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this function only applies when using K3d or do you think orther providers will use that.
If only k3d, maybe it belongs in third_party package path if that is the case.

Copy link
Contributor Author

@harshanarayana harshanarayana Aug 1, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Other providers like minikube can support this.. And kwok technically can be simulated to enable node add and remove workflows. So it can work with other providers as well. We can do that even for kind. But we will have to use adhoc docker command to simulate power on and off kind of workflows

clusterVal := ctx.Value(support.ClusterNameContextKey(node.Cluster))
if clusterVal == nil {
return fmt.Errorf("%s node to cluster: context cluster is nil", action)
}

clusterProvider, ok := clusterVal.(support.E2EClusterProviderWithLifeCycle)
if !ok {
return fmt.Errorf("cluster provider %s doesn't support node lifecycle operations", node.Cluster)
}

switch action {
case support.AddNode:
return clusterProvider.AddNode(ctx, node, args...)
case support.RemoveNode:
return clusterProvider.RemoveNode(ctx, node, args...)
case support.StartNode:
return clusterProvider.StartNode(ctx, node, args...)
case support.StopNode:
return clusterProvider.StopNode(ctx, node, args...)
default:
return fmt.Errorf("unknown node operation: %s", action)
}
}
Loading