scaleway: e2e test

.gitignore

@@ -92,3 +92,9 @@ _output
 # Used by E2E testing
 _artifacts
 _rundir
+zzz-dev-Mia-Cross
+zzz-testing
+zzz-dev-Mia-Cross
+zzz-testing
+hack/upload
+logs

cmd/kops-controller/controllers/scalewayipam.go

@@ -0,0 +1,131 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/go-logr/logr"
+ "github.com/scaleway/scaleway-sdk-go/api/ipam/v1"
+ "github.com/scaleway/scaleway-sdk-go/scw"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+ "k8s.io/klog/v2"
+ kopsv "k8s.io/kops"
+ "k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// NewScalewayIPAMReconciler is the constructor for a IPAMReconciler
+func NewScalewayIPAMReconciler(mgr manager.Manager) (*ScalewayIPAMReconciler, error) {
+ klog.Info("Starting scaleway ipam controller")
+ r := &ScalewayIPAMReconciler{
+ client: mgr.GetClient(),
+ log: ctrl.Log.WithName("controllers").WithName("IPAM"),
+ }
+
+ coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
+ if err != nil {
+ return nil, fmt.Errorf("error building corev1 client: %v", err)
+ }
+ r.coreV1Client = coreClient
+
+ profile, err := scaleway.CreateValidScalewayProfile()
+ if err != nil {
+ return nil, err
+ }
+ scwClient, err := scw.NewClient(
+ scw.WithProfile(profile),
+ scw.WithUserAgent(scaleway.KopsUserAgentPrefix+kopsv.Version),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("creating client for Scaleway IPAM controller: %w", err)
+ }
+ r.scwClient = scwClient
+
+ return r, nil
+}
+
+// ScalewayIPAMReconciler observes Node objects, and labels them with the correct labels for the instancegroup
+// This used to be done by the kubelet, but is moving to a central controller for greater security in 1.16
+type ScalewayIPAMReconciler struct {
+ // client is the controller-runtime client
+ client client.Client
+
+ // log is a logr
+ log logr.Logger
+
+ // coreV1Client is a client-go client for patching nodes
+ coreV1Client *corev1client.CoreV1Client
+
+ scwClient *scw.Client
+}
+
+// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
+// Reconcile is the main reconciler function that observes node changes.
+func (r *ScalewayIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+ _ = r.log.WithValues("ipam-controller", req.NamespacedName)
+
+ node := &corev1.Node{}
+ if err := r.client.Get(ctx, req.NamespacedName, node); err != nil {
+ klog.Warningf("unable to fetch node %s: %v", node.Name, err)
+ if apierrors.IsNotFound(err) {
+ // we'll ignore not-found errors, since they can't be fixed by an immediate
+ // requeue (we'll need to wait for a new notification), and we can get them
+ // on deleted requests.
+ return ctrl.Result{}, nil
+ }
+ return ctrl.Result{}, err
+ }
+
+ if len(node.Spec.PodCIDRs) == 0 {
+ // CCM Node Controller has not done its thing yet
+ if node.Spec.ProviderID == "" {
+ klog.Infof("Node %q has empty provider ID", node.Name)
+ return ctrl.Result{}, nil
+ }
+
+ // providerID scaleway://instance/fr-par-1/instance-id
+ uuid := strings.Split(node.Spec.ProviderID, "/")
+ if len(uuid) != 3 {
+ return ctrl.Result{}, fmt.Errorf("unexpected format for server id %s", node.Spec.ProviderID)
+ }
+ serverID := uuid[2]
+ zone := scw.Zone(uuid[1])
+
+ ip, err := scaleway.GetIPAMPublicIP(ipam.NewAPI(r.scwClient), serverID, zone)
+ if err != nil {
+ return ctrl.Result{}, fmt.Errorf("could not get IPAM public IP for server %s: %w", serverID, err)
+ }
+ if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ip); err != nil {
+ return ctrl.Result{}, err
+ }
+ }
+
+ return ctrl.Result{}, nil
+}
+
+func (r *ScalewayIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
+ return ctrl.NewControllerManagedBy(mgr).
+ For(&corev1.Node{}).
+ Complete(r)
+}

cmd/kops-controller/main.go

@@ -385,6 +385,12 @@ func setupCloudIPAM(ctx context.Context, mgr manager.Manager, opt *config.Option
  return fmt.Errorf("creating gce IPAM controller: %w", err)
  }
  controller = ipamController
+ //case "scaleway":
+ // ipamController, err := controllers.NewScalewayIPAMReconciler(mgr)
+ // if err != nil {
+ // return fmt.Errorf("creating scaleway IPAM controller: %w", err)
+ // }
+ // controller = ipamController
  default:
  return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud)
  }

go.mod

@@ -51,7 +51,7 @@ require (
  github.com/pelletier/go-toml v1.9.5
  github.com/pkg/sftp v1.13.6
  github.com/prometheus/client_golang v1.19.0
- github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26
+ github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26.0.20240422085138-c14591c368f2
  github.com/sergi/go-diff v1.3.1
  github.com/spf13/cobra v1.8.0
  github.com/spf13/pflag v1.0.5

go.sum

@@ -595,8 +595,8 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
 github.com/sahilm/fuzzy v0.1.0 h1:FzWGaw2Opqyu+794ZQ9SYifWv2EIXpwP4q8dY1kDAwI=
 github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26 h1:F+GIVtGqCFxPxO46ujf8cEOP574MBoRm3gNbPXECbxs=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26.0.20240422085138-c14591c368f2 h1:9ZT4vcFKGvp1dcr5CJkVQivBkecrZybOtpnOfrJ0crc=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.26.0.20240422085138-c14591c368f2/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=

hack/upload

@@ -56,10 +56,24 @@ fi
 
 if [[ "${DEST:0:6}" == "scw://" ]]; then
  SCW_BUCKET=$(echo "${DEST}" | cut -c 7-)
- echo "--> s3cmd put ${SRC} s3://$SCW_BUCKET --recursive ${PUBLIC:+--acl-public} --progress"
- s3cmd put ${SRC} s3://$SCW_BUCKET --recursive ${PUBLIC:+--acl-public} --progress
- exit 0
-fi
+ if [[ $(pwd) == "/home/leila/Desktop/kops" ]]; then
+ REGION="par"
+ elif [[ $(pwd) == "/home/leila/Desktop/kops_v2" ]]; then
+ REGION="ams"
+ fi
+ if [[ $SCW_BUCKET == "kops-images-testing" ]]; then
+ PROFILE="kops"
+ REGION="par"
+ else
+ PROFILE="normal"
+ fi
+ echo "--> rclone sync ${SRC} $PROFILE-$REGION://$SCW_BUCKET --progress"
+ rclone sync ${SRC} $PROFILE-$REGION://$SCW_BUCKET --progress
+ # echo "--> s3cmd put ${SRC} s3://$SCW_BUCKET --recursive ${PUBLIC:+--acl-public} --progress"
+ # s3cmd put ${SRC} s3://$SCW_BUCKET --recursive ${PUBLIC:+--acl-public} --progress
+ # fi
+ exit 0
+ fi
 
 echo "Unsupported destination - supports s3://, gs:// and oss:// urls: ${DEST}"
 exit 1

nodeup/pkg/model/context.go

@@ -29,13 +29,16 @@ import (
  "github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
  "github.com/blang/semver/v4"
  hcloudmetadata "github.com/hetznercloud/hcloud-go/hcloud/metadata"
+ "github.com/scaleway/scaleway-sdk-go/api/instance/v1"
+ "github.com/scaleway/scaleway-sdk-go/scw"
  "k8s.io/klog/v2"
  "k8s.io/kops/pkg/apis/kops"
  "k8s.io/kops/pkg/apis/kops/model"
  "k8s.io/kops/pkg/apis/kops/util"
  "k8s.io/kops/pkg/apis/nodeup"
  "k8s.io/kops/pkg/systemd"
  "k8s.io/kops/upup/pkg/fi"
+ "k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
  "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
  "k8s.io/kops/upup/pkg/fi/utils"
  "k8s.io/kops/util/pkg/architectures"
@@ -604,6 +607,24 @@ func (c *NodeupModelContext) GetMetadataLocalIP(ctx context.Context) (string, er
  }
  }
 
+ case kops.CloudProviderScaleway:
+ metadataAPI := instance.NewMetadataAPI()
+ metadata, err := metadataAPI.GetMetadata()
+ if err != nil {
+ return "", fmt.Errorf("failed to retrieve server metadata: %w", err)
+ }
+
+ zone, err := scw.ParseZone(metadata.Location.ZoneID)
+ if err != nil {
+ return "", fmt.Errorf("unable to parse Scaleway zone: %w", err)
+ }
+
+ ip, err := scaleway.GetIPAMPublicIP(nil, metadata.ID, zone)
+ if err != nil {
+ return "", fmt.Errorf("failed to retrieve server IP: %w", err)
+ }
+ internalIP = ip
+
  default:
  return "", fmt.Errorf("getting local IP from metadata is not supported for cloud provider: %q", c.BootConfig.CloudProvider)
  }

nodeup/pkg/model/kube_apiserver.go

@@ -67,7 +67,7 @@ func (b *KubeAPIServerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
  kubeAPIServer = *b.NodeupConfig.APIServerConfig.KubeAPIServer
  }
 
- if b.CloudProvider() == kops.CloudProviderHetzner {
+ if b.CloudProvider() == kops.CloudProviderHetzner || b.CloudProvider() == kops.CloudProviderScaleway {
  localIP, err := b.GetMetadataLocalIP(c.Context())
  if err != nil {
  return err

nodeup/pkg/model/kubelet.go

@@ -30,6 +30,8 @@ import (
  awsconfig "github.com/aws/aws-sdk-go-v2/config"
  "github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
  ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+ "github.com/scaleway/scaleway-sdk-go/api/instance/v1"
+ "github.com/scaleway/scaleway-sdk-go/scw"
  "k8s.io/apimachinery/pkg/runtime"
  "k8s.io/apimachinery/pkg/runtime/serializer"
  "k8s.io/klog/v2"
@@ -89,12 +91,29 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
  return err
  }
  providerID = fmt.Sprintf("aws:///%s/%s", instanceIdentity.AvailabilityZone, instanceIdentity.InstanceID)
+
+ } else if b.CloudProvider() == kops.CloudProviderScaleway {
+ klog.Info("DOING STUFF FOR SCW")
+ metadataAPI := instance.NewMetadataAPI()
+ metadata, err := metadataAPI.GetMetadata()
+ if err != nil {
+ return fmt.Errorf("failed to retrieve server metadata: %w", err)
+ }
+ instanceID := metadata.ID
+ zone, err := scw.ParseZone(metadata.Location.ZoneID)
+ if err != nil {
+ return fmt.Errorf("unable to parse Scaleway zone: %w", err)
+ }
+ providerID = fmt.Sprintf("scaleway://instance/%s/%s", zone, instanceID)
+ klog.Infof("PROVIDER ID = %s", providerID)
  }
 
+ klog.Info("DID THE STUFF, NOW APPLY IT")
  t, err := buildKubeletComponentConfig(kubeletConfig, providerID)
  if err != nil {
  return err
  }
+ klog.Info("APPLIED !")
 
  c.AddTask(t)
  }
@@ -206,13 +225,13 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 
  }
  /* Kubelet incorrectly interprets this value when CgroupDriver is systemd
- See https://github.com/kubernetes/kubernetes/issues/101189
- {
- cgroup := kubeletConfig.KubeReservedCgroup
- if cgroup != "" {
- c.EnsureTask(b.buildCgroupService(cgroup))
- }
- }
+  See https://github.com/kubernetes/kubernetes/issues/101189
+  {
+   cgroup := kubeletConfig.KubeReservedCgroup
+   if cgroup != "" {
+   c.EnsureTask(b.buildCgroupService(cgroup))
+   }
+  }
  */
 
  {
@@ -223,12 +242,12 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
  }
 
  /* This suffers from the same issue as KubeReservedCgroup
- {
- cgroup := kubeletConfig.SystemReservedCgroup
- if cgroup != "" {
- c.EnsureTask(b.buildCgroupService(cgroup))
- }
- }
+  {
+   cgroup := kubeletConfig.SystemReservedCgroup
+   if cgroup != "" {
+   c.EnsureTask(b.buildCgroupService(cgroup))
+   }
+  }
  */
  }
 

pkg/model/components/etcdmanager/model.go

@@ -171,7 +171,8 @@ metadata:
 spec:
  containers:
  - name: etcd-manager
- image: registry.k8s.io/etcdadm/etcd-manager-slim:v3.0.20230925
+ # TODO(Mia-Cross): revert this change with the etcd-manager's next release 
+ image: rg.nl-ams.scw.cloud/kops-v2/etcd-manager:latest
  resources:
  requests:
  cpu: 100m

pkg/model/components/kubedns.go

@@ -17,6 +17,7 @@ limitations under the License.
 package components
 
 import (
+ corev1 "k8s.io/api/core/v1"
  "k8s.io/apimachinery/pkg/api/resource"
 
  "k8s.io/kops/pkg/apis/kops"
@@ -37,6 +38,19 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error {
 
  if clusterSpec.KubeDNS == nil {
  clusterSpec.KubeDNS = &kops.KubeDNSConfig{}
+ if clusterSpec.CloudProvider.Scaleway != nil {
+ clusterSpec.KubeDNS.Tolerations = []corev1.Toleration{
+ {
+ Key: "node-role.kubernetes.io/control-plane",
+ Operator: corev1.TolerationOpExists,
+ Effect: corev1.TaintEffectNoSchedule,
+ },
+ {
+ Key: "CriticalAddonsOnly",
+ Operator: corev1.TolerationOpExists,
+ },
+ }
+ }
  }
 
  if clusterSpec.KubeDNS.CacheMaxSize == 0 {
@@ -80,6 +94,10 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error {
  }
  }
 
+ //if clusterSpec.GetCloudProvider() == kops.CloudProviderScaleway {
+ // clusterSpec.KubeDNS.UpstreamNameservers = []string{"100.64.0.10"}
+ //}
+
  nodeLocalDNS := clusterSpec.KubeDNS.NodeLocalDNS
  if nodeLocalDNS == nil {
  nodeLocalDNS = &kops.NodeLocalDNSConfig{}

pkg/model/scalewaymodel/instances.go

@@ -78,6 +78,7 @@ func (b *InstanceModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
  instance.Tags = append(instance.Tags, scaleway.TagNameRolePrefix+"="+scaleway.TagRoleControlPlane)
  instance.Role = fi.PtrTo(scaleway.TagRoleControlPlane)
  } else {
+ instance.Tags = append(instance.Tags, scaleway.TagNameRolePrefix+"="+scaleway.TagRoleWorker)
  instance.Role = fi.PtrTo(scaleway.TagRoleWorker)
  }
 

protokube/pkg/protokube/scaleway_volumes.go

@@ -21,7 +21,7 @@ import (
  "net"
 
  "github.com/scaleway/scaleway-sdk-go/api/instance/v1"
- ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1"
+ "github.com/scaleway/scaleway-sdk-go/api/ipam/v1"
  "github.com/scaleway/scaleway-sdk-go/scw"
  "k8s.io/klog/v2"
  kopsv "k8s.io/kops"