From e18096d07eaf8a4dfec17c2c60705f18c1a86a7e Mon Sep 17 00:00:00 2001 From: Miguel Duarte Barroso Date: Wed, 20 Oct 2021 12:10:12 +0200 Subject: [PATCH 1/3] build: install kubebuilder tools for podman Signed-off-by: Miguel Duarte Barroso --- hack/install-kubebuilder-tools.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hack/install-kubebuilder-tools.sh b/hack/install-kubebuilder-tools.sh index 12b822b29..93fa2b13c 100755 --- a/hack/install-kubebuilder-tools.sh +++ b/hack/install-kubebuilder-tools.sh @@ -1,15 +1,17 @@ #!/bin/bash +OCI_BIN=${OCI_BIN:-docker} # install controller-gen go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1 # install kubebuilder tools to bin/ mkdir -p bin -containerID=$(docker create gcr.io/kubebuilder/thirdparty-linux:1.16.4) -docker cp ${containerID}:/kubebuilder_linux_amd64.tar.gz ./kubebuilder_linux_amd64.tar.gz -docker rm ${containerID} +containerID=$("$OCI_BIN" create gcr.io/kubebuilder/thirdparty-linux:1.16.4) +"$OCI_BIN" cp ${containerID}:/kubebuilder_linux_amd64.tar.gz ./kubebuilder_linux_amd64.tar.gz +"$OCI_BIN" rm ${containerID} tar -xzvf kubebuilder_linux_amd64.tar.gz rm kubebuilder_linux_amd64.tar.gz mv kubebuilder/bin/* bin/ rm -rf kubebuilder/ -chmod +x bin/ \ No newline at end of file +chmod +x bin/ + From 74483162d256c9b678bd39b56262ea355abf4376 Mon Sep 17 00:00:00 2001 From: Miguel Duarte Barroso Date: Wed, 20 Oct 2021 12:11:34 +0200 Subject: [PATCH 2/3] ip-reconciler: run without a defined kubeconfig With this change the reconciler can run in two different ways: - when ran in a k8s pod, it does not require to be told how to connect to the cluster. - when ran as a binary, it does require to know how to connect to the cluster, via the -kubeconfig config option. The reconciler cron spec is updated to use the correct service account name, and also is updated to run in the `kube-system` namespace. Signed-off-by: Miguel Duarte Barroso --- cmd/reconciler/ip.go | 13 ++++++++----- cmd/reconciler/ip_test.go | 8 ++++---- doc/crds/ip-reconciler-job.yaml | 3 ++- pkg/reconciler/iploop.go | 19 +++++++++++++++---- pkg/storage/kubernetes/client.go | 22 ++++++++++++++++++++-- pkg/storage/kubernetes/ipam.go | 2 +- 6 files changed, 50 insertions(+), 17 deletions(-) diff --git a/cmd/reconciler/ip.go b/cmd/reconciler/ip.go index dcaad3ced..7487175cd 100644 --- a/cmd/reconciler/ip.go +++ b/cmd/reconciler/ip.go @@ -15,15 +15,18 @@ func main() { logLevel := flag.String("log-level", "error", "the logging level for the `ip-reconciler` app. Valid values are: \"debug\", \"verbose\", \"error\", and \"panic\".") flag.Parse() - if *kubeConfigFile == "" { - _ = logging.Errorf("must specify the kubernetes config file, via the '-kubeconfig' flag") - os.Exit(kubeconfigNotFound) - } logging.SetLogLevel(*logLevel) ctx, cancel := context.WithTimeout(context.Background(), storage.RequestTimeout) defer cancel() - ipReconcileLoop, err := reconciler.NewReconcileLooper(*kubeConfigFile, ctx) + + var err error + var ipReconcileLoop *reconciler.ReconcileLooper + if kubeConfigFile == nil { + ipReconcileLoop, err = reconciler.NewReconcileLooper(ctx) + } else { + ipReconcileLoop, err = reconciler.NewReconcileLooperWithKubeconfig(*kubeConfigFile, ctx) + } if err != nil { _ = logging.Errorf("failed to create the reconcile looper: %v", err) os.Exit(couldNotStartOrphanedIPMonitor) diff --git a/cmd/reconciler/ip_test.go b/cmd/reconciler/ip_test.go index b2b838bd6..8fac94a5b 100644 --- a/cmd/reconciler/ip_test.go +++ b/cmd/reconciler/ip_test.go @@ -10,9 +10,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + multusv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" "github.com/k8snetworkplumbingwg/whereabouts/pkg/api/v1alpha1" "github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler" - multusv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -66,7 +66,7 @@ var _ = Describe("Whereabouts IP reconciler", func() { Context("reconciling the IPPool", func() { BeforeEach(func() { var err error - reconcileLooper, err = reconciler.NewReconcileLooper(kubeConfigPath, context.TODO()) + reconcileLooper, err = reconciler.NewReconcileLooperWithKubeconfig(kubeConfigPath, context.TODO()) Expect(err).NotTo(HaveOccurred()) }) @@ -137,7 +137,7 @@ var _ = Describe("Whereabouts IP reconciler", func() { Context("reconciling the IPPool", func() { BeforeEach(func() { var err error - reconcileLooper, err = reconciler.NewReconcileLooper(kubeConfigPath, context.TODO()) + reconcileLooper, err = reconciler.NewReconcileLooperWithKubeconfig(kubeConfigPath, context.TODO()) Expect(err).NotTo(HaveOccurred()) }) @@ -242,7 +242,7 @@ var _ = Describe("Whereabouts IP reconciler", func() { It("will delete an orphaned IP address", func() { Expect(k8sClientSet.CoreV1().Pods(namespace).Delete(context.TODO(), pods[podIndexToRemove].Name, metav1.DeleteOptions{})).NotTo(HaveOccurred()) - newReconciler, err := reconciler.NewReconcileLooper(kubeConfigPath, context.TODO()) + newReconciler, err := reconciler.NewReconcileLooperWithKubeconfig(kubeConfigPath, context.TODO()) Expect(err).NotTo(HaveOccurred()) Expect(newReconciler.ReconcileOverlappingIPAddresses()).To(Succeed()) diff --git a/doc/crds/ip-reconciler-job.yaml b/doc/crds/ip-reconciler-job.yaml index 6695b00e1..70c60cf35 100644 --- a/doc/crds/ip-reconciler-job.yaml +++ b/doc/crds/ip-reconciler-job.yaml @@ -2,6 +2,7 @@ apiVersion: batch/v1beta1 kind: CronJob metadata: name: ip-reconciler + namespace: kube-system labels: tier: node app: whereabouts @@ -12,6 +13,7 @@ spec: template: spec: priorityClassName: "system-node-critical" + serviceAccountName: whereabouts containers: - name: whereabouts image: ghcr.io/k8snetworkplumbingwg/whereabouts:latest-amd64 @@ -21,7 +23,6 @@ spec: memory: "50Mi" command: - /ip-reconciler - - -kubeconfig=/host/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig - -log-level=verbose volumeMounts: - name: cni-net-dir diff --git a/pkg/reconciler/iploop.go b/pkg/reconciler/iploop.go index 25fd42dd2..19c591a89 100644 --- a/pkg/reconciler/iploop.go +++ b/pkg/reconciler/iploop.go @@ -28,14 +28,25 @@ type OrphanedIPReservations struct { Allocations []types.IPReservation } -func NewReconcileLooper(kubeConfigPath string, ctx context.Context) (*ReconcileLooper, error) { - logging.Debugf("NewReconcileLooper - Kubernetes config file located at: %s", kubeConfigPath) - k8sClient, err := kubernetes.NewClient(kubeConfigPath) +func NewReconcileLooperWithKubeconfig(kubeconfigPath string, ctx context.Context) (*ReconcileLooper, error) { + logging.Debugf("NewReconcileLooper - Kubernetes config file located at: %s", kubeconfigPath) + k8sClient, err := kubernetes.NewClientViaKubeconfig(kubeconfigPath) if err != nil { return nil, logging.Errorf("failed to instantiate the Kubernetes client: %+v", err) } - logging.Debugf("successfully read the kubernetes configuration file located at: %s", kubeConfigPath) + return newReconcileLooper(k8sClient, ctx) +} + +func NewReconcileLooper(ctx context.Context) (*ReconcileLooper, error) { + logging.Debugf("NewReconcileLooper - inferred connection data") + k8sClient, err := kubernetes.NewClient() + if err != nil { + return nil, logging.Errorf("failed to instantiate the Kubernetes client: %+v", err) + } + return newReconcileLooper(k8sClient, ctx) +} +func newReconcileLooper(k8sClient *kubernetes.Client, ctx context.Context) (*ReconcileLooper, error) { pods, err := k8sClient.ListPods() if err != nil { return nil, err diff --git a/pkg/storage/kubernetes/client.go b/pkg/storage/kubernetes/client.go index c9b32f877..d4f099448 100644 --- a/pkg/storage/kubernetes/client.go +++ b/pkg/storage/kubernetes/client.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -22,17 +23,34 @@ type Client struct { retries int } -func NewClient(kubeconfigPath string) (*Client, error) { +func NewClient() (*Client, error) { + scheme := runtime.NewScheme() + _ = whereaboutsv1alpha1.AddToScheme(scheme) + + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + return newClient(config, scheme) +} + +func NewClientViaKubeconfig(kubeconfigPath string) (*Client, error) { scheme := runtime.NewScheme() _ = whereaboutsv1alpha1.AddToScheme(scheme) config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { return nil, err } + return newClient(config, scheme) +} + +func newClient(config *rest.Config, schema *runtime.Scheme) (*Client, error) { clientSet, err := kubernetes.NewForConfig(config) if err != nil { return nil, err @@ -42,7 +60,7 @@ func NewClient(kubeconfigPath string) (*Client, error) { if err != nil { return nil, err } - c, err := client.New(config, client.Options{Scheme: scheme, Mapper: mapper}) + c, err := client.New(config, client.Options{Scheme: schema, Mapper: mapper}) if err != nil { return nil, err } diff --git a/pkg/storage/kubernetes/ipam.go b/pkg/storage/kubernetes/ipam.go index 789e6e581..3f365b173 100644 --- a/pkg/storage/kubernetes/ipam.go +++ b/pkg/storage/kubernetes/ipam.go @@ -38,7 +38,7 @@ func NewKubernetesIPAM(containerID string, ipamConf whereaboutstypes.IPAMConfig) return nil, fmt.Errorf("k8s config: namespace not present in context") } - kubernetesClient, err := NewClient(ipamConf.Kubernetes.KubeConfigPath) + kubernetesClient, err := NewClientViaKubeconfig(ipamConf.Kubernetes.KubeConfigPath) if err != nil { return nil, fmt.Errorf("failed instantiating kubernetes client: %v", err) } From b9a910261438c68d4b3b28123a6c54fe3fb7d1ab Mon Sep 17 00:00:00 2001 From: Miguel Duarte Barroso Date: Wed, 20 Oct 2021 17:56:19 +0200 Subject: [PATCH 3/3] reconciler: update logging call All (non error) logs have the same verbosity; it would be interesting to see the reconciled IPs by default, as such I'm bumping this particular logging call to `verbose`. That enables users to `kubectl logs --- pkg/reconciler/iploop.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/reconciler/iploop.go b/pkg/reconciler/iploop.go index 19c591a89..46b4584b1 100644 --- a/pkg/reconciler/iploop.go +++ b/pkg/reconciler/iploop.go @@ -179,7 +179,7 @@ func (rl ReconcileLooper) ReconcileOverlappingIPAddresses() error { failedReconciledClusterWideIPs = append(failedReconciledClusterWideIPs, overlappingIPStruct.GetName()) continue } - logging.Debugf("removed stale overlappingIP allocation [%s]", overlappingIPStruct.GetName()) + logging.Verbosef("removed stale overlappingIP allocation [%s]", overlappingIPStruct.GetName()) } if len(failedReconciledClusterWideIPs) != 0 {