From 4da8c105352e182a9874fdcfa8868b6223d0bb9d Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 14:42:24 +0100 Subject: [PATCH 01/12] Add traefikiov1alpha1 to scheme --- cmd/main.go | 3 +++ internal/controller/atom_controller.go | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 23cc54b..7f21193 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -26,6 +26,7 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + traefikiov1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -55,6 +56,8 @@ func init() { utilruntime.Must(pdoknlv3.AddToScheme(scheme)) utilruntime.Must(pdoknlv2beta1.AddToScheme(scheme)) + utilruntime.Must(traefikiov1alpha1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme } diff --git a/internal/controller/atom_controller.go b/internal/controller/atom_controller.go index 3955bd4..cf76ef1 100644 --- a/internal/controller/atom_controller.go +++ b/internal/controller/atom_controller.go @@ -31,7 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" pdoknlv3 "github.com/pdok/atom-operator/api/v3" - traefikv1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1" + traefikiov1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -149,19 +149,19 @@ func (r *AtomReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. r.Update(ctx, service) // Define the IngressRoute for Traefik - ingressRoute := &traefikv1.IngressRoute{ + ingressRoute := &traefikiov1alpha1.IngressRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "testecho-api", Namespace: testEcho.Namespace, }, - Spec: traefikv1.IngressRouteSpec{ - Routes: []traefikv1.Route{ + Spec: traefikiov1alpha1.IngressRouteSpec{ + Routes: []traefikiov1alpha1.Route{ { Match: "Host(`localhost`) || Host(`kangaroo.test.pdok.nl`) && PathPrefix(`/testecho`)", Kind: "Rule", - Services: []traefikv1.Service{ + Services: []traefikiov1alpha1.Service{ { - LoadBalancerSpec: traefikv1.LoadBalancerSpec{ + LoadBalancerSpec: traefikiov1alpha1.LoadBalancerSpec{ Name: "testecho-api", Port: intstr.FromInt32(80), }, From a34bdcc34984d3ff3dffd169c8be619774ca0537 Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 14:42:37 +0100 Subject: [PATCH 02/12] Update RBAC permissions --- config/rbac/role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 97090d3..9d2f8ca 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -14,6 +14,7 @@ rules: - get - list - update + - watch - apiGroups: - apps resources: From a8f9ab3621964a72398a824aea58a9c9f15166ea Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 15:01:46 +0100 Subject: [PATCH 03/12] Update RBAC permissions --- config/rbac/role.yaml | 4 +++- internal/controller/atom_controller.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9d2f8ca..b1218db 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -25,6 +25,7 @@ rules: - get - list - update + - watch - apiGroups: - pdok.nl resources: @@ -52,7 +53,7 @@ rules: - patch - update - apiGroups: - - traefik.containo.us + - traefik.io resources: - ingressroutes verbs: @@ -61,3 +62,4 @@ rules: - get - list - update + - watch diff --git a/internal/controller/atom_controller.go b/internal/controller/atom_controller.go index cf76ef1..9662f48 100644 --- a/internal/controller/atom_controller.go +++ b/internal/controller/atom_controller.go @@ -49,9 +49,9 @@ type AtomReconciler struct { // +kubebuilder:rbac:groups=pdok.nl,resources=atoms,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=pdok.nl,resources=atoms/status,verbs=get;update;patch // +kubebuilder:rbac:groups=pdok.nl,resources=atoms/finalizers,verbs=update -// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;get;update;list;delete +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=watch;create;get;update;list;delete // +kubebuilder:rbac:groups=core,resources=services,verbs=watch;create;get;update;list;delete -// +kubebuilder:rbac:groups=traefik.containo.us,resources=ingressroutes,verbs=create;get;update;list;delete +// +kubebuilder:rbac:groups=traefik.io,resources=ingressroutes,verbs=get;list;watch;create;update;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From f78d9de29c26a7fae2c1236f67db4220f035236c Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 16:44:45 +0100 Subject: [PATCH 04/12] Add MetadataLinks mapping --- api/v2beta1/atom_conversion.go | 32 +++++++------ api/v3/atom_types.go | 36 +++++++-------- api/v3/zz_generated.deepcopy.go | 21 +-------- config/crd/bases/pdok.nl_atoms.yaml | 71 ++++++++++------------------- 4 files changed, 63 insertions(+), 97 deletions(-) diff --git a/api/v2beta1/atom_conversion.go b/api/v2beta1/atom_conversion.go index 2e96d09..bc101b5 100644 --- a/api/v2beta1/atom_conversion.go +++ b/api/v2beta1/atom_conversion.go @@ -57,15 +57,17 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { // Service log.Printf("Start mapping the Service...") dst.Spec.Service = pdoknlv3.Service{ - BaseURL: createBaseURL(host, src.Spec.General), - Lang: "nl", - Stylesheet: "https://service.pdok.nl/atom/style/style.xsl", - Title: src.Spec.Service.Title, - Subtitle: src.Spec.Service.Subtitle, - OwnerInfoRef: "pdok", - Links: []pdoknlv3.Link{}, - ServiceMetadataLinks: []pdoknlv3.MetadataLink{}, // Todo - Rights: src.Spec.Service.Rights, + BaseURL: createBaseURL(host, src.Spec.General), + Lang: "nl", + Stylesheet: "https://service.pdok.nl/atom/style/style.xsl", + Title: src.Spec.Service.Title, + Subtitle: src.Spec.Service.Subtitle, + OwnerInfoRef: "pdok", + ServiceMetadataLinks: pdoknlv3.MetadataLink{ + MetadataIdentifier: src.Spec.Service.MetadataIdentifier, + Templates: []string{"csv", "opensearch", "html"}, + }, + Rights: src.Spec.Service.Rights, } log.Printf("Done mapping the Service...") @@ -73,10 +75,13 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { log.Printf("Start mapping the Datasets...") for _, srcDataset := range src.Spec.Service.Datasets { dstDatasetFeed := pdoknlv3.DatasetFeed{ - TechnicalName: srcDataset.Name, - Title: srcDataset.Title, - Subtitle: srcDataset.Subtitle, - DatasetMetadataLinks: []pdoknlv3.MetadataLink{}, // Todo + TechnicalName: srcDataset.Name, + Title: srcDataset.Title, + Subtitle: srcDataset.Subtitle, + DatasetMetadataLinks: pdoknlv3.MetadataLink{ + MetadataIdentifier: srcDataset.MetadataIdentifier, + Templates: []string{"csv", "html"}, + }, SpatialDatasetIdentifierCode: srcDataset.SourceIdentifier, SpatialDatasetIdentifierNamespace: "http://www.pdok.nl", } @@ -214,6 +219,7 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { Name: "PDOK Beheer", Email: "beheerPDOK@kadaster.nl", }, + MetadataIdentifier: src.Spec.Service.ServiceMetadataLinks.MetadataIdentifier, } log.Printf("Done mapping the Service...") diff --git a/api/v3/atom_types.go b/api/v3/atom_types.go index bd352da..186978a 100644 --- a/api/v3/atom_types.go +++ b/api/v3/atom_types.go @@ -42,15 +42,15 @@ type Lifecycle struct { // Service defines the service configuration for the Atom feed type Service struct { - BaseURL string `json:"baseUrl"` - Lang string `json:"lang,omitempty"` - Stylesheet string `json:"stylesheet,omitempty"` - Title string `json:"title"` - Subtitle string `json:"subtitle,omitempty"` - OwnerInfoRef string `json:"ownerInfoRef"` - ServiceMetadataLinks []MetadataLink `json:"serviceMetadataLinks,omitempty"` - Links []Link `json:"links,omitempty"` // Todo kan weg? - Rights string `json:"rights,omitempty"` + BaseURL string `json:"baseUrl"` + Lang string `json:"lang,omitempty"` + Stylesheet string `json:"stylesheet,omitempty"` + Title string `json:"title"` + Subtitle string `json:"subtitle,omitempty"` + OwnerInfoRef string `json:"ownerInfoRef"` + ServiceMetadataLinks MetadataLink `json:"serviceMetadataLinks,omitempty"` + //Links []Link `json:"links,omitempty"` // Todo kan weg? + Rights string `json:"rights,omitempty"` } // Link represents a link in the service or dataset feed @@ -72,15 +72,15 @@ type Author struct { // DatasetFeed represents individual dataset feeds within the Atom service type DatasetFeed struct { - TechnicalName string `json:"technicalName"` - Title string `json:"title"` - Subtitle string `json:"subtitle,omitempty"` - Links []Link `json:"links,omitempty"` // Todo kan weg? - DatasetMetadataLinks []MetadataLink `json:"datasetMetadataLinks,omitempty"` - Author Author `json:"author,omitempty"` - SpatialDatasetIdentifierCode string `json:"spatial_dataset_identifier_code,omitempty"` - SpatialDatasetIdentifierNamespace string `json:"spatial_dataset_identifier_namespace,omitempty"` - Entries []Entry `json:"entries,omitempty"` + TechnicalName string `json:"technicalName"` + Title string `json:"title"` + Subtitle string `json:"subtitle,omitempty"` + Links []Link `json:"links,omitempty"` // Todo kan weg? + DatasetMetadataLinks MetadataLink `json:"datasetMetadataLinks,omitempty"` + Author Author `json:"author,omitempty"` + SpatialDatasetIdentifierCode string `json:"spatial_dataset_identifier_code,omitempty"` + SpatialDatasetIdentifierNamespace string `json:"spatial_dataset_identifier_namespace,omitempty"` + Entries []Entry `json:"entries,omitempty"` } // Metadatalink represents a link in the service or dataset feed diff --git a/api/v3/zz_generated.deepcopy.go b/api/v3/zz_generated.deepcopy.go index e3c7719..0709659 100644 --- a/api/v3/zz_generated.deepcopy.go +++ b/api/v3/zz_generated.deepcopy.go @@ -168,13 +168,7 @@ func (in *DatasetFeed) DeepCopyInto(out *DatasetFeed) { *out = make([]Link, len(*in)) copy(*out, *in) } - if in.DatasetMetadataLinks != nil { - in, out := &in.DatasetMetadataLinks, &out.DatasetMetadataLinks - *out = make([]MetadataLink, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } + in.DatasetMetadataLinks.DeepCopyInto(&out.DatasetMetadataLinks) out.Author = in.Author if in.Entries != nil { in, out := &in.Entries, &out.Entries @@ -350,18 +344,7 @@ func (in *SRS) DeepCopy() *SRS { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Service) DeepCopyInto(out *Service) { *out = *in - if in.ServiceMetadataLinks != nil { - in, out := &in.ServiceMetadataLinks, &out.ServiceMetadataLinks - *out = make([]MetadataLink, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Links != nil { - in, out := &in.Links, &out.Links - *out = make([]Link, len(*in)) - copy(*out, *in) - } + in.ServiceMetadataLinks.DeepCopyInto(&out.ServiceMetadataLinks) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Service. diff --git a/config/crd/bases/pdok.nl_atoms.yaml b/config/crd/bases/pdok.nl_atoms.yaml index 023a9ac..00a7ee6 100644 --- a/config/crd/bases/pdok.nl_atoms.yaml +++ b/config/crd/bases/pdok.nl_atoms.yaml @@ -365,20 +365,18 @@ spec: - name type: object datasetMetadataLinks: - items: - description: Metadatalink represents a link in the service - or dataset feed - properties: - metadataIdentifier: + description: Metadatalink represents a link in the service or + dataset feed + properties: + metadataIdentifier: + type: string + templates: + items: type: string - templates: - items: - type: string - type: array - required: - - metadataIdentifier - type: object - type: array + type: array + required: + - metadataIdentifier + type: object entries: items: description: Entry represents an entry within a dataset feed, @@ -534,46 +532,25 @@ spec: type: string lang: type: string - links: - items: - description: Link represents a link in the service or dataset - feed - properties: - category: - type: string - href: - type: string - hreflang: - type: string - rel: - type: string - title: - type: string - type: - type: string - required: - - href - type: object - type: array ownerInfoRef: type: string rights: + description: Links []Link `json:"links,omitempty"` + // Todo kan weg? type: string serviceMetadataLinks: - items: - description: Metadatalink represents a link in the service or - dataset feed - properties: - metadataIdentifier: + description: Metadatalink represents a link in the service or + dataset feed + properties: + metadataIdentifier: + type: string + templates: + items: type: string - templates: - items: - type: string - type: array - required: - - metadataIdentifier - type: object - type: array + type: array + required: + - metadataIdentifier + type: object stylesheet: type: string subtitle: From eaf3787e10c10465c6918a6ca7baf1ba96681dbe Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 16:57:50 +0100 Subject: [PATCH 05/12] Linting --- api/v3/atom_types.go | 3 +- internal/controller/atom_controller.go | 28 +++++++++++++++---- internal/controller/atom_generator/mapping.go | 5 ++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/api/v3/atom_types.go b/api/v3/atom_types.go index 186978a..2a09260 100644 --- a/api/v3/atom_types.go +++ b/api/v3/atom_types.go @@ -49,8 +49,7 @@ type Service struct { Subtitle string `json:"subtitle,omitempty"` OwnerInfoRef string `json:"ownerInfoRef"` ServiceMetadataLinks MetadataLink `json:"serviceMetadataLinks,omitempty"` - //Links []Link `json:"links,omitempty"` // Todo kan weg? - Rights string `json:"rights,omitempty"` + Rights string `json:"rights,omitempty"` } // Link represents a link in the service or dataset feed diff --git a/internal/controller/atom_controller.go b/internal/controller/atom_controller.go index 9662f48..2ff870c 100644 --- a/internal/controller/atom_controller.go +++ b/internal/controller/atom_controller.go @@ -26,6 +26,7 @@ package controller import ( "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -112,7 +113,10 @@ func (r *AtomReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. ll.Error(err, "failed to get object") return ctrl.Result{}, err } - r.Create(ctx, deployment) + if err := r.Create(ctx, deployment); err != nil { + ll.Error(err, "failed to create Deployment") + return ctrl.Result{}, err + } } err := r.Update(ctx, deployment) if err != nil { @@ -144,9 +148,16 @@ func (r *AtomReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. ll.Error(err, "failed to get object") return ctrl.Result{}, err } - r.Create(ctx, service) + if err := r.Create(ctx, service); err != nil { + ll.Error(err, "failed to create Service") + return ctrl.Result{}, err + } + } + + if err := r.Update(ctx, service); err != nil { + ll.Error(err, "failed to update Service") + return ctrl.Result{}, err } - r.Update(ctx, service) // Define the IngressRoute for Traefik ingressRoute := &traefikiov1alpha1.IngressRoute{ @@ -186,9 +197,16 @@ func (r *AtomReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. ll.Error(err, "failed to get object: %v") return ctrl.Result{}, err } - r.Create(ctx, ingressRoute) + + if err := r.Create(ctx, ingressRoute); err != nil { + ll.Error(err, "failed to create IngressRoute") + return ctrl.Result{}, err + } + } + if err := r.Update(ctx, ingressRoute); err != nil { + ll.Error(err, "failed to update IngressRoute") + return ctrl.Result{}, err } - r.Update(ctx, ingressRoute) return ctrl.Result{}, nil } diff --git a/internal/controller/atom_generator/mapping.go b/internal/controller/atom_generator/mapping.go index 130f34b..abf53fb 100644 --- a/internal/controller/atom_generator/mapping.go +++ b/internal/controller/atom_generator/mapping.go @@ -1,8 +1,9 @@ package atom_generator import ( - pdoknlv3 "github.com/pdok/atom-operator/api/v3" "time" + + pdoknlv3 "github.com/pdok/atom-operator/api/v3" ) func MapAtomV3ToAtomGeneratorConfig(atom pdoknlv3.Atom) (atomGeneratorConfig AtomGeneratorConfig, err error) { @@ -121,5 +122,5 @@ func MapAtomV3ToAtomGeneratorConfig(atom pdoknlv3.Atom) (atomGeneratorConfig Ato }, } - return + return atomGeneratorConfig, err } From d800417b43503ec176ffaec9c97c8bc07c071fd3 Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 17:26:42 +0100 Subject: [PATCH 06/12] Update generated CRD --- config/crd/bases/pdok.nl_atoms.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/crd/bases/pdok.nl_atoms.yaml b/config/crd/bases/pdok.nl_atoms.yaml index 00a7ee6..a1fb720 100644 --- a/config/crd/bases/pdok.nl_atoms.yaml +++ b/config/crd/bases/pdok.nl_atoms.yaml @@ -535,8 +535,6 @@ spec: ownerInfoRef: type: string rights: - description: Links []Link `json:"links,omitempty"` - // Todo kan weg? type: string serviceMetadataLinks: description: Metadatalink represents a link in the service or From 5e14ec94da2ad416cef28e68d563f2ae81aa06ec Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 17:27:54 +0100 Subject: [PATCH 07/12] Fix controller test for Reconcile --- internal/controller/atom_controller_test.go | 7 ++++ internal/controller/suite_test.go | 40 ++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/internal/controller/atom_controller_test.go b/internal/controller/atom_controller_test.go index 7d8e8af..5df2c28 100644 --- a/internal/controller/atom_controller_test.go +++ b/internal/controller/atom_controller_test.go @@ -90,3 +90,10 @@ var _ = Describe("Atom Controller", func() { }) }) }) + +func must[T any](t T, err error) T { + if err != nil { + panic(err) + } + return t +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 531d5d1..c00f4b3 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -26,10 +26,16 @@ package controller import ( "context" + "encoding/json" + "errors" "os" + "os/exec" "path/filepath" "testing" + traefikiov1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1" + "golang.org/x/tools/go/packages" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -70,12 +76,23 @@ var _ = BeforeSuite(func() { err = pdoknlv3.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = traefikiov1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + // +kubebuilder:scaffold:scheme By("bootstrapping test environment") + traefikCRDPath := must(getTraefikCRDPath()) testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, + CRDInstallOptions: envtest.CRDInstallOptions{ + Scheme: nil, + Paths: []string{ + filepath.Join("..", "..", "config", "crd", "bases"), + traefikCRDPath, + }, + ErrorIfPathMissing: true, + }, } // Retrieve the first found binary directory to allow running tests from IDEs @@ -122,3 +139,24 @@ func getFirstFoundEnvTestBinaryDir() string { } return "" } + +func getTraefikCRDPath() (string, error) { + traefikModule, err := getModule("github.com/traefik/traefik/v2") + if err != nil { + return "", err + } + if traefikModule.Dir == "" { + return "", errors.New("cannot find path for traefik module") + } + return filepath.Join(traefikModule.Dir, "integration", "fixtures", "k8s", "01-traefik-crd.yml"), nil +} + +func getModule(name string) (module *packages.Module, err error) { + out, err := exec.Command("go", "list", "-json", "-m", name).Output() + if err != nil { + return + } + module = &packages.Module{} + err = json.Unmarshal(out, module) + return +} From 4f1a958b2c6585cb63e76a58e15f1958afb6f9f4 Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Wed, 19 Feb 2025 17:49:51 +0100 Subject: [PATCH 08/12] Remove debug logging --- api/v2beta1/atom_conversion.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/api/v2beta1/atom_conversion.go b/api/v2beta1/atom_conversion.go index bc101b5..aca4619 100644 --- a/api/v2beta1/atom_conversion.go +++ b/api/v2beta1/atom_conversion.go @@ -48,14 +48,11 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { dst.ObjectMeta = src.ObjectMeta // Lifecycle - log.Printf("Start mapping the Lifecycle specs...") if src.Spec.Kubernetes != nil && src.Spec.Kubernetes.Lifecycle != nil && src.Spec.Kubernetes.Lifecycle.TTLInDays != nil { dst.Spec.Lifecycle.TTLInDays = GetInt32Pointer(int32(*src.Spec.Kubernetes.Lifecycle.TTLInDays)) } - log.Printf("Done mapping the Lifecycle specs...") // Service - log.Printf("Start mapping the Service...") dst.Spec.Service = pdoknlv3.Service{ BaseURL: createBaseURL(host, src.Spec.General), Lang: "nl", @@ -69,10 +66,8 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { }, Rights: src.Spec.Service.Rights, } - log.Printf("Done mapping the Service...") dst.Spec.DatasetFeeds = []pdoknlv3.DatasetFeed{} - log.Printf("Start mapping the Datasets...") for _, srcDataset := range src.Spec.Service.Datasets { dstDatasetFeed := pdoknlv3.DatasetFeed{ TechnicalName: srcDataset.Name, @@ -87,7 +82,6 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { } // Map the links - log.Printf("Start mapping the Links...") for _, srcLink := range srcDataset.Links { dstLink := pdoknlv3.Link{ Title: srcLink.Type, @@ -102,10 +96,8 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { dstDatasetFeed.Links = append(dstDatasetFeed.Links, dstLink) } - log.Printf("Done mapping the Links...") // Map the entries - log.Printf("Start mapping the Entries...") for _, srcDownload := range srcDataset.Downloads { dstEntry := pdoknlv3.Entry{ TechnicalName: srcDownload.Name, @@ -140,7 +132,6 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { } // Map the links - log.Printf("Start mapping the DownloadLinks...") for _, srcLink := range srcDownload.Links { dstDownloadLink := pdoknlv3.DownloadLink{} @@ -167,15 +158,12 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { dstEntry.DownloadLinks = append(dstEntry.DownloadLinks, dstDownloadLink) } - log.Printf("Done mapping the DownloadLinks...") dstDatasetFeed.Entries = append(dstDatasetFeed.Entries, dstEntry) } - log.Printf("Done mapping the Entries...") dst.Spec.DatasetFeeds = append(dst.Spec.DatasetFeeds, dstDatasetFeed) } - log.Printf("Done mapping the Datasets...") return nil } @@ -190,7 +178,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { dst.ObjectMeta = src.ObjectMeta // General - log.Printf("Start mapping the General specs...") dst.Spec.General = General{ Dataset: src.ObjectMeta.Labels["dataset"], DatasetOwner: src.ObjectMeta.Labels["dataset-owner"], @@ -207,10 +194,7 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { dst.Spec.General.Theme = &theme } - log.Printf("Done mapping the General specs...") - // Service - log.Printf("Start mapping the Service...") dst.Spec.Service = AtomService{ Title: src.Spec.Service.Title, Subtitle: src.Spec.Service.Subtitle, @@ -221,10 +205,8 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { }, MetadataIdentifier: src.Spec.Service.ServiceMetadataLinks.MetadataIdentifier, } - log.Printf("Done mapping the Service...") // Datasets - log.Printf("Start mapping the Datasets...") dst.Spec.Service.Datasets = []Dataset{} for _, srcDatasetFeed := range src.Spec.DatasetFeeds { dstDataset := Dataset{ @@ -235,7 +217,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { } // Map the links - log.Printf("Start mapping the Links...") for _, srcLink := range srcDatasetFeed.Links { dstDataset.Links = append(dstDataset.Links, OtherLink{ Type: srcLink.Title, @@ -244,7 +225,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { Language: &srcLink.Hreflang, }) } - log.Printf("Done mapping the Links...") if len(srcDatasetFeed.Entries) > 0 && srcDatasetFeed.Entries[0].Polygon != nil { // We can assume all entries have the same bbox, so we take the first one @@ -258,7 +238,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { } // Map the downloads - log.Printf("Start mapping the Entries...") for _, srcEntry := range srcDatasetFeed.Entries { dstDownload := Download{ Name: srcEntry.TechnicalName, @@ -279,9 +258,7 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { } // Map the links - log.Printf("Start mapping the DownloadLinks...") for _, srcDownloadLink := range srcEntry.DownloadLinks { - dstLink := Link{ BlobKey: &srcDownloadLink.Data, Rel: &srcDownloadLink.Rel, @@ -303,23 +280,18 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { } } - log.Printf("Done mapping the DownloadLinks...") dstDataset.Downloads = append(dstDataset.Downloads, dstDownload) } - log.Printf("Done mapping the Entries...") dst.Spec.Service.Datasets = append(dst.Spec.Service.Datasets, dstDataset) } - log.Printf("Start mapping the Datasets...") // Kubernetes - log.Printf("Start mapping the Kubernetes Specs...") dst.Spec.Kubernetes = &Kubernetes{ Lifecycle: &Lifecycle{}, } if src.Spec.Lifecycle.TTLInDays != nil { dst.Spec.Kubernetes.Lifecycle.TTLInDays = GetIntPointer(int(*src.Spec.Lifecycle.TTLInDays)) } - log.Printf("Done mapping the Kubernetes Specs...") return nil } From 0efa5909ff672114afbe3f076beb61345c157649 Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Thu, 20 Feb 2025 15:36:54 +0100 Subject: [PATCH 09/12] Test global baseURLHost --- api/v2beta1/atom_conversion.go | 4 ++-- api/v3/atom_types.go | 11 ++++++++++- cmd/main.go | 7 +++++++ config/crd/bases/pdok.nl_atoms.yaml | 4 ++-- config/manager/manager.yaml | 1 + 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/api/v2beta1/atom_conversion.go b/api/v2beta1/atom_conversion.go index aca4619..4aa53f3 100644 --- a/api/v2beta1/atom_conversion.go +++ b/api/v2beta1/atom_conversion.go @@ -42,7 +42,7 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { log.Printf("ConvertTo: Converting Atom from Spoke version v2beta1 to Hub version v3;"+ "source: %s/%s, target: %s/%s", src.Namespace, src.Name, dst.Namespace, dst.Name) - host := "https://service.pdok.nl/" // Todo read from flag + log.Printf("Check Atom BaseURLHost: %s", pdoknlv3.BaseURLHost) // ObjectMeta dst.ObjectMeta = src.ObjectMeta @@ -54,7 +54,7 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { // Service dst.Spec.Service = pdoknlv3.Service{ - BaseURL: createBaseURL(host, src.Spec.General), + BaseURL: createBaseURL(pdoknlv3.BaseURLHost, src.Spec.General), Lang: "nl", Stylesheet: "https://service.pdok.nl/atom/style/style.xsl", Title: src.Spec.Service.Title, diff --git a/api/v3/atom_types.go b/api/v3/atom_types.go index 2a09260..549fcca 100644 --- a/api/v3/atom_types.go +++ b/api/v3/atom_types.go @@ -28,6 +28,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// BaseURLHost is accessed by other api versions (i.e. v2beta1) +var BaseURLHost string + // AtomSpec defines the desired state of Atom. type AtomSpec struct { Lifecycle Lifecycle `json:"lifecycle,omitempty"` @@ -82,7 +85,7 @@ type DatasetFeed struct { Entries []Entry `json:"entries,omitempty"` } -// Metadatalink represents a link in the service or dataset feed +// MetadataLink represents a link in the service or dataset feed type MetadataLink struct { MetadataIdentifier string `json:"metadataIdentifier"` Templates []string `json:"templates,omitempty"` @@ -169,3 +172,9 @@ type AtomList struct { func init() { SchemeBuilder.Register(&Atom{}, &AtomList{}) } + +// SetAtomBaseURLHost is used to set the BaseURL Host in main +func SetAtomBaseURLHost(atomBaseURLHost string) { + BaseURLHost := atomBaseURLHost + _ = BaseURLHost // Prevent unused variable +} diff --git a/cmd/main.go b/cmd/main.go index 7f21193..62ee94d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -70,6 +70,7 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool + var atomBaseURLHost string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -88,6 +89,9 @@ func main() { flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") + + flag.StringVar(&atomBaseURLHost, "atom-baseurl-host", "https://service.pdok.nl/main/", + "The host which is used to create the Atom BaseURL.") opts := zap.Options{ Development: true, } @@ -96,6 +100,9 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + setupLog.Info("Checking atomBaseURLHost value: " + atomBaseURLHost) + pdoknlv3.SetAtomBaseURLHost(atomBaseURLHost) + // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and diff --git a/config/crd/bases/pdok.nl_atoms.yaml b/config/crd/bases/pdok.nl_atoms.yaml index a1fb720..0e46caf 100644 --- a/config/crd/bases/pdok.nl_atoms.yaml +++ b/config/crd/bases/pdok.nl_atoms.yaml @@ -365,7 +365,7 @@ spec: - name type: object datasetMetadataLinks: - description: Metadatalink represents a link in the service or + description: MetadataLink represents a link in the service or dataset feed properties: metadataIdentifier: @@ -537,7 +537,7 @@ spec: rights: type: string serviceMetadataLinks: - description: Metadatalink represents a link in the service or + description: MetadataLink represents a link in the service or dataset feed properties: metadataIdentifier: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4efab2e..14b6656 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -54,6 +54,7 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 + - --atom-baseurl-host="https://service.pdok.nl/yaml/" image: controller:latest name: manager ports: [] From 6d8c59c171770b442edd6598f5984a8bc335c6f1 Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Thu, 20 Feb 2025 16:02:32 +0100 Subject: [PATCH 10/12] Test global baseURLHost --- api/v2beta1/atom_conversion.go | 4 ++-- api/v3/atom_types.go | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/v2beta1/atom_conversion.go b/api/v2beta1/atom_conversion.go index 4aa53f3..25d1f69 100644 --- a/api/v2beta1/atom_conversion.go +++ b/api/v2beta1/atom_conversion.go @@ -42,7 +42,7 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { log.Printf("ConvertTo: Converting Atom from Spoke version v2beta1 to Hub version v3;"+ "source: %s/%s, target: %s/%s", src.Namespace, src.Name, dst.Namespace, dst.Name) - log.Printf("Check Atom BaseURLHost: %s", pdoknlv3.BaseURLHost) + log.Printf("Check Atom BaseURLHost: %s", pdoknlv3.GetAtomBaseURLHost()) // ObjectMeta dst.ObjectMeta = src.ObjectMeta @@ -54,7 +54,7 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { // Service dst.Spec.Service = pdoknlv3.Service{ - BaseURL: createBaseURL(pdoknlv3.BaseURLHost, src.Spec.General), + BaseURL: createBaseURL(pdoknlv3.GetAtomBaseURLHost(), src.Spec.General), Lang: "nl", Stylesheet: "https://service.pdok.nl/atom/style/style.xsl", Title: src.Spec.Service.Title, diff --git a/api/v3/atom_types.go b/api/v3/atom_types.go index 549fcca..9cb26d4 100644 --- a/api/v3/atom_types.go +++ b/api/v3/atom_types.go @@ -29,7 +29,7 @@ import ( ) // BaseURLHost is accessed by other api versions (i.e. v2beta1) -var BaseURLHost string +var baseURLHost string // AtomSpec defines the desired state of Atom. type AtomSpec struct { @@ -175,6 +175,9 @@ func init() { // SetAtomBaseURLHost is used to set the BaseURL Host in main func SetAtomBaseURLHost(atomBaseURLHost string) { - BaseURLHost := atomBaseURLHost - _ = BaseURLHost // Prevent unused variable + baseURLHost = atomBaseURLHost +} + +func GetAtomBaseURLHost() string { + return baseURLHost } From eecd31aa187c91a331a13c1551bc806406af1fa6 Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Thu, 20 Feb 2025 17:01:24 +0100 Subject: [PATCH 11/12] Cleanup createBaseURL() --- api/v2beta1/atom_conversion.go | 6 ++---- cmd/main.go | 3 +-- config/manager/manager.yaml | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/api/v2beta1/atom_conversion.go b/api/v2beta1/atom_conversion.go index 25d1f69..9dd2c5d 100644 --- a/api/v2beta1/atom_conversion.go +++ b/api/v2beta1/atom_conversion.go @@ -28,6 +28,7 @@ import ( "fmt" "log" "strconv" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,8 +43,6 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { log.Printf("ConvertTo: Converting Atom from Spoke version v2beta1 to Hub version v3;"+ "source: %s/%s, target: %s/%s", src.Namespace, src.Name, dst.Namespace, dst.Name) - log.Printf("Check Atom BaseURLHost: %s", pdoknlv3.GetAtomBaseURLHost()) - // ObjectMeta dst.ObjectMeta = src.ObjectMeta @@ -297,7 +296,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error { } func createBaseURL(host string, general General) (baseURL string) { - atomURI := fmt.Sprintf("%s/%s", general.DatasetOwner, general.Dataset) if general.Theme != nil { atomURI += fmt.Sprintf("/%s", *general.Theme) @@ -308,7 +306,7 @@ func createBaseURL(host string, general General) (baseURL string) { atomURI += fmt.Sprintf("/%s", *general.ServiceVersion) } - baseURL = fmt.Sprintf("%s/%s/index.xml", host, atomURI) + baseURL = fmt.Sprintf("%s/%s/index.xml", strings.TrimSuffix(host, "/"), atomURI) return } diff --git a/cmd/main.go b/cmd/main.go index 62ee94d..879852d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -90,7 +90,7 @@ func main() { flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") - flag.StringVar(&atomBaseURLHost, "atom-baseurl-host", "https://service.pdok.nl/main/", + flag.StringVar(&atomBaseURLHost, "atom-baseurl-host", "http://localhost:32788/", "The host which is used to create the Atom BaseURL.") opts := zap.Options{ Development: true, @@ -100,7 +100,6 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - setupLog.Info("Checking atomBaseURLHost value: " + atomBaseURLHost) pdoknlv3.SetAtomBaseURLHost(atomBaseURLHost) // if the enable-http2 flag is false (the default), http/2 should be disabled diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 14b6656..50c7cd4 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -54,7 +54,7 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 - - --atom-baseurl-host="https://service.pdok.nl/yaml/" + - --atom-baseurl-host=https://service.pdok.nl/ image: controller:latest name: manager ports: [] From 6b2c950df0188d63fa9966782eb90834d8399985 Mon Sep 17 00:00:00 2001 From: Damien Mulder Date: Thu, 20 Feb 2025 17:05:35 +0100 Subject: [PATCH 12/12] Fix typo --- api/v2beta1/atom_conversion.go | 4 ++-- config/samples/v3_atom.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/v2beta1/atom_conversion.go b/api/v2beta1/atom_conversion.go index 9dd2c5d..9c7ba17 100644 --- a/api/v2beta1/atom_conversion.go +++ b/api/v2beta1/atom_conversion.go @@ -61,7 +61,7 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { OwnerInfoRef: "pdok", ServiceMetadataLinks: pdoknlv3.MetadataLink{ MetadataIdentifier: src.Spec.Service.MetadataIdentifier, - Templates: []string{"csv", "opensearch", "html"}, + Templates: []string{"csw", "opensearch", "html"}, }, Rights: src.Spec.Service.Rights, } @@ -74,7 +74,7 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error { Subtitle: srcDataset.Subtitle, DatasetMetadataLinks: pdoknlv3.MetadataLink{ MetadataIdentifier: srcDataset.MetadataIdentifier, - Templates: []string{"csv", "html"}, + Templates: []string{"csw", "html"}, }, SpatialDatasetIdentifierCode: srcDataset.SourceIdentifier, SpatialDatasetIdentifierNamespace: "http://www.pdok.nl", diff --git a/config/samples/v3_atom.yaml b/config/samples/v3_atom.yaml index eefeb77..12772dd 100644 --- a/config/samples/v3_atom.yaml +++ b/config/samples/v3_atom.yaml @@ -21,7 +21,7 @@ spec: serviceMetadataLinks: metadataIdentifier: 9615cd15-631b-45f8-a22a-d9d9c48cf211 templates: - - csv + - csw - opensearch - html rights: https://creativecommons.org/publicdomain/zero/1.0/deed.nl @@ -32,7 +32,7 @@ spec: datasetMetadataLinks: metadataIdentifier: 9615cd15-631b-45f8-a22a-d9d9c48cf211 templates: - - csv + - csw - html author: name: owner @@ -59,7 +59,7 @@ spec: datasetMetadataLinks: metadataIdentifier: 9615cd15-631b-45f8-a22a-d9d9c48cf211 templates: - - csv + - csw - html links: - href: https://www.eionet.europa.eu/reportnet/docs/noise/guidelines/geopackage-encoding-rule-end.pdf