Skip to content

Commit

Permalink
Merge pull request #7 from PDOK/PDOK-17553-Atom-crd_converter-v2-v3
Browse files Browse the repository at this point in the history 
Pdok 17553 atom crd converter v2 v3
  • Loading branch information
@rahmabPublic
rahmabPublic authored Feb 21, 2025
2 parents 96044bd + 6b2c950 commit 1687dec
Show file tree
Hide file tree
Showing 12 changed files with 174 additions and 152 deletions.
66 changes: 21 additions & 45 deletions api/v2beta1/atom_conversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"fmt"
"log"
"strconv"
"strings"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -42,47 +43,44 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error {
log.Printf("ConvertTo: Converting Atom from Spoke version v2beta1 to Hub version v3;"+
"source: %s/%s, target: %s/%s", src.Namespace, src.Name, dst.Namespace, dst.Name)

host := "https://service.pdok.nl/" // Todo read from flag

// ObjectMeta
dst.ObjectMeta = src.ObjectMeta

// Lifecycle
log.Printf("Start mapping the Lifecycle specs...")
if src.Spec.Kubernetes != nil && src.Spec.Kubernetes.Lifecycle != nil && src.Spec.Kubernetes.Lifecycle.TTLInDays != nil {
dst.Spec.Lifecycle.TTLInDays = GetInt32Pointer(int32(*src.Spec.Kubernetes.Lifecycle.TTLInDays))
}
log.Printf("Done mapping the Lifecycle specs...")

// Service
log.Printf("Start mapping the Service...")
dst.Spec.Service = pdoknlv3.Service{
BaseURL: createBaseURL(host, src.Spec.General),
Lang: "nl",
Stylesheet: "https://service.pdok.nl/atom/style/style.xsl",
Title: src.Spec.Service.Title,
Subtitle: src.Spec.Service.Subtitle,
OwnerInfoRef: "pdok",
Links: []pdoknlv3.Link{},
ServiceMetadataLinks: []pdoknlv3.MetadataLink{}, // Todo
Rights: src.Spec.Service.Rights,
BaseURL: createBaseURL(pdoknlv3.GetAtomBaseURLHost(), src.Spec.General),
Lang: "nl",
Stylesheet: "https://service.pdok.nl/atom/style/style.xsl",
Title: src.Spec.Service.Title,
Subtitle: src.Spec.Service.Subtitle,
OwnerInfoRef: "pdok",
ServiceMetadataLinks: pdoknlv3.MetadataLink{
MetadataIdentifier: src.Spec.Service.MetadataIdentifier,
Templates: []string{"csw", "opensearch", "html"},
},
Rights: src.Spec.Service.Rights,
}
log.Printf("Done mapping the Service...")

dst.Spec.DatasetFeeds = []pdoknlv3.DatasetFeed{}
log.Printf("Start mapping the Datasets...")
for _, srcDataset := range src.Spec.Service.Datasets {
dstDatasetFeed := pdoknlv3.DatasetFeed{
TechnicalName: srcDataset.Name,
Title: srcDataset.Title,
Subtitle: srcDataset.Subtitle,
DatasetMetadataLinks: []pdoknlv3.MetadataLink{}, // Todo
TechnicalName: srcDataset.Name,
Title: srcDataset.Title,
Subtitle: srcDataset.Subtitle,
DatasetMetadataLinks: pdoknlv3.MetadataLink{
MetadataIdentifier: srcDataset.MetadataIdentifier,
Templates: []string{"csw", "html"},
},
SpatialDatasetIdentifierCode: srcDataset.SourceIdentifier,
SpatialDatasetIdentifierNamespace: "http://www.pdok.nl",
}

// Map the links
log.Printf("Start mapping the Links...")
for _, srcLink := range srcDataset.Links {
dstLink := pdoknlv3.Link{
Title: srcLink.Type,
Expand All @@ -97,10 +95,8 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error {

dstDatasetFeed.Links = append(dstDatasetFeed.Links, dstLink)
}
log.Printf("Done mapping the Links...")

// Map the entries
log.Printf("Start mapping the Entries...")
for _, srcDownload := range srcDataset.Downloads {
dstEntry := pdoknlv3.Entry{
TechnicalName: srcDownload.Name,
Expand Down Expand Up @@ -135,7 +131,6 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error {
}

// Map the links
log.Printf("Start mapping the DownloadLinks...")
for _, srcLink := range srcDownload.Links {
dstDownloadLink := pdoknlv3.DownloadLink{}

Expand All @@ -162,15 +157,12 @@ func (src *Atom) ConvertTo(dstRaw conversion.Hub) error {

dstEntry.DownloadLinks = append(dstEntry.DownloadLinks, dstDownloadLink)
}
log.Printf("Done mapping the DownloadLinks...")

dstDatasetFeed.Entries = append(dstDatasetFeed.Entries, dstEntry)
}
log.Printf("Done mapping the Entries...")

dst.Spec.DatasetFeeds = append(dst.Spec.DatasetFeeds, dstDatasetFeed)
}
log.Printf("Done mapping the Datasets...")

return nil
}
Expand All @@ -185,7 +177,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
dst.ObjectMeta = src.ObjectMeta

// General
log.Printf("Start mapping the General specs...")
dst.Spec.General = General{
Dataset: src.ObjectMeta.Labels["dataset"],
DatasetOwner: src.ObjectMeta.Labels["dataset-owner"],
Expand All @@ -202,10 +193,7 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
dst.Spec.General.Theme = &theme
}

log.Printf("Done mapping the General specs...")

// Service
log.Printf("Start mapping the Service...")
dst.Spec.Service = AtomService{
Title: src.Spec.Service.Title,
Subtitle: src.Spec.Service.Subtitle,
Expand All @@ -214,11 +202,10 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
Name: "PDOK Beheer",
Email: "[email protected]",
},
MetadataIdentifier: src.Spec.Service.ServiceMetadataLinks.MetadataIdentifier,
}
log.Printf("Done mapping the Service...")

// Datasets
log.Printf("Start mapping the Datasets...")
dst.Spec.Service.Datasets = []Dataset{}
for _, srcDatasetFeed := range src.Spec.DatasetFeeds {
dstDataset := Dataset{
Expand All @@ -229,7 +216,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
}

// Map the links
log.Printf("Start mapping the Links...")
for _, srcLink := range srcDatasetFeed.Links {
dstDataset.Links = append(dstDataset.Links, OtherLink{
Type: srcLink.Title,
Expand All @@ -238,7 +224,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
Language: &srcLink.Hreflang,
})
}
log.Printf("Done mapping the Links...")

if len(srcDatasetFeed.Entries) > 0 && srcDatasetFeed.Entries[0].Polygon != nil {
// We can assume all entries have the same bbox, so we take the first one
Expand All @@ -252,7 +237,6 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
}

// Map the downloads
log.Printf("Start mapping the Entries...")
for _, srcEntry := range srcDatasetFeed.Entries {
dstDownload := Download{
Name: srcEntry.TechnicalName,
Expand All @@ -273,9 +257,7 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
}

// Map the links
log.Printf("Start mapping the DownloadLinks...")
for _, srcDownloadLink := range srcEntry.DownloadLinks {

dstLink := Link{
BlobKey: &srcDownloadLink.Data,
Rel: &srcDownloadLink.Rel,
Expand All @@ -297,29 +279,23 @@ func (dst *Atom) ConvertFrom(srcRaw conversion.Hub) error {
}
}

log.Printf("Done mapping the DownloadLinks...")
dstDataset.Downloads = append(dstDataset.Downloads, dstDownload)
}
log.Printf("Done mapping the Entries...")
dst.Spec.Service.Datasets = append(dst.Spec.Service.Datasets, dstDataset)
}
log.Printf("Start mapping the Datasets...")

// Kubernetes
log.Printf("Start mapping the Kubernetes Specs...")
dst.Spec.Kubernetes = &Kubernetes{
Lifecycle: &Lifecycle{},
}
if src.Spec.Lifecycle.TTLInDays != nil {
dst.Spec.Kubernetes.Lifecycle.TTLInDays = GetIntPointer(int(*src.Spec.Lifecycle.TTLInDays))
}
log.Printf("Done mapping the Kubernetes Specs...")

return nil
}

func createBaseURL(host string, general General) (baseURL string) {

atomURI := fmt.Sprintf("%s/%s", general.DatasetOwner, general.Dataset)
if general.Theme != nil {
atomURI += fmt.Sprintf("/%s", *general.Theme)
Expand All @@ -330,7 +306,7 @@ func createBaseURL(host string, general General) (baseURL string) {
atomURI += fmt.Sprintf("/%s", *general.ServiceVersion)
}

baseURL = fmt.Sprintf("%s/%s/index.xml", host, atomURI)
baseURL = fmt.Sprintf("%s/%s/index.xml", strings.TrimSuffix(host, "/"), atomURI)
return
}

Expand Down
53 changes: 32 additions & 21 deletions api/v3/atom_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// BaseURLHost is accessed by other api versions (i.e. v2beta1)
var baseURLHost string

// AtomSpec defines the desired state of Atom.
type AtomSpec struct {
Lifecycle Lifecycle `json:"lifecycle,omitempty"`
Expand All @@ -42,15 +45,14 @@ type Lifecycle struct {

// Service defines the service configuration for the Atom feed
type Service struct {
BaseURL string `json:"baseUrl"`
Lang string `json:"lang,omitempty"`
Stylesheet string `json:"stylesheet,omitempty"`
Title string `json:"title"`
Subtitle string `json:"subtitle,omitempty"`
OwnerInfoRef string `json:"ownerInfoRef"`
ServiceMetadataLinks []MetadataLink `json:"serviceMetadataLinks,omitempty"`
Links []Link `json:"links,omitempty"` // Todo kan weg?
Rights string `json:"rights,omitempty"`
BaseURL string `json:"baseUrl"`
Lang string `json:"lang,omitempty"`
Stylesheet string `json:"stylesheet,omitempty"`
Title string `json:"title"`
Subtitle string `json:"subtitle,omitempty"`
OwnerInfoRef string `json:"ownerInfoRef"`
ServiceMetadataLinks MetadataLink `json:"serviceMetadataLinks,omitempty"`
Rights string `json:"rights,omitempty"`
}

// Link represents a link in the service or dataset feed
Expand All @@ -72,18 +74,18 @@ type Author struct {

// DatasetFeed represents individual dataset feeds within the Atom service
type DatasetFeed struct {
TechnicalName string `json:"technicalName"`
Title string `json:"title"`
Subtitle string `json:"subtitle,omitempty"`
Links []Link `json:"links,omitempty"` // Todo kan weg?
DatasetMetadataLinks []MetadataLink `json:"datasetMetadataLinks,omitempty"`
Author Author `json:"author,omitempty"`
SpatialDatasetIdentifierCode string `json:"spatial_dataset_identifier_code,omitempty"`
SpatialDatasetIdentifierNamespace string `json:"spatial_dataset_identifier_namespace,omitempty"`
Entries []Entry `json:"entries,omitempty"`
}

// Metadatalink represents a link in the service or dataset feed
TechnicalName string `json:"technicalName"`
Title string `json:"title"`
Subtitle string `json:"subtitle,omitempty"`
Links []Link `json:"links,omitempty"` // Todo kan weg?
DatasetMetadataLinks MetadataLink `json:"datasetMetadataLinks,omitempty"`
Author Author `json:"author,omitempty"`
SpatialDatasetIdentifierCode string `json:"spatial_dataset_identifier_code,omitempty"`
SpatialDatasetIdentifierNamespace string `json:"spatial_dataset_identifier_namespace,omitempty"`
Entries []Entry `json:"entries,omitempty"`
}

// MetadataLink represents a link in the service or dataset feed
type MetadataLink struct {
MetadataIdentifier string `json:"metadataIdentifier"`
Templates []string `json:"templates,omitempty"`
Expand Down Expand Up @@ -170,3 +172,12 @@ type AtomList struct {
func init() {
SchemeBuilder.Register(&Atom{}, &AtomList{})
}

// SetAtomBaseURLHost is used to set the BaseURL Host in main
func SetAtomBaseURLHost(atomBaseURLHost string) {
baseURLHost = atomBaseURLHost
}

func GetAtomBaseURLHost() string {
return baseURLHost
}
21 changes: 2 additions & 19 deletions api/v3/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

traefikiov1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
Expand Down Expand Up @@ -55,6 +56,8 @@ func init() {

utilruntime.Must(pdoknlv3.AddToScheme(scheme))
utilruntime.Must(pdoknlv2beta1.AddToScheme(scheme))
utilruntime.Must(traefikiov1alpha1.AddToScheme(scheme))

// +kubebuilder:scaffold:scheme
}

Expand All @@ -67,6 +70,7 @@ func main() {
var probeAddr string
var secureMetrics bool
var enableHTTP2 bool
var atomBaseURLHost string
var tlsOpts []func(*tls.Config)
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
Expand All @@ -85,6 +89,9 @@ func main() {
flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")

flag.StringVar(&atomBaseURLHost, "atom-baseurl-host", "http://localhost:32788/",
"The host which is used to create the Atom BaseURL.")
opts := zap.Options{
Development: true,
}
Expand All @@ -93,6 +100,8 @@ func main() {

ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))

pdoknlv3.SetAtomBaseURLHost(atomBaseURLHost)

// if the enable-http2 flag is false (the default), http/2 should be disabled
// due to its vulnerabilities. More specifically, disabling http/2 will
// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
Expand Down
Loading

0 comments on commit 1687dec

Please sign in to comment.