Skip to content

Commit

Permalink
Periodic Sync to periodically calculate and patch storagequota Reserv…
Browse files Browse the repository at this point in the history
…ed values (#3075)
  • Loading branch information
nikhilbarge authored Oct 11, 2024
1 parent 4d85dd7 commit f98a445
Show file tree
Hide file tree
Showing 9 changed files with 688 additions and 7 deletions.
7 changes: 5 additions & 2 deletions cmd/syncer/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,10 @@ var (
"Name of the feature state switch configmap in supervisor cluster")
supervisorFSSNamespace = flag.String("supervisor-fss-namespace", "",
"Namespace of the feature state switch configmap in supervisor cluster")
internalFSSName = flag.String("fss-name", "", "Name of the feature state switch configmap")
internalFSSNamespace = flag.String("fss-namespace", "", "Namespace of the feature state switch configmap")
internalFSSName = flag.String("fss-name", "", "Name of the feature state switch configmap")
internalFSSNamespace = flag.String("fss-namespace", "", "Namespace of the feature state switch configmap")
periodicSyncIntervalInMin = flag.Duration("storagequota-sync-interval", 30*time.Minute,
"Periodic sync interval in Minutes")
)

// main for vsphere syncer.
Expand Down Expand Up @@ -356,6 +358,7 @@ func initSyncerComponents(ctx context.Context, clusterFlavor cnstypes.CnsCluster
os.Exit(0)
}
}()
syncer.PeriodicSyncIntervalInMin = *periodicSyncIntervalInMin
if err := syncer.InitMetadataSyncer(ctx, clusterFlavor, configInfo); err != nil {
log.Errorf("Error initializing Metadata Syncer. Error: %+v", err)
utils.LogoutAllvCenterSessions(ctx)
Expand Down
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/evanphx/json-patch/v5 v5.6.0
github.com/fsnotify/fsnotify v1.7.0
github.com/go-co-op/gocron v1.37.0
github.com/go-logr/zapr v1.2.4
github.com/golang/protobuf v1.5.4
github.com/google/uuid v1.6.0
Expand Down Expand Up @@ -129,6 +130,7 @@ require (
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
Expand All @@ -155,6 +157,7 @@ require (
go.opentelemetry.io/otel/trace v1.21.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
Expand Down
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0=
github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
Expand Down Expand Up @@ -344,6 +346,7 @@ github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
Expand Down Expand Up @@ -413,6 +416,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
Expand Down Expand Up @@ -491,6 +495,7 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand Down Expand Up @@ -534,9 +539,13 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
Expand Down Expand Up @@ -589,6 +598,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
Expand Down Expand Up @@ -678,6 +688,8 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0H
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
Expand Down
7 changes: 7 additions & 0 deletions manifests/supervisorcluster/1.30/cns-csi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ rules:
- apiGroups: ["cns.vmware.com"]
resources: ["storagepolicyusages/status"]
verbs: ["update", "patch"]
- apiGroups: ["cns.vmware.com"]
resources: ["storagequotaperiodicsyncs"]
verbs: ["create", "get", "list", "patch", "delete" ,"watch"]
- apiGroups: ["cns.vmware.com"]
resources: ["storagequotaperiodicsyncs/status"]
verbs: ["update", "patch"]
- apiGroups: ["cns.vmware.com"]
resources: ["storagepolicyquotas"]
verbs: ["get", "list", "watch"]
Expand Down Expand Up @@ -377,6 +383,7 @@ spec:
- "--leader-election-lease-duration=120s"
- "--leader-election-renew-deadline=60s"
- "--leader-election-retry-period=30s"
- "--storagequota-sync-interval=30m"
env:
- name: CLUSTER_FLAVOR
value: "WORKLOAD"
Expand Down
11 changes: 11 additions & 0 deletions pkg/apis/cnsoperator/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
cnsvolumemetadatav1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/cnsvolumemetadata/v1alpha1"
storagepolicyv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/storagepolicy/v1alpha1"
storagepolicyv1alpha2 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/storagepolicy/v1alpha2"
storagequotaperiodicsyncv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/storagequotaperiodicsync/v1alpha1"
)

// GroupName represents the group for cns operator apis
Expand Down Expand Up @@ -125,21 +126,30 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&storagepolicyv1alpha1.StoragePolicyQuotaList{},
)

scheme.AddKnownTypes(
SchemeGroupVersion,
&storagequotaperiodicsyncv1alpha1.StorageQuotaPeriodicSync{},
&storagequotaperiodicsyncv1alpha1.StorageQuotaPeriodicSyncList{},
)

scheme.AddKnownTypes(
SchemeGroupVersion,
&storagepolicyv1alpha1.StoragePolicyUsage{},
&storagepolicyv1alpha1.StoragePolicyUsageList{},
)

scheme.AddKnownTypes(
SchemeGroupVersionV2,
&storagepolicyv1alpha2.StoragePolicyUsage{},
&storagepolicyv1alpha2.StoragePolicyUsageList{},
)

scheme.AddKnownTypes(
SchemeGroupVersionV2,
&storagepolicyv1alpha2.StoragePolicyQuota{},
&storagepolicyv1alpha2.StoragePolicyQuotaList{},
)

scheme.AddKnownTypes(
SchemeGroupVersion,
&metav1.Status{},
Expand All @@ -149,6 +159,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
scheme,
SchemeGroupVersion,
)

metav1.AddToGroupVersion(
scheme,
SchemeGroupVersionV2,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright (c) 2024 VMware, Inc. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

// Package v1alpha1 contains API Schema definitions for the cns v1alpha1 API group
// +kubebuilder:object:generate=true
// +groupName=cns.vmware.com
package v1alpha1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)

var (
// GroupVersion is group version used to register these objects
GroupVersion = schema.GroupVersion{Group: "cns.vmware.com", Version: "v1alpha1"}

// schemeBuilder is used to add go types to the GroupVersionKind scheme.
schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)

// AddToScheme adds the types in this group-version to the given scheme.
AddToScheme = schemeBuilder.AddToScheme

objectTypes = []runtime.Object{}
)

func addKnownTypes(scheme *runtime.Scheme) error {
scheme.AddKnownTypes(GroupVersion, objectTypes...)
metav1.AddToGroupVersion(scheme, GroupVersion)
return nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
// Copyright (c) 2024 VMware, Inc. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package v1alpha1

import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// StorageQuotaPeriodicSyncSpec defines the desired state of StorageQuotaPeriodicSync
type StorageQuotaPeriodicSyncSpec struct {
// SyncIntervalInMinutes is the time in minutes after which StorageQuota CR will be
// synced periodically
SyncIntervalInMinutes int `json:"syncIntervalInMinutes,omitempty"`
}

// StorageQuotaPeriodicSyncStatus defines the observed state of StorageQuotaPeriodicSync
type StorageQuotaPeriodicSyncStatus struct {
// LastSyncTimestamp defines the timestamp when last sync operation executed
LastSyncTimestamp metav1.Time `json:"lastSyncTimestamp,omitempty"`

// ExpectedReservedValues indicates the expected "reserved" value for each namespace
// during each execution cycle
ExpectedReservedValues []ExpectedReservedValues `json:"expectedReservedValues,omitempty"`
}

type ExpectedReservedValues struct {
// Namespace is the namespace for which reserved values are calculated
Namespace string `json:"namespace,omitempty"`

// Reserved indicates the map of storagePolicyId to expected reserved value for that policy.
// It is calculated by combining the sizes of pending PVC objects, pending PVC expands and not ready
// VolumeSnapshots etc. for each storagePolicyId.
Reserved map[string]*resource.Quantity `json:"reserved,omitempty"`
}

//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
//+kubebuilder:storageversion

// StorageQuotaPeriodicSync is the Schema for the storagequotaperiodicsync API
type StorageQuotaPeriodicSync struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Spec StorageQuotaPeriodicSyncSpec `json:"spec,omitempty"`
Status StorageQuotaPeriodicSyncStatus `json:"status,omitempty"`
}

//+kubebuilder:object:root=true

// StorageQuotaPeriodicSyncList contains a list of StorageQuotaPeriodicSync
type StorageQuotaPeriodicSyncList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []StorageQuotaPeriodicSync `json:"items"`
}

func init() {
objectTypes = append(objectTypes, &StorageQuotaPeriodicSync{}, &StorageQuotaPeriodicSyncList{})
}
Loading

0 comments on commit f98a445

Please sign in to comment.