Skip to content

Commit ddb9ffd

Browse files
MaxFedotovMaksim Fedotov
MaxFedotov
and
Maksim Fedotov
authored
refactor: split tenant controller to separate files
Co-authored-by: Maksim Fedotov <[email protected]>
1 parent cae65c9 commit ddb9ffd

File tree

9 files changed

+948
-868
lines changed

9 files changed

+948
-868
lines changed

‎controllers/tenant/limitranges.go

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package tenant
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strconv"
7+
8+
"golang.org/x/sync/errgroup"
9+
corev1 "k8s.io/api/core/v1"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
12+
13+
capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
14+
)
15+
16+
// Ensuring all the LimitRange are applied to each Namespace handled by the Tenant.
17+
func (r *Manager) syncLimitRanges(tenant *capsulev1beta1.Tenant) error {
18+
// getting requested LimitRange keys
19+
keys := make([]string, 0, len(tenant.Spec.LimitRanges.Items))
20+
21+
for i := range tenant.Spec.LimitRanges.Items {
22+
keys = append(keys, strconv.Itoa(i))
23+
}
24+
25+
group := new(errgroup.Group)
26+
27+
for _, ns := range tenant.Status.Namespaces {
28+
namespace := ns
29+
30+
group.Go(func() error {
31+
return r.syncLimitRange(tenant, namespace, keys)
32+
})
33+
}
34+
35+
return group.Wait()
36+
}
37+
38+
func (r *Manager) syncLimitRange(tenant *capsulev1beta1.Tenant, namespace string, keys []string) (err error) {
39+
// getting LimitRange labels for the mutateFn
40+
var tenantLabel, limitRangeLabel string
41+
42+
if tenantLabel, err = capsulev1beta1.GetTypeLabel(&capsulev1beta1.Tenant{}); err != nil {
43+
return
44+
}
45+
if limitRangeLabel, err = capsulev1beta1.GetTypeLabel(&corev1.LimitRange{}); err != nil {
46+
return
47+
}
48+
49+
if err = r.pruningResources(namespace, keys, &corev1.LimitRange{}); err != nil {
50+
return
51+
}
52+
53+
for i, spec := range tenant.Spec.LimitRanges.Items {
54+
target := &corev1.LimitRange{
55+
ObjectMeta: metav1.ObjectMeta{
56+
Name: fmt.Sprintf("capsule-%s-%d", tenant.Name, i),
57+
Namespace: namespace,
58+
},
59+
}
60+
61+
var res controllerutil.OperationResult
62+
res, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, target, func() (err error) {
63+
target.ObjectMeta.Labels = map[string]string{
64+
tenantLabel: tenant.Name,
65+
limitRangeLabel: strconv.Itoa(i),
66+
}
67+
target.Spec = spec
68+
return controllerutil.SetControllerReference(tenant, target, r.Scheme)
69+
})
70+
71+
r.emitEvent(tenant, target.GetNamespace(), res, fmt.Sprintf("Ensuring LimitRange %s", target.GetName()), err)
72+
73+
r.Log.Info("LimitRange sync result: "+string(res), "name", target.Name, "namespace", target.Namespace)
74+
if err != nil {
75+
return
76+
}
77+
}
78+
79+
return
80+
}

‎controllers/tenant/manager.go

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package tenant
2+
3+
import (
4+
"context"
5+
6+
"github.com/go-logr/logr"
7+
corev1 "k8s.io/api/core/v1"
8+
networkingv1 "k8s.io/api/networking/v1"
9+
rbacv1 "k8s.io/api/rbac/v1"
10+
"k8s.io/apimachinery/pkg/api/errors"
11+
"k8s.io/apimachinery/pkg/runtime"
12+
"k8s.io/client-go/tools/record"
13+
"k8s.io/client-go/util/retry"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
17+
18+
capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
19+
)
20+
21+
type Manager struct {
22+
client.Client
23+
Log logr.Logger
24+
Scheme *runtime.Scheme
25+
Recorder record.EventRecorder
26+
}
27+
28+
func (r *Manager) SetupWithManager(mgr ctrl.Manager) error {
29+
return ctrl.NewControllerManagedBy(mgr).
30+
For(&capsulev1beta1.Tenant{}).
31+
Owns(&corev1.Namespace{}).
32+
Owns(&networkingv1.NetworkPolicy{}).
33+
Owns(&corev1.LimitRange{}).
34+
Owns(&corev1.ResourceQuota{}).
35+
Owns(&rbacv1.RoleBinding{}).
36+
Complete(r)
37+
}
38+
39+
func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) {
40+
r.Log = r.Log.WithValues("Request.Name", request.Name)
41+
42+
// Fetch the Tenant instance
43+
instance := &capsulev1beta1.Tenant{}
44+
if err = r.Get(ctx, request.NamespacedName, instance); err != nil {
45+
if errors.IsNotFound(err) {
46+
r.Log.Info("Request object not found, could have been deleted after reconcile request")
47+
return reconcile.Result{}, nil
48+
}
49+
r.Log.Error(err, "Error reading the object")
50+
return
51+
}
52+
// Ensuring the Tenant Status
53+
if err = r.updateTenantStatus(instance); err != nil {
54+
r.Log.Error(err, "Cannot update Tenant status")
55+
return
56+
}
57+
58+
// Ensuring all namespaces are collected
59+
r.Log.Info("Ensuring all Namespaces are collected")
60+
if err = r.collectNamespaces(instance); err != nil {
61+
r.Log.Error(err, "Cannot collect Namespace resources")
62+
return
63+
}
64+
65+
r.Log.Info("Starting processing of Namespaces", "items", len(instance.Status.Namespaces))
66+
if err = r.syncNamespaces(instance); err != nil {
67+
r.Log.Error(err, "Cannot sync Namespace items")
68+
return
69+
}
70+
71+
if instance.Spec.NetworkPolicies != nil {
72+
r.Log.Info("Starting processing of Network Policies", "items", len(instance.Spec.NetworkPolicies.Items))
73+
if err = r.syncNetworkPolicies(instance); err != nil {
74+
r.Log.Error(err, "Cannot sync NetworkPolicy items")
75+
return
76+
}
77+
}
78+
79+
if instance.Spec.LimitRanges != nil {
80+
r.Log.Info("Starting processing of Limit Ranges", "items", len(instance.Spec.LimitRanges.Items))
81+
if err = r.syncLimitRanges(instance); err != nil {
82+
r.Log.Error(err, "Cannot sync LimitRange items")
83+
return
84+
}
85+
}
86+
87+
if instance.Spec.ResourceQuota != nil {
88+
r.Log.Info("Starting processing of Resource Quotas", "items", len(instance.Spec.ResourceQuota.Items))
89+
if err = r.syncResourceQuotas(instance); err != nil {
90+
r.Log.Error(err, "Cannot sync ResourceQuota items")
91+
return
92+
}
93+
}
94+
95+
r.Log.Info("Ensuring additional RoleBindings for owner")
96+
if err = r.syncAdditionalRoleBindings(instance); err != nil {
97+
r.Log.Error(err, "Cannot sync additional RoleBindings items")
98+
return
99+
}
100+
101+
r.Log.Info("Ensuring RoleBinding for owner")
102+
if err = r.ownerRoleBinding(instance); err != nil {
103+
r.Log.Error(err, "Cannot sync owner RoleBinding")
104+
return
105+
}
106+
107+
r.Log.Info("Ensuring Namespace count")
108+
if err = r.ensureNamespaceCount(instance); err != nil {
109+
r.Log.Error(err, "Cannot sync Namespace count")
110+
return
111+
}
112+
113+
r.Log.Info("Tenant reconciling completed")
114+
return ctrl.Result{}, err
115+
}
116+
117+
func (r *Manager) updateTenantStatus(tnt *capsulev1beta1.Tenant) error {
118+
return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
119+
if tnt.IsCordoned() {
120+
tnt.Status.State = capsulev1beta1.TenantStateCordoned
121+
} else {
122+
tnt.Status.State = capsulev1beta1.TenantStateActive
123+
}
124+
125+
return r.Client.Status().Update(context.Background(), tnt)
126+
})
127+
}

‎controllers/tenant/namespaces.go

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package tenant
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
"golang.org/x/sync/errgroup"
9+
corev1 "k8s.io/api/core/v1"
10+
"k8s.io/apimachinery/pkg/fields"
11+
"k8s.io/apimachinery/pkg/types"
12+
"k8s.io/client-go/util/retry"
13+
"sigs.k8s.io/controller-runtime/pkg/client"
14+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
15+
16+
capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
17+
)
18+
19+
// Ensuring all annotations are applied to each Namespace handled by the Tenant.
20+
func (r *Manager) syncNamespaces(tenant *capsulev1beta1.Tenant) (err error) {
21+
group := new(errgroup.Group)
22+
23+
for _, item := range tenant.Status.Namespaces {
24+
namespace := item
25+
26+
group.Go(func() error {
27+
return r.syncNamespaceMetadata(namespace, tenant)
28+
})
29+
}
30+
31+
if err = group.Wait(); err != nil {
32+
r.Log.Error(err, "Cannot sync Namespaces")
33+
34+
err = fmt.Errorf("cannot sync Namespaces: %s", err.Error())
35+
}
36+
return
37+
}
38+
39+
func (r *Manager) syncNamespaceMetadata(namespace string, tnt *capsulev1beta1.Tenant) (err error) {
40+
var res controllerutil.OperationResult
41+
42+
err = retry.RetryOnConflict(retry.DefaultBackoff, func() (conflictErr error) {
43+
ns := &corev1.Namespace{}
44+
if conflictErr = r.Client.Get(context.TODO(), types.NamespacedName{Name: namespace}, ns); err != nil {
45+
return
46+
}
47+
48+
capsuleLabel, _ := capsulev1beta1.GetTypeLabel(&capsulev1beta1.Tenant{})
49+
50+
res, conflictErr = controllerutil.CreateOrUpdate(context.TODO(), r.Client, ns, func() error {
51+
annotations := make(map[string]string)
52+
53+
if tnt.Spec.NamespacesMetadata != nil {
54+
for k, v := range tnt.Spec.NamespacesMetadata.AdditionalAnnotations {
55+
annotations[k] = v
56+
}
57+
}
58+
59+
if tnt.Spec.NodeSelector != nil {
60+
var selector []string
61+
for k, v := range tnt.Spec.NodeSelector {
62+
selector = append(selector, fmt.Sprintf("%s=%s", k, v))
63+
}
64+
annotations["scheduler.alpha.kubernetes.io/node-selector"] = strings.Join(selector, ",")
65+
}
66+
67+
if tnt.Spec.IngressClasses != nil {
68+
if len(tnt.Spec.IngressClasses.Exact) > 0 {
69+
annotations[capsulev1beta1.AvailableIngressClassesAnnotation] = strings.Join(tnt.Spec.IngressClasses.Exact, ",")
70+
}
71+
if len(tnt.Spec.IngressClasses.Regex) > 0 {
72+
annotations[capsulev1beta1.AvailableIngressClassesRegexpAnnotation] = tnt.Spec.IngressClasses.Regex
73+
}
74+
}
75+
76+
if tnt.Spec.StorageClasses != nil {
77+
if len(tnt.Spec.StorageClasses.Exact) > 0 {
78+
annotations[capsulev1beta1.AvailableStorageClassesAnnotation] = strings.Join(tnt.Spec.StorageClasses.Exact, ",")
79+
}
80+
if len(tnt.Spec.StorageClasses.Regex) > 0 {
81+
annotations[capsulev1beta1.AvailableStorageClassesRegexpAnnotation] = tnt.Spec.StorageClasses.Regex
82+
}
83+
}
84+
85+
if tnt.Spec.ContainerRegistries != nil {
86+
if len(tnt.Spec.ContainerRegistries.Exact) > 0 {
87+
annotations[capsulev1beta1.AllowedRegistriesAnnotation] = strings.Join(tnt.Spec.ContainerRegistries.Exact, ",")
88+
}
89+
if len(tnt.Spec.ContainerRegistries.Regex) > 0 {
90+
annotations[capsulev1beta1.AllowedRegistriesRegexpAnnotation] = tnt.Spec.ContainerRegistries.Regex
91+
}
92+
}
93+
94+
ns.SetAnnotations(annotations)
95+
96+
newLabels := map[string]string{
97+
"name": namespace,
98+
capsuleLabel: tnt.GetName(),
99+
}
100+
101+
if tnt.Spec.NamespacesMetadata != nil {
102+
for k, v := range tnt.Spec.NamespacesMetadata.AdditionalLabels {
103+
newLabels[k] = v
104+
}
105+
}
106+
107+
ns.SetLabels(newLabels)
108+
109+
return nil
110+
})
111+
112+
return
113+
})
114+
115+
r.emitEvent(tnt, namespace, res, "Ensuring Namespace metadata", err)
116+
117+
return
118+
}
119+
120+
func (r *Manager) ensureNamespaceCount(tenant *capsulev1beta1.Tenant) error {
121+
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
122+
tenant.Status.Size = uint(len(tenant.Status.Namespaces))
123+
124+
found := &capsulev1beta1.Tenant{}
125+
if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: tenant.GetName()}, found); err != nil {
126+
return err
127+
}
128+
129+
found.Status.Size = tenant.Status.Size
130+
131+
return r.Client.Status().Update(context.TODO(), found, &client.UpdateOptions{})
132+
})
133+
}
134+
135+
func (r *Manager) collectNamespaces(tenant *capsulev1beta1.Tenant) error {
136+
return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
137+
list := &corev1.NamespaceList{}
138+
err = r.Client.List(context.TODO(), list, client.MatchingFieldsSelector{
139+
Selector: fields.OneTermEqualSelector(".metadata.ownerReferences[*].capsule", tenant.GetName()),
140+
})
141+
142+
if err != nil {
143+
return
144+
}
145+
146+
_, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, tenant.DeepCopy(), func() error {
147+
tenant.AssignNamespaces(list.Items)
148+
149+
return r.Client.Status().Update(context.TODO(), tenant, &client.UpdateOptions{})
150+
})
151+
return
152+
})
153+
}

0 commit comments

Comments
 (0)