Skip to content

Commit 9d9da64

Browse files
dli357dli357
authored
Merge pull request #73 from cdapio/feature/CDAP-18568-Custom-Volumes
[CDAP-18568] Support CustomVolumes and CustomVolumeMounts for CDAP System Services
2 parents 123c752 + f71a513 commit 9d9da64

20 files changed

+16113
-547
lines changed

‎api/v1alpha1/cdapmaster_types.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,12 @@ type CDAPMasterSpec struct {
9393
Authentication *AuthenticationSpec `json:"authentication,omitempty"`
9494
// SecurityContext defines the security context for all pods for all services.
9595
SecurityContext *SecurityContext `json:"securityContext,omitempty"`
96+
// AdditionalVolumes defines a list of additional volumes for all services.
97+
// For information on supported volume types, see https://kubernetes.io/docs/concepts/storage/volumes/.
98+
AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
99+
// AdditionalVolumeMounts defines a list of additional volume mounts for all services.
100+
// For information on suported volume mount types, see https://kubernetes.io/docs/concepts/storage/volumes/.
101+
AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty"`
96102
}
97103

98104
// CDAPServiceSpec defines the base set of specifications applicable to all master services.
@@ -122,6 +128,12 @@ type CDAPServiceSpec struct {
122128
// Key is the secret object name. Value is the mount path.
123129
// This adds Secret data to the directory specified by the volume mount path.
124130
SecretVolumes map[string]string `json:"secretVolumes,omitempty"`
131+
// AdditionalVolumes defines a list of additional volumes to mount to the service.
132+
// For information on supported volume types, see https://kubernetes.io/docs/concepts/storage/volumes/.
133+
AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
134+
// AdditionalVolumeMounts defines a list of additional volume mounts for the service.
135+
// For information on suported volume mount types, see https://kubernetes.io/docs/concepts/storage/volumes/.
136+
AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty"`
125137
// SecurityContext overrides the security context for the service pods.
126138
SecurityContext *SecurityContext `json:"securityContext,omitempty"`
127139
}

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 28 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/cdap.cdap.io_cdapmasters.yaml‎

Lines changed: 15479 additions & 410 deletions
Large diffs are not rendered by default.

‎config/rbac/role.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
12
---
23
apiVersion: rbac.authorization.k8s.io/v1
34
kind: ClusterRole

‎controllers/deployment.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,12 @@ func buildStatefulSets(master *v1alpha1.CDAPMaster, name string, services Servic
199199
if _, err := spec.addSecretVolumes(ss.SecretVolumes); err != nil {
200200
return nil, err
201201
}
202+
if _, err := spec.addAdditionalVolumes(ss.AdditionalVolumes); err != nil {
203+
return nil, err
204+
}
205+
if _, err := spec.addAdditionalVolumeMounts(ss.AdditionalVolumeMounts); err != nil {
206+
return nil, err
207+
}
202208
}
203209

204210
// All services are optional services and are disabled in CR.
@@ -284,6 +290,12 @@ func buildDeployment(master *v1alpha1.CDAPMaster, name string, services ServiceG
284290
if _, err := spec.addSecretVolumes(ss.SecretVolumes); err != nil {
285291
return nil, err
286292
}
293+
if _, err := spec.addAdditionalVolumes(ss.AdditionalVolumes); err != nil {
294+
return nil, err
295+
}
296+
if _, err := spec.addAdditionalVolumeMounts(ss.AdditionalVolumeMounts); err != nil {
297+
return nil, err
298+
}
287299
}
288300
// All services are optional services and are disabled in CR.
289301
// Return nil to indicate no deployment is built.
@@ -327,6 +339,28 @@ func buildStatefulSetsObject(spec *StatefulSpec) (*reconciler.Object, error) {
327339
if err != nil {
328340
return nil, err
329341
}
342+
// For custom volumes and custom volume mounts, we directly pass structs from the spec to bypass the YAML templating logic.
343+
k8sObj, ok := obj.Obj.(*k8s.Object)
344+
if !ok {
345+
return nil, fmt.Errorf("failed to convert object to k8s object")
346+
}
347+
statefulSetObj, ok := k8sObj.Obj.(*appsv1.StatefulSet)
348+
if !ok {
349+
return nil, fmt.Errorf("failed to convert meta object to statefulset object")
350+
}
351+
if err := addVolumeToPodSpec(&statefulSetObj.Spec.Template.Spec, spec.Base.AdditionalVolumes); err != nil {
352+
return nil, err
353+
}
354+
for index, _ := range statefulSetObj.Spec.Template.Spec.InitContainers {
355+
if err := addVolumeMountToContainer(&statefulSetObj.Spec.Template.Spec.InitContainers[index], spec.Base.AdditionalVolumeMounts); err != nil {
356+
return nil, err
357+
}
358+
}
359+
for index, _ := range statefulSetObj.Spec.Template.Spec.Containers {
360+
if err := addVolumeMountToContainer(&statefulSetObj.Spec.Template.Spec.Containers[index], spec.Base.AdditionalVolumeMounts); err != nil {
361+
return nil, err
362+
}
363+
}
330364
return obj, nil
331365
}
332366

@@ -336,9 +370,55 @@ func buildDeploymentObject(spec *DeploymentSpec) (*reconciler.Object, error) {
336370
if err != nil {
337371
return nil, err
338372
}
373+
// For custom volumes and volume mounts, we directly pass structs from the spec to bypass the YAML templating logic.
374+
k8sObj, ok := obj.Obj.(*k8s.Object)
375+
if !ok {
376+
return nil, fmt.Errorf("failed to convert object to k8s object")
377+
}
378+
deploymentObj, ok := k8sObj.Obj.(*appsv1.Deployment)
379+
if !ok {
380+
return nil, fmt.Errorf("failed to convert meta object to statefulset object")
381+
}
382+
if err := addVolumeToPodSpec(&deploymentObj.Spec.Template.Spec, spec.Base.AdditionalVolumes); err != nil {
383+
return nil, err
384+
}
385+
for index, _ := range deploymentObj.Spec.Template.Spec.InitContainers {
386+
if err := addVolumeMountToContainer(&deploymentObj.Spec.Template.Spec.InitContainers[index], spec.Base.AdditionalVolumeMounts); err != nil {
387+
return nil, err
388+
}
389+
}
390+
for index, _ := range deploymentObj.Spec.Template.Spec.Containers {
391+
if err := addVolumeMountToContainer(&deploymentObj.Spec.Template.Spec.Containers[index], spec.Base.AdditionalVolumeMounts); err != nil {
392+
return nil, err
393+
}
394+
}
339395
return obj, nil
340396
}
341397

398+
func addVolumeToPodSpec(podSpec *corev1.PodSpec, volumesToAdd []corev1.Volume) error {
399+
for _, volumeToAdd := range volumesToAdd {
400+
for _, volume := range podSpec.Volumes {
401+
if volume.Name == volumeToAdd.Name {
402+
return fmt.Errorf("failed to add custom volume %q to pod spec: already exists", volumeToAdd.Name)
403+
}
404+
}
405+
}
406+
podSpec.Volumes = append(podSpec.Volumes, volumesToAdd...)
407+
return nil
408+
}
409+
410+
func addVolumeMountToContainer(container *corev1.Container, volumeMountsToAdd []corev1.VolumeMount) error {
411+
for _, volumeMountToAdd := range volumeMountsToAdd {
412+
for _, volumeMount := range container.VolumeMounts {
413+
if volumeMount.Name == volumeMountToAdd.Name {
414+
return fmt.Errorf("failed to mount custom volume %q to container %q at path %q: already mounted", volumeMountToAdd.Name, container.Name, volumeMountToAdd.MountPath)
415+
}
416+
}
417+
}
418+
container.VolumeMounts = append(container.VolumeMounts, volumeMountsToAdd...)
419+
return nil
420+
}
421+
342422
// Return a NodePort service to expose the supplied target service
343423
func buildNetworkService(master *v1alpha1.CDAPMaster, name NetworkServiceName, target ServiceName, labels map[string]string) (*NetworkServiceSpec, error) {
344424
s, err := getCDAPExternalServiceSpec(master, target)

‎controllers/spec.go‎

Lines changed: 72 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -122,22 +122,24 @@ func (s *ContainerSpec) setResources(resources *corev1.ResourceRequirements) *Co
122122

123123
// BaseSpec contains command fields for both StatefulSet and Deployment
124124
type BaseSpec struct {
125-
Name string `json:"name,omitempty"`
126-
Namespace string `json:"namespace,omitempty"`
127-
Labels map[string]string `json:"labels,omitempty"`
128-
ServiceAccountName string `json:"serviceAccountName,omitempty"`
129-
Replicas int32 `json:"replicas,omitempty"`
130-
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
131-
RuntimeClassName string `json:"runtimeClassName,omitempty"`
132-
PriorityClassName string `json:"priorityClassName,omitempty"`
133-
SecuritySecret string `json:"securitySecret,omitempty"`
134-
SecuritySecretPath string `json:"securitySecretPath,omitempty"`
135-
CConf string `json:"cdapConf,omitempty"`
136-
HConf string `json:"hadoopConf,omitempty"`
137-
SysAppConf string `json:"sysAppConf,omitempty"`
138-
ConfigMapVolumes map[string]string `json:"configMapVolumes,omitempty"`
139-
SecretVolumes map[string]string `json:"secretVolumes,omitempty"`
140-
SecurityContext *v1alpha1.SecurityContext `json:"securityContext,omitempty"`
125+
Name string `json:"name,omitempty"`
126+
Namespace string `json:"namespace,omitempty"`
127+
Labels map[string]string `json:"labels,omitempty"`
128+
ServiceAccountName string `json:"serviceAccountName,omitempty"`
129+
Replicas int32 `json:"replicas,omitempty"`
130+
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
131+
RuntimeClassName string `json:"runtimeClassName,omitempty"`
132+
PriorityClassName string `json:"priorityClassName,omitempty"`
133+
SecuritySecret string `json:"securitySecret,omitempty"`
134+
SecuritySecretPath string `json:"securitySecretPath,omitempty"`
135+
CConf string `json:"cdapConf,omitempty"`
136+
HConf string `json:"hadoopConf,omitempty"`
137+
SysAppConf string `json:"sysAppConf,omitempty"`
138+
ConfigMapVolumes map[string]string `json:"configMapVolumes,omitempty"`
139+
SecretVolumes map[string]string `json:"secretVolumes,omitempty"`
140+
AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
141+
AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty"`
142+
SecurityContext *v1alpha1.SecurityContext `json:"securityContext,omitempty"`
141143
}
142144

143145
func newBaseSpec(master *v1alpha1.CDAPMaster, name string, labels map[string]string, cconf, hconf, sysappconf string) *BaseSpec {
@@ -156,7 +158,8 @@ func newBaseSpec(master *v1alpha1.CDAPMaster, name string, labels map[string]str
156158
s.SysAppConf = sysappconf
157159
s.ConfigMapVolumes = cloneMap(master.Spec.ConfigMapVolumes)
158160
s.SecretVolumes = cloneMap(master.Spec.SecretVolumes)
159-
161+
s.AdditionalVolumes = master.Spec.AdditionalVolumes
162+
s.AdditionalVolumeMounts = master.Spec.AdditionalVolumeMounts
160163
return s
161164
}
162165

@@ -218,6 +221,30 @@ func addVolumes(volumes, newVolumes map[string]string, typeName string) error {
218221
return nil
219222
}
220223

224+
func (s *BaseSpec) addAdditionalVolumes(additionalVolumes []corev1.Volume) (*BaseSpec, error) {
225+
for _, additionalVolume := range additionalVolumes {
226+
for _, specVolume := range s.AdditionalVolumes {
227+
if specVolume.Name == additionalVolume.Name {
228+
return nil, fmt.Errorf("failed to add custom volume %q due to already mounted as %+v", additionalVolume.Name, specVolume)
229+
}
230+
}
231+
s.AdditionalVolumes = append(s.AdditionalVolumes, additionalVolume)
232+
}
233+
return s, nil
234+
}
235+
236+
func (s *BaseSpec) addAdditionalVolumeMounts(additionalVolumeMounts []corev1.VolumeMount) (*BaseSpec, error) {
237+
for _, additionalVolumeMount := range additionalVolumeMounts {
238+
for _, specVolumeMount := range s.AdditionalVolumeMounts {
239+
if specVolumeMount.Name == additionalVolumeMount.Name {
240+
return nil, fmt.Errorf("failed to mount custom volume %q at path %q due to already mounted as %+v", additionalVolumeMount.Name, additionalVolumeMount.MountPath, specVolumeMount)
241+
}
242+
}
243+
s.AdditionalVolumeMounts = append(s.AdditionalVolumeMounts, additionalVolumeMount)
244+
}
245+
return s, nil
246+
}
247+
221248
func (s *BaseSpec) setSecurityContext(securityContext *v1alpha1.SecurityContext) *BaseSpec {
222249
s.SecurityContext = securityContext
223250
if securityContext != nil {
@@ -306,6 +333,20 @@ func (s *DeploymentSpec) addSecretVolumes(volumes map[string]string) (*Deploymen
306333
return s, nil
307334
}
308335

336+
func (s *DeploymentSpec) addAdditionalVolumes(additionalVolumes []corev1.Volume) (*DeploymentSpec, error) {
337+
if _, err := s.Base.addAdditionalVolumes(additionalVolumes); err != nil {
338+
return nil, err
339+
}
340+
return s, nil
341+
}
342+
343+
func (s *DeploymentSpec) addAdditionalVolumeMounts(additionalVolumeMounts []corev1.VolumeMount) (*DeploymentSpec, error) {
344+
if _, err := s.Base.addAdditionalVolumeMounts(additionalVolumeMounts); err != nil {
345+
return nil, err
346+
}
347+
return s, nil
348+
}
349+
309350
func (s *DeploymentSpec) setSecurityContext(securityContext *v1alpha1.SecurityContext) *DeploymentSpec {
310351
s.Base.setSecurityContext(securityContext)
311352
return s
@@ -397,6 +438,20 @@ func (s *StatefulSpec) addSecretVolumes(volumes map[string]string) (*StatefulSpe
397438
return s, nil
398439
}
399440

441+
func (s *StatefulSpec) addAdditionalVolumes(additionalVolumes []corev1.Volume) (*StatefulSpec, error) {
442+
if _, err := s.Base.addAdditionalVolumes(additionalVolumes); err != nil {
443+
return nil, err
444+
}
445+
return s, nil
446+
}
447+
448+
func (s *StatefulSpec) addAdditionalVolumeMounts(additionalVolumeMounts []corev1.VolumeMount) (*StatefulSpec, error) {
449+
if _, err := s.Base.addAdditionalVolumeMounts(additionalVolumeMounts); err != nil {
450+
return nil, err
451+
}
452+
return s, nil
453+
}
454+
400455
func (s *StatefulSpec) setSecurityContext(securityContext *v1alpha1.SecurityContext) *StatefulSpec {
401456
s.Base.setSecurityContext(securityContext)
402457
return s

‎controllers/testdata/appfabric.json‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,15 @@
136136
{
137137
"mountPath": "/my/secret/1",
138138
"name": "cdap-se-vol-my-secret-1"
139+
},
140+
{
141+
"mountPath": "/mnt/test",
142+
"name": "test-volume",
143+
"readOnly": true
144+
},
145+
{
146+
"mountPath": "/mnt/teststorage",
147+
"name": "test-persistent-storage"
139148
}
140149
]
141150
}
@@ -179,16 +188,13 @@
179188
"mountPath": "/etc/cdap/security"
180189
},
181190
{
182-
"name": "cdap-cm-vol-my-config-map-1",
183-
"mountPath": "/my/config/map/1"
184-
},
185-
{
186-
"mountPath": "/my/config/map/2",
187-
"name": "cdap-cm-vol-my-config-map-2"
191+
"mountPath": "/mnt/test",
192+
"name": "test-volume",
193+
"readOnly": true
188194
},
189195
{
190-
"mountPath": "/my/secret/1",
191-
"name": "cdap-se-vol-my-secret-1"
196+
"mountPath": "/mnt/teststorage",
197+
"name": "test-persistent-storage"
192198
}
193199
]
194200
}
@@ -283,6 +289,39 @@
283289
"defaultMode": 420,
284290
"secretName": "my-secret-1"
285291
}
292+
},
293+
{
294+
"name": "test-volume",
295+
"projected": {
296+
"defaultMode": 420,
297+
"sources": [
298+
{
299+
"serviceAccountToken": {
300+
"audience": "test-aud",
301+
"expirationSeconds": 3600,
302+
"path": "token"
303+
}
304+
},
305+
{
306+
"configMap": {
307+
"name": "test-projected-config",
308+
"items": [
309+
{
310+
"key": "config",
311+
"path": "test-projected-config"
312+
}
313+
],
314+
"optional": false
315+
}
316+
}
317+
]
318+
}
319+
},
320+
{
321+
"name": "test-persistent-storage",
322+
"persistentVolumeClaim": {
323+
"claimName": "test-persistent-volume-claim"
324+
}
286325
}
287326
]
288327
}

0 commit comments

Comments
 (0)