Making backups work with Solr 8.6+

Solr 8.6 introduced a new security feature, where the backup & restore features could not be used with arbitrary paths. Only the SOLR_HOME, the SOLR_DATA_HOME or explicitly defined paths could be provided. Instead of using the "allowedPaths" input to specify additional paths, the solr operator now puts backup & restore data within SOLR_HOME (the data volume). This allows us to support both 8.5- and 8.6+. Additionally the data PVCs will not retain the backup information, because volumes to not persist data that is mounted to other volumes within their directories. Signed-off-by: Houston Putman <[email protected]>
apache · Dec 8, 2020 · 6024117 · 6024117
1 parent 57e6f15
commit 6024117
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 46 deletions.
diff --git a/README.md b/README.md
@@ -48,7 +48,12 @@ Please visit the following pages for documentation on using and developing the S
 
   **This means that Kubernetes support is now limited to 1.16+.**
   If you are unable to use a newer version of Kubernetes, please install the `v0.2.6` version of the Solr Operator for use with Kubernetes 1.15 and below.
-
+
+- The location of backup-restore volume mounts in Solr containers has changed from `/var/solr/solr-backup-restore` to `/var/solr/data/backup-restore`.
+This change was made to ensure that there were no issues using the backup API with solr 8.6+, which restricts the locations that backup data can be saved to and read from.
+This change should be transparent if you are merely using the SolrBackup CRD.
+All files permissions issues with SolrBackups should now be addressed.
+
 - The default `PodManagementPolicy` for StatefulSets has been changed to `Parallel` from `OrderedReady`.
 This change will not affect existing StatefulSets, as `PodManagementPolicy` cannot be updated.
 In order to continue using `OrderedReady` on new SolrClouds, please use the following setting:  

diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go
@@ -91,7 +91,7 @@ func (r *SolrBackupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error)
 	if allCollectionsComplete && collectionActionTaken {
 		// Requeue immediately to start the persisting job
 		// From here on in the backup lifecycle, requeueing will not happen for the backup.
-		requeueOrNot = reconcile.Result{Requeue: true}
+		requeueOrNot = reconcile.Result{RequeueAfter: time.Second * 10}
 	} else if solrCloud == nil {
 		requeueOrNot = reconcile.Result{}
 	} else {
@@ -102,9 +102,9 @@ func (r *SolrBackupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error)
 			// We will count on the Job updates to be notifified
 			requeueOrNot = reconcile.Result{}
 			err = persistSolrCloudBackups(r, backup, solrCloud)
-		}
-		if err != nil {
-			r.Log.Error(err, "Error while persisting SolrCloud backup")
+			if err != nil {
+				r.Log.Error(err, "Error while persisting SolrCloud backup")
+			}
 		}
 	}
 

diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go
@@ -32,10 +32,10 @@ import (
 )
 
 const (
-	BaseBackupRestorePath = "/var/solr/solr-backup-restore"
-	TarredFile            = "/var/solr/solr-backup-restore/backup.tgz"
-	BackupTarCommand      = "cd " + BaseBackupRestorePath + " && tar -czf /tmp/backup.tgz * && mv /tmp/backup.tgz " + TarredFile + " && chmod -R a+rwx " + TarredFile + " && cd - && "
-	CleanupCommand        = " && rm -rf " + BaseBackupRestorePath + "/{*,.*}"
+	BaseBackupRestorePath = "/var/solr/data/backup-restore"
+	TarredFile            = "/var/solr/data/backup-restore/backup.tgz"
+	CleanupCommand        = " && rm -rf " + BaseBackupRestorePath + "/*"
+	BackupTarCommand      = "cd " + BaseBackupRestorePath + " && tar -czf /tmp/backup.tgz * " + CleanupCommand + " && mv /tmp/backup.tgz " + TarredFile + " && chmod -R a+rwx " + TarredFile + " && cd - && "
 
 	AWSSecretDir = "/var/aws"
 
@@ -115,7 +115,7 @@ func GenerateBackupPersistenceJob(solrBackup *solr.SolrBackup, solrBackupVolume
 
 	// ttlSeconds := JobTTLSeconds
 
-	image, env, command, volume, volumeMount, numRetries := GeneratePersistenceOptions(solrBackup)
+	image, env, command, volume, volumeMount, numRetries := GeneratePersistenceOptions(solrBackup, solrBackupVolume)
 
 	volumes := []corev1.Volume{
 		{
@@ -137,6 +137,8 @@ func GenerateBackupPersistenceJob(solrBackup *solr.SolrBackup, solrBackupVolume
 	}
 
 	parallelismAndCompletions := int32(1)
+	solrGroup := int64(DefaultSolrGroup)
+	solrUser := int64(DefaultSolrUser)
 
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
@@ -165,6 +167,11 @@ func GenerateBackupPersistenceJob(solrBackup *solr.SolrBackup, solrBackupVolume
 							Command:         command,
 						},
 					},
+					SecurityContext: &corev1.PodSecurityContext{
+						RunAsUser:  &solrUser,
+						RunAsGroup: &solrGroup,
+						FSGroup:    &solrGroup,
+					},
 					RestartPolicy: corev1.RestartPolicyNever,
 				},
 			},
@@ -174,7 +181,7 @@ func GenerateBackupPersistenceJob(solrBackup *solr.SolrBackup, solrBackupVolume
 }
 
 // GeneratePersistenceOptions creates options for a Job that will persist backup data
-func GeneratePersistenceOptions(solrBackup *solr.SolrBackup) (image solr.ContainerImage, envVars []corev1.EnvVar, command []string, volume *corev1.Volume, volumeMount *corev1.VolumeMount, numRetries *int32) {
+func GeneratePersistenceOptions(solrBackup *solr.SolrBackup, solrBackupVolume corev1.VolumeSource) (image solr.ContainerImage, envVars []corev1.EnvVar, command []string, volume *corev1.Volume, volumeMount *corev1.VolumeMount, numRetries *int32) {
 	persistenceSource := solrBackup.Spec.Persistence
 	if persistenceSource.Volume != nil {
 		// Options for persisting to a volume
@@ -185,19 +192,25 @@ func GeneratePersistenceOptions(solrBackup *solr.SolrBackup) (image solr.Contain
 				Value: persistenceSource.Volume.Filename,
 			},
 		}
-		// Copy the information to the persistent storage, and delete it from the backup-restore volume.
-		command = []string{"sh", "-c", BackupTarCommand + "cp " + TarredFile + " \"/var/backup-persistence/${FILE_NAME}\"" + CleanupCommand}
 
-		volume = &corev1.Volume{
-			Name:         "persistence",
-			VolumeSource: persistenceSource.Volume.VolumeSource,
-		}
-		volumeMount = &corev1.VolumeMount{
-			Name:      "persistence",
-			SubPath:   persistenceSource.Volume.Path,
-			ReadOnly:  false,
-			MountPath: "/var/backup-persistence",
+		finalLocation := BaseBackupRestorePath
+		// If the persistence volume is the same as the backup volume, we cannot mount the same volume twice.
+		if !DeepEqualWithNils(solrBackupVolume, persistenceSource.Volume.VolumeSource) {
+			finalLocation = "/var/backup-persistence"
+			volume = &corev1.Volume{
+				Name:         "persistence",
+				VolumeSource: persistenceSource.Volume.VolumeSource,
+			}
+			volumeMount = &corev1.VolumeMount{
+				Name:      "persistence",
+				SubPath:   persistenceSource.Volume.Path,
+				ReadOnly:  false,
+				MountPath: finalLocation,
+			}
 		}
+		// Copy the information to the persistent storage, and delete it from the backup-restore volume.
+		command = []string{"sh", "-c", BackupTarCommand + "mv " + TarredFile + " \"" + finalLocation + "/${FILE_NAME}\""}
+
 		r := int32(1)
 		numRetries = &r
 	} else if persistenceSource.S3 != nil {
@@ -300,7 +313,7 @@ func GeneratePersistenceOptions(solrBackup *solr.SolrBackup) (image solr.Contain
 			includeUrl = "--endpoint-url \"${ENDPOINT_URL}\" "
 		}
 
-		command = []string{"sh", "-c", BackupTarCommand + "aws s3 cp " + includeUrl + TarredFile + " \"s3://${BUCKET}/${KEY}\"" + CleanupCommand}
+		command = []string{"sh", "-c", BackupTarCommand + "aws s3 cp " + includeUrl + TarredFile + " \"s3://${BUCKET}/${KEY}\""}
 		numRetries = persistenceSource.S3.Retries
 	}
 

diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
@@ -17,6 +17,7 @@ limitations under the License.
 package util
 
 import (
+	"fmt"
 	"sort"
 	"strconv"
 	"strings"
@@ -35,6 +36,9 @@ const (
 
 	SolrNodeContainer = "solrcloud-node"
 
+	DefaultSolrUser  = 8983
+	DefaultSolrGroup = 8983
+
 	SolrStorageFinalizer             = "storage.finalizers.solr.apache.org"
 	SolrZKConnectionStringAnnotation = "solr.apache.org/zkConnectionString"
 	SolrPVCTechnologyLabel           = "solr.apache.org/technology"
@@ -73,7 +77,7 @@ const (
 func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCloudStatus, hostNameIPs map[string]string, solrXmlConfigMapName string, solrXmlMd5 string) *appsv1.StatefulSet {
 	gracePeriodTerm := int64(10)
 	solrPodPort := solrCloud.Spec.SolrAddressability.PodPort
-	fsGroup := int64(solrPodPort)
+	fsGroup := int64(DefaultSolrGroup)
 	defaultMode := int32(420)
 	defaultHandler := corev1.Handler{
 		HTTPGet: &corev1.HTTPGetAction{
@@ -108,6 +112,10 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
 		podAnnotations = customPodOptions.Annotations
 	}
 
+	// Keep track of the SolrOpts that the Solr Operator needs to set
+	// These will be added to the SolrOpts given by the user.
+	allSolrOpts := []string{"-DhostPort=$(SOLR_NODE_PORT)"}
+
 	// Volumes & Mounts
 	solrVolumes := []corev1.Volume{
 		{
@@ -191,6 +199,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
 			Name:      BackupRestoreVolume,
 			MountPath: BaseBackupRestorePath,
 			SubPath:   BackupRestoreSubPathForCloud(solrCloud.Spec.StorageOptions.BackupRestoreOptions.Directory, solrCloud.Name),
+			ReadOnly:  false,
 		})
 	}
 
@@ -250,10 +259,6 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
 		}
 	}
 
-	// Keep track of the SolrOpts that the Solr Operator needs to set
-	// These will be added to the SolrOpts given by the user.
-	allSolrOpts := []string{"-DhostPort=$(SOLR_NODE_PORT)"}
-
 	// Environment Variables
 	envVars := []corev1.EnvVar{
 		{
@@ -341,24 +346,7 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
 		Value: strings.Join(allSolrOpts, " "),
 	})
 
-	initContainers := []corev1.Container{
-		{
-			Name:            "cp-solr-xml",
-			Image:           solrCloud.Spec.BusyBoxImage.ToImageName(),
-			ImagePullPolicy: solrCloud.Spec.BusyBoxImage.PullPolicy,
-			Command:         []string{"sh", "-c", "cp /tmp/solr.xml /tmp-config/solr.xml"},
-			VolumeMounts: []corev1.VolumeMount{
-				{
-					Name:      "solr-xml",
-					MountPath: "/tmp",
-				},
-				{
-					Name:      solrDataVolumeName,
-					MountPath: "/tmp-config",
-				},
-			},
-		},
-	}
+	initContainers := generateSolrSetupInitContainers(solrCloud, solrDataVolumeName)
 
 	// Add user defined additional init containers
 	if customPodOptions != nil && len(customPodOptions.InitContainers) > 0 {
@@ -510,6 +498,44 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
 	return stateful
 }
 
+func generateSolrSetupInitContainers(solrCloud *solr.SolrCloud, solrDataVolumeName string) []corev1.Container {
+	// The setup of the solr.xml will always be necessary
+	volumeMounts := []corev1.VolumeMount{
+		{
+			Name:      "solr-xml",
+			MountPath: "/tmp",
+		},
+		{
+			Name:      solrDataVolumeName,
+			MountPath: "/tmp-config",
+		},
+	}
+	setupCommands := []string{"cp /tmp/solr.xml /tmp-config/solr.xml"}
+
+	// Add prep for backup-restore volume
+	// This entails setting the correct permissions for the directory
+	if solrCloud.Spec.StorageOptions.BackupRestoreOptions != nil {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      BackupRestoreVolume,
+			MountPath: "/backup-restore",
+			SubPath:   BackupRestoreSubPathForCloud(solrCloud.Spec.StorageOptions.BackupRestoreOptions.Directory, solrCloud.Name),
+			ReadOnly:  false,
+		})
+
+		setupCommands = append(setupCommands, fmt.Sprintf("chown -R %d:%d /backup-restore", DefaultSolrUser, DefaultSolrGroup))
+	}
+
+	volumePrepInitContainer := corev1.Container{
+		Name:            "cp-solr-xml",
+		Image:           solrCloud.Spec.BusyBoxImage.ToImageName(),
+		ImagePullPolicy: solrCloud.Spec.BusyBoxImage.PullPolicy,
+		Command:         []string{"sh", "-c", strings.Join(setupCommands, " && ")},
+		VolumeMounts:    volumeMounts,
+	}
+
+	return []corev1.Container{volumePrepInitContainer}
+}
+
 // GenerateConfigMap returns a new corev1.ConfigMap pointer generated for the SolrCloud instance solr.xml
 // solrCloud: SolrCloud instance
 func GenerateConfigMap(solrCloud *solr.SolrCloud) *corev1.ConfigMap {