fix(in-cluster): do not allow the cluster to be used when disabled (#…

…21208)

Signed-off-by: Alexandre Gaudreault <[email protected]>

applicationset/controllers/applicationset_controller_test.go

@@ -41,6 +41,39 @@ import (
 	"github.com/argoproj/argo-cd/v3/util/settings"
 )
 
+// getDefaultTestClientSet creates a Clientset with the default argo objects
+// and objects specified in parameters
+func getDefaultTestClientSet(obj ...runtime.Object) *kubefake.Clientset {
+	argoCDSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      argocommon.ArgoCDSecretName,
+			Namespace: "argocd",
+			Labels: map[string]string{
+				"app.kubernetes.io/part-of": "argocd",
+			},
+		},
+		Data: map[string][]byte{
+			"admin.password":   nil,
+			"server.secretkey": nil,
+		},
+	}
+
+	emptyArgoCDConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      argocommon.ArgoCDConfigMapName,
+			Namespace: "argocd",
+			Labels: map[string]string{
+				"app.kubernetes.io/part-of": "argocd",
+			},
+		},
+		Data: map[string]string{},
+	}
+
+	objects := append(obj, emptyArgoCDConfigMap, argoCDSecret)
+	kubeclientset := kubefake.NewClientset(objects...)
+	return kubeclientset
+}
+
 func TestCreateOrUpdateInCluster(t *testing.T) {
 	scheme := runtime.NewScheme()
 	err := v1alpha1.AddToScheme(scheme)
@@ -1317,8 +1350,7 @@ func TestRemoveFinalizerOnInvalidDestination_DestinationTypes(t *testing.T) {
 				},
 			}
 
-			objects := append([]runtime.Object{}, secret)
-			kubeclientset := kubefake.NewSimpleClientset(objects...)
+			kubeclientset := getDefaultTestClientSet(secret)
 			metrics := appsetmetrics.NewFakeAppsetMetrics()
 
 			argodb := db.NewDB("argocd", settings.NewSettingsManager(context.TODO(), kubeclientset, "argocd"), kubeclientset)
@@ -2054,8 +2086,7 @@ func TestValidateGeneratedApplications(t *testing.T) {
 				},
 			}
 
-			objects := append([]runtime.Object{}, secret)
-			kubeclientset := kubefake.NewSimpleClientset(objects...)
+			kubeclientset := getDefaultTestClientSet(secret)
 
 			argodb := db.NewDB("argocd", settings.NewSettingsManager(context.TODO(), kubeclientset, "argocd"), kubeclientset)
 
@@ -2117,7 +2148,7 @@ func TestReconcilerValidationProjectErrorBehaviour(t *testing.T) {
 		},
 	}
 
-	kubeclientset := kubefake.NewSimpleClientset()
+	kubeclientset := getDefaultTestClientSet()
 
 	client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appSet, &project).WithStatusSubresource(&appSet).WithIndex(&v1alpha1.Application{}, ".metadata.controller", appControllerIndexer).Build()
 	metrics := appsetmetrics.NewFakeAppsetMetrics()
@@ -2404,8 +2435,7 @@ func applicationsUpdateSyncPolicyTest(t *testing.T, applicationsSyncPolicy v1alp
 		},
 	}
 
-	objects := append([]runtime.Object{}, secret)
-	kubeclientset := kubefake.NewSimpleClientset(objects...)
+	kubeclientset := getDefaultTestClientSet(secret)
 
 	client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appSet, &defaultProject).WithStatusSubresource(&appSet).WithIndex(&v1alpha1.Application{}, ".metadata.controller", appControllerIndexer).Build()
 	metrics := appsetmetrics.NewFakeAppsetMetrics()
@@ -2580,8 +2610,7 @@ func applicationsDeleteSyncPolicyTest(t *testing.T, applicationsSyncPolicy v1alp
 		},
 	}
 
-	objects := append([]runtime.Object{}, secret)
-	kubeclientset := kubefake.NewSimpleClientset(objects...)
+	kubeclientset := getDefaultTestClientSet(secret)
 
 	client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&appSet, &defaultProject).WithStatusSubresource(&appSet).WithIndex(&v1alpha1.Application{}, ".metadata.controller", appControllerIndexer).Build()
 	metrics := appsetmetrics.NewFakeAppsetMetrics()
@@ -2700,7 +2729,7 @@ func TestPolicies(t *testing.T) {
 		Spec:       v1alpha1.AppProjectSpec{SourceRepos: []string{"*"}, Destinations: []v1alpha1.ApplicationDestination{{Namespace: "*", Server: "https://kubernetes.default.svc"}}},
 	}
 
-	kubeclientset := kubefake.NewSimpleClientset()
+	kubeclientset := getDefaultTestClientSet()
 
 	for _, c := range []struct {
 		name          string

cmd/argocd/commands/admin/app_test.go

@@ -16,6 +16,7 @@ import (
 	kubefake "k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/cache"
 
+	"github.com/argoproj/argo-cd/v3/common"
 	statecache "github.com/argoproj/argo-cd/v3/controller/cache"
 	cachemocks "github.com/argoproj/argo-cd/v3/controller/cache/mocks"
 	"github.com/argoproj/argo-cd/v3/controller/metrics"
@@ -57,15 +58,28 @@ func TestGetReconcileResults(t *testing.T) {
 func TestGetReconcileResults_Refresh(t *testing.T) {
 	ctx := context.Background()
 
-	cm := corev1.ConfigMap{
+	argoCM := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "argocd-cm",
+			Name:      common.ArgoCDConfigMapName,
 			Namespace: "default",
 			Labels: map[string]string{
 				"app.kubernetes.io/part-of": "argocd",
 			},
 		},
 	}
+	argoCDSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      common.ArgoCDSecretName,
+			Namespace: "default",
+			Labels: map[string]string{
+				"app.kubernetes.io/part-of": "argocd",
+			},
+		},
+		Data: map[string][]byte{
+			"admin.password":   nil,
+			"server.secretkey": nil,
+		},
+	}
 	proj := &v1alpha1.AppProject{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "default",
@@ -91,7 +105,7 @@ func TestGetReconcileResults_Refresh(t *testing.T) {
 
 	appClientset := appfake.NewSimpleClientset(app, proj)
 	deployment := test.NewDeployment()
-	kubeClientset := kubefake.NewClientset(deployment, &cm)
+	kubeClientset := kubefake.NewClientset(deployment, argoCM, argoCDSecret)
 	clusterCache := clustermocks.ClusterCache{}
 	clusterCache.On("IsNamespaced", mock.Anything).Return(true, nil)
 	clusterCache.On("GetGVKParser", mock.Anything).Return(nil)

docs/operator-manual/upgrading/2.14-3.0.md

@@ -22,3 +22,16 @@ The v2 behavior can be preserved by setting the config value `server.rbac.disabl
 to `false` in the Argo CD ConfigMap `argocd-cm`.
 
 Read the [RBAC documentation](../rbac.md#fine-grained-permissions-for-updatedelete-action) for more detailed inforamtion.
+
+## Other changes
+
+### Using `cluster.inClusterEnabled: "false"`
+
+When `cluster.inClusterEnabled: "false"` is explicitly configured, Applications currently configured to
+sync on the in-cluster cluster will now be in an Unknown state, without the possibility to sync resources.
+
+It will not be possible to create new Applications using the in-cluster cluster. When deleting existing
+Application, it will not delete the previously managed resources.
+
+It is recommended to perform any cleanup or migration to existing in-cluster Application before upgrading
+when in-cluster is disabled. To perform cleanup post-migration, the in-cluster will need to be enabled temporarily.

test/e2e/app_management_test.go

@@ -732,6 +732,28 @@ func TestComparisonFailsIfDestinationClusterIsInvalid(t *testing.T) {
 		Expect(Condition(ApplicationConditionInvalidSpecError, "there are no clusters with this name"))
 }
 
+func TestComparisonFailsIfInClusterDisabled(t *testing.T) {
+	Given(t).
+		Path(guestbookPath).
+		DestServer(KubernetesInternalAPIServerAddr).
+		When().
+		CreateApp().
+		Refresh(RefreshTypeNormal).
+		Sync().
+		Then().
+		Expect(Success("")).
+		Expect(HealthIs(health.HealthStatusHealthy)).
+		Expect(SyncStatusIs(SyncStatusCodeSynced)).
+		When().
+		SetParamInSettingConfigMap("cluster.inClusterEnabled", "false").
+		Refresh(RefreshTypeNormal).
+		Then().
+		Expect(Success("")).
+		Expect(HealthIs(health.HealthStatusUnknown)).
+		Expect(SyncStatusIs(SyncStatusCodeUnknown)).
+		Expect(Condition(ApplicationConditionInvalidSpecError, fmt.Sprintf("cluster %q is disabled", KubernetesInternalAPIServerAddr)))
+}
+
 func TestCannotSetInvalidPath(t *testing.T) {
 	Given(t).
 		Path(guestbookPath).
@@ -2189,6 +2211,19 @@ func TestCreateAppWithNoNameSpaceWhenRequired2(t *testing.T) {
 		})
 }
 
+func TestCreateAppWithInClusterDisabled(t *testing.T) {
+	Given(t).
+		Path(guestbookPath).
+		DestServer(KubernetesInternalAPIServerAddr).
+		When().
+		SetParamInSettingConfigMap("cluster.inClusterEnabled", "false").
+		IgnoreErrors().
+		CreateApp().
+		Then().
+		// RPC error messages are quoted: time="2024-12-18T04:13:58Z" level=fatal msg="<Quoted value>"
+		Expect(Error("", fmt.Sprintf(`cluster \"%s\" is disabled`, KubernetesInternalAPIServerAddr)))
+}
+
 func TestListResource(t *testing.T) {
 	fixture.SkipOnEnv(t, "OPENSHIFT")
 	Given(t).

test/e2e/fixture/app/expectation.go

@@ -370,7 +370,7 @@ func Error(message, err string, matchers ...func(string, string) bool) Expectati
 			return failed, fmt.Sprintf("output does not contain '%s'", message)
 		}
 		if !match(c.actions.lastError.Error(), err) {
-			return failed, fmt.Sprintf("error does not contain '%s'", message)
+			return failed, fmt.Sprintf("error does not contain '%s'", err)
 		}
 		return succeeded, fmt.Sprintf("error '%s'", message)
 	}

util/db/cluster.go

@@ -142,11 +142,19 @@ func (db *db) WatchClusters(ctx context.Context,
 	handleModEvent func(oldCluster *appv1.Cluster, newCluster *appv1.Cluster),
 	handleDeleteEvent func(clusterServer string),
 ) error {
-	localCls, err := db.GetCluster(ctx, appv1.KubernetesInternalAPIServerAddr)
+	argoSettings, err := db.settingsMgr.GetSettings()
 	if err != nil {
 		return err
 	}
-	handleAddEvent(localCls)
+
+	localCls := db.getLocalCluster()
+	if argoSettings.InClusterEnabled {
+		localCls, err = db.GetCluster(ctx, appv1.KubernetesInternalAPIServerAddr)
+		if err != nil {
+			return fmt.Errorf("could not get local cluster: %w", err)
+		}
+		handleAddEvent(localCls)
+	}
 
 	db.watchSecrets(
 		ctx,
@@ -159,9 +167,11 @@ func (db *db) WatchClusters(ctx context.Context,
 				return
 			}
 			if cluster.Server == appv1.KubernetesInternalAPIServerAddr {
-				// change local cluster event to modified or deleted, since it cannot be re-added or deleted
-				handleModEvent(localCls, cluster)
-				localCls = cluster
+				if argoSettings.InClusterEnabled {
+					// change local cluster event to modified, since it cannot be added at runtime
+					handleModEvent(localCls, cluster)
+					localCls = cluster
+				}
 				return
 			}
 			handleAddEvent(cluster)
@@ -185,10 +195,11 @@ func (db *db) WatchClusters(ctx context.Context,
 		},
 
 		func(secret *corev1.Secret) {
-			if string(secret.Data["server"]) == appv1.KubernetesInternalAPIServerAddr {
-				// change local cluster event to modified or deleted, since it cannot be re-added or deleted
-				handleModEvent(localCls, db.getLocalCluster())
-				localCls = db.getLocalCluster()
+			if string(secret.Data["server"]) == appv1.KubernetesInternalAPIServerAddr && argoSettings.InClusterEnabled {
+				// change local cluster event to modified, since it cannot be deleted at runtime, unless disabled.
+				newLocalCls := db.getLocalCluster()
+				handleModEvent(localCls, newLocalCls)
+				localCls = newLocalCls
 			} else {
 				handleDeleteEvent(string(secret.Data["server"]))
 			}
@@ -214,6 +225,14 @@ func (db *db) getClusterSecret(server string) (*corev1.Secret, error) {
 
 // GetCluster returns a cluster from a query
 func (db *db) GetCluster(_ context.Context, server string) (*appv1.Cluster, error) {
+	argoSettings, err := db.settingsMgr.GetSettings()
+	if err != nil {
+		return nil, err
+	}
+	if server == appv1.KubernetesInternalAPIServerAddr && !argoSettings.InClusterEnabled {
+		return nil, status.Errorf(codes.NotFound, "cluster %q is disabled", server)
+	}
+
 	informer, err := db.settingsMgr.GetSecretsInformer()
 	if err != nil {
 		return nil, err
@@ -222,6 +241,7 @@ func (db *db) GetCluster(_ context.Context, server string) (*appv1.Cluster, erro
 	if err != nil {
 		return nil, err
 	}
+
 	if len(res) > 0 {
 		return SecretToCluster(res[0].(*corev1.Secret))
 	}
@@ -254,6 +274,10 @@ func (db *db) GetProjectClusters(_ context.Context, project string) ([]*appv1.Cl
 }
 
 func (db *db) GetClusterServersByName(_ context.Context, name string) ([]string, error) {
+	argoSettings, err := db.settingsMgr.GetSettings()
+	if err != nil {
+		return nil, err
+	}
 	informer, err := db.settingsMgr.GetSecretsInformer()
 	if err != nil {
 		return nil, err
@@ -265,7 +289,7 @@ func (db *db) GetClusterServersByName(_ context.Context, name string) ([]string,
 		return nil, err
 	}
 
-	if len(localClusterSecrets) == 0 && db.getLocalCluster().Name == name {
+	if len(localClusterSecrets) == 0 && db.getLocalCluster().Name == name && argoSettings.InClusterEnabled {
 		return []string{appv1.KubernetesInternalAPIServerAddr}, nil
 	}
 
@@ -276,7 +300,11 @@ func (db *db) GetClusterServersByName(_ context.Context, name string) ([]string,
 	var res []string
 	for i := range secrets {
 		s := secrets[i].(*corev1.Secret)
-		res = append(res, strings.TrimRight(string(s.Data["server"]), "/"))
+		server := strings.TrimRight(string(s.Data["server"]), "/")
+		if !argoSettings.InClusterEnabled && server == appv1.KubernetesInternalAPIServerAddr {
+			continue
+		}
+		res = append(res, server)
 	}
 	return res, nil
 }

util/db/cluster_norace_test.go

@@ -49,7 +49,7 @@ func TestWatchClusters_CreateRemoveCluster(t *testing.T) {
 	kubeclientset := fake.NewClientset(emptyArgoCDConfigMap, argoCDSecret)
 	settingsManager := settings.NewSettingsManager(context.Background(), kubeclientset, fakeNamespace)
 	db := NewDB(fakeNamespace, settingsManager, kubeclientset)
-	runWatchTest(t, db, []func(old *v1alpha1.Cluster, new *v1alpha1.Cluster){
+	completed := runWatchTest(t, db, []func(old *v1alpha1.Cluster, new *v1alpha1.Cluster){
 		func(old *v1alpha1.Cluster, new *v1alpha1.Cluster) {
 			assert.Nil(t, old)
 			assert.Equal(t, v1alpha1.KubernetesInternalAPIServerAddr, new.Server)
@@ -72,6 +72,7 @@ func TestWatchClusters_CreateRemoveCluster(t *testing.T) {
 			assert.Equal(t, "https://minikube", old.Server)
 		},
 	})
+	assert.True(t, completed, "Failed due to timeout")
 }
 
 func TestWatchClusters_LocalClusterModifications(t *testing.T) {
@@ -104,7 +105,7 @@ func TestWatchClusters_LocalClusterModifications(t *testing.T) {
 	kubeclientset := fake.NewClientset(emptyArgoCDConfigMap, argoCDSecret)
 	settingsManager := settings.NewSettingsManager(context.Background(), kubeclientset, fakeNamespace)
 	db := NewDB(fakeNamespace, settingsManager, kubeclientset)
-	runWatchTest(t, db, []func(old *v1alpha1.Cluster, new *v1alpha1.Cluster){
+	completed := runWatchTest(t, db, []func(old *v1alpha1.Cluster, new *v1alpha1.Cluster){
 		func(old *v1alpha1.Cluster, new *v1alpha1.Cluster) {
 			assert.Nil(t, old)
 			assert.Equal(t, v1alpha1.KubernetesInternalAPIServerAddr, new.Server)
@@ -127,4 +128,43 @@ func TestWatchClusters_LocalClusterModifications(t *testing.T) {
 			assert.Equal(t, "in-cluster", new.Name)
 		},
 	})
+	assert.True(t, completed, "Failed due to timeout")
+}
+
+func TestWatchClusters_LocalClusterModificationsWhenDisabled(t *testing.T) {
+	// !race:
+	// Intermittent failure when running TestWatchClusters_LocalClusterModifications with -race, likely due to race condition
+	// https://github.com/argoproj/argo-cd/issues/4755
+	argoCDConfigMapWithInClusterServerAddressDisabled := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      common.ArgoCDConfigMapName,
+			Namespace: fakeNamespace,
+			Labels: map[string]string{
+				"app.kubernetes.io/part-of": "argocd",
+			},
+		},
+		Data: map[string]string{"cluster.inClusterEnabled": "false"},
+	}
+	argoCDSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      common.ArgoCDSecretName,
+			Namespace: fakeNamespace,
+			Labels: map[string]string{
+				"app.kubernetes.io/part-of": "argocd",
+			},
+		},
+		Data: map[string][]byte{
+			"admin.password":   nil,
+			"server.secretkey": nil,
+		},
+	}
+	kubeclientset := fake.NewClientset(argoCDConfigMapWithInClusterServerAddressDisabled, argoCDSecret)
+	settingsManager := settings.NewSettingsManager(context.Background(), kubeclientset, fakeNamespace)
+	db := NewDB(fakeNamespace, settingsManager, kubeclientset)
+	completed := runWatchTest(t, db, []func(_ *v1alpha1.Cluster, _ *v1alpha1.Cluster){
+		func(_ *v1alpha1.Cluster, _ *v1alpha1.Cluster) {
+			assert.Fail(t, "The in-cluster should not be added when disabled")
+		},
+	})
+	assert.False(t, completed, "Expecting the method to never complete because no cluster is ever added")
 }