authorizer/webhook: make webhook authorizer cluster aware

.gitignore

@@ -106,7 +106,9 @@ kubernetes.tar.gz
 # generated files in any directory
 # TODO(thockin): uncomment this when we stop committing the generated files.
 #zz_generated.*
-zz_generated.openapi.go
+# HACK: Needed for short term dependencies on core openapi docs, will need to be refactored
+#       out of api dependencies in order to vendor a "pure" Kube API
+# zz_generated.openapi.go
 zz_generated_*_test.go
 
 # TODO(roycaihw): remove this when we stop committing the generated definition

KCP_RELATED_CHANGES.md

@@ -0,0 +1,97 @@
+# Why this forked repository ?
+
+This repository carries the prototype branch which accumulates the hacks, prototypes, proto-KEP experiments, and workarounds required to make [KCP](https://github.com/kcp-dev/kcp/blob/main/README.md) a reality.
+It is based on K8S 1.22 for now and commits are identified with basic labels like HACK/FEATURE/WORKAROUND.
+
+# Summary of changes
+
+The detailed explanation of the changes made on top of the Kubernetes code can be found in both the commit messages, and comments of the associated code.
+
+However here is a summary of the changes, with the underlying requirements and motivations. Reading the linked investigation document first will help.
+
+## A. Minimal API Server
+
+__Investigation document:__ [minimal-api-server.md](https://github.com/kcp-dev/kcp/blob/main/docs/investigations/minimal-api-server.md)
+
+1.  New generic control plane based on kube api-server
+
+    It is mainly provided by code:
+
+    1. initially duplicated from kube api-server main code, and legacy scheme (`DUPLICATE` commits),
+
+    2. then stripped down from unnecessary things (ergress, api aggregation, webhooks) and APIs (Pods, Nodes, Deployments, etc ...) (`NEW` commits)
+
+2.  Support adding K8S built-in resources (`core/Pod`, `apps/Deployment`, ...) as CRDs
+
+    This is required since the new generic control plane scheme doesn't contain those resources any more.
+
+    This is provided by:
+
+    - hacks (`HACK` commits) that:
+
+      1. <a id="A-2-1"></a> allow the go-restful server to be bypassed for those resources, and route them to the CRD handler
+      2. <a id="A-2-2"></a> allow the CRD handler, and opanapi publisher, to support resources of the `core` group
+      3. <a id="A-2-3"></a> convert the `protobuf` requests sent to those resources resources to requests in the `application/json` content type, before letting the CRD handler serve them
+      4. <a id="A-2-4"></a> replace the table converter of CRDs that bring back those resources, with the default table converter of the related built-in resource
+
+    - a new feature, or potential kube fix (`KUBEFIX` commit), that:
+
+      5. <a id="A-2-5"></a> introduces the support of strategic merge patch for CRDs.
+          This support uses the OpenAPI v3 schema of the CRD to drive the SMP execution, but only adds a minimal implementation and doesn't fully support OpenAPI schemas that don't have expected `patchStrategy` and `patchMergeKey` annotations.
+          In order to avoid changing the behavior of existing client tools, the support is only added for those K8S built-in resources 
+
+## B. Logical clusters
+
+__Investigation document:__ [logical-clusters.md](https://github.com/kcp-dev/kcp/blob/main/docs/investigations/logical-clusters.md)
+
+1.  Logical clusters represented as a prefix in etcd
+
+    It is mainly provided by hacks (`HACK` commits) that:
+
+    1. <a id="B-1-1"></a> allow intercepting the api server handler chain to set the expected logical cluster context value from either a given suffix in the request APIServer base URL, or a given header in the http request
+
+    2. <a id="B-1-2"></a> change etcd storage layer in order to use the logical cluster as a prefix in the etcd key
+
+    3. <a id="B-1-3"></a> allow wildcard watches that retrieve objects from all the logical clusters
+
+    4. <a id="B-1-4"></a> correctly get or set the `clusterName` metadata field in the storage layer operations based on the etcd key and its new prefix
+
+2.  Support of logical clusters (== tenancy) in the CRD management, OpenAPI and discovery endpoints, and clients used by controllers
+
+    <a id="B-2"></a>It is mainly provided by a hack (`HACK` commit) that adds CRD tenancy by ensuring that logical clusters are taken in account in:
+    - CRD-related controllers
+    - APIServices-related controllers
+    - Discovery + OpenAPI endpoints
+
+    In the current Kubernetes design, those 3 areas are highly coupled and intricated, which explains why this commit had to hack the code at various levels:
+    - client-go level
+    - controllers level,
+    - http handlers level.
+
+    While this gives a detailed idea of which code needs to be touched in order to enable CRD tenancy, a clean implementation would first require some refactoring, in order to build the required abstraction layers that would allow decoupling those areas.
+
+# Potential client problems
+
+Although these changes in the K8S codebase were made in order to keep the compatibility with Kuberntes client tools, there might be some problems:
+
+## Incomplete protobuf support for built-in resources
+
+In some contexts, like the `controller-runtime` library used by the Operator SDK, all the resources of the `client-go` scheme are created / updated using the `application/vnd.kubernetes.protobuf` content type.
+
+However when these resources are in fact added as CRDs, in the KCP minimal API server scenario, these resources cannot be created / updated since the protobuf (de)serialization is not (and probably cannot be) supported for CRDs.
+So for now in this case, the [A.2.3 hack mentioned above](#A-2-3) just converts the `protobuf` request to a `json` one, but this might not cover all the use-cases or corner cases.
+
+The clean solution would probably be the negotiation of serialization type in `client-go`, which we haven't implemented yet, but which would work like this:
+When a request for an unsupported serialization is returned, the server should reject it with a 406
+and provide a list of supported content types. `client-go` should then examine whether it can satisfy such a request by encoding the object with a different scheme.
+This would require a KEP but at least is in keeping with content negotiation on GET / WATCH in Kube
+
+## Incomplete Strategic merge patch support for built-in resources
+
+Client tools like `kubectl` assume that all K8S native resources (== `client-go` schema resources)
+support strategic merge patch, and use it by default when updating or patching a resource.
+
+In Kube, currently, strategic merge patch is not supported for CRDs, which would break compatibility with client tools for all the K8S natives resources that are in fact added as CRD in the KCP minimal api server.
+The [A-2-5 change mentioned above](#A-2-5) tries to fix this by using the CRD openAPI v3 schema as the source of the required information that will drive the strategic merge patch execution.
+
+While this fixes the problem in most cases, there might still be errors in case the OpenAPI v2 schema for such a resource is missing `x-kubernetes-patch-strategy` and `x-kubernetes-patch-merge-key` annotations when imported from the CRD OpenAPI v3 schema.

cmd/kube-apiserver/app/aggregator.go

@@ -286,7 +286,7 @@ var apiVersionPriorities = map[schema.GroupVersion]priority{
 func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService {
 	apiServices := []*v1.APIService{}
 
-	for _, curr := range delegateAPIServer.ListedPaths() {
+	for _, curr := range delegateAPIServer.ListedPaths("") {
 		if curr == "/api/v1" {
 			apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"})
 			registration.AddAPIServiceToSyncOnStart(apiService)

cmd/kube-apiserver/app/server.go

@@ -409,7 +409,7 @@ func buildGenericConfig(
 	kubeVersion := version.Get()
 	genericConfig.Version = &kubeVersion
 
-	storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
+	storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig(legacyscheme.Scheme, legacyscheme.Codecs)
 	storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
 	completedStorageFactoryConfig, err := storageFactoryConfig.Complete(s.Etcd)
 	if err != nil {

cmd/kube-controller-manager/app/core.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/klog/v2"
 
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/quota/v1/generic"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -466,7 +467,9 @@ func startModifiedNamespaceController(ctx context.Context, controllerContext Con
 		return nil, true, err
 	}
 
-	discoverResourcesFn := namespaceKubeClient.Discovery().ServerPreferredNamespacedResources
+	discoverResourcesFn := func(clusterName string) ([]*metav1.APIResourceList, error) {
+		return namespaceKubeClient.Discovery().ServerPreferredNamespacedResources()
+	}
 
 	namespaceController := namespacecontroller.NewNamespaceController(
 		namespaceKubeClient,

hack/run-kcp-e2e.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+time make WHAT=test/e2e/e2e.test
+
+kubeconfig="${WORKDIR}/.kcp2/data/admin.kubeconfig"
+
+## each of the single-cluster contexts passes normal e2e
+for context in "admin" "user" "other"; do
+  _output/local/bin/linux/amd64/e2e.test --e2e-verify-service-account=false --ginkgo.focus "Watchers" \
+    --kubeconfig "${kubeconfig}" --context "${context}" \
+    --provider local
+done
+
+# the multi-cluster list/watcher works with all the single-cluster contexts
+_output/local/bin/linux/amd64/e2e.test --e2e-verify-service-account=false --ginkgo.focus "MultiClusterWorkflow" \
+  --kubeconfig "${kubeconfig}" --context "admin" \
+  --provider kcp \
+  --kcp-multi-cluster-kubeconfig "${kubeconfig}" --kcp-multi-cluster-context "cross-cluster" \
+  --kcp-secondary-kubeconfig "${kubeconfig}" --kcp-secondary-context "user" \
+  --kcp-tertiary-kubeconfig "${kubeconfig}" --kcp-tertiary-context "other" \
+  --kcp-clusterless-kubeconfig "${kubeconfig}" --kcp-clusterless-context "admin"

pkg/api/genericcontrolplanescheme/scheme.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package genericcontrolplanescheme
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+)
+
+var (
+	// Scheme is the default instance of runtime.Scheme to which control plane types
+	// in the Kubernetes API are already registered.
+	// NOTE: If you are copying this file to start a new api group, STOP! This Scheme
+	// is special and should appear ONLY in the api group, unless you really know what
+	// you're doing.
+	Scheme = runtime.NewScheme()
+
+	// Codecs provides access to encoding and decoding for the scheme
+	Codecs = serializer.NewCodecFactory(Scheme)
+
+	// ParameterCodec handles versioning of objects that are converted to query parameters.
+	ParameterCodec = runtime.NewParameterCodec(Scheme)
+)

pkg/apis/core/install/genericcontrolplane/install.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package install installs the v1 monolithic api, making it available as an
+// option to all of the API encoding/decoding machinery.
+package genericcontrolplane
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kubernetes/pkg/apis/core"
+	v1 "k8s.io/kubernetes/pkg/apis/core/v1"
+)
+
+// Install registers the API group and adds types to a scheme
+func Install(scheme *runtime.Scheme) {
+	utilruntime.Must(core.AddToGenericControlPlaneScheme(scheme))
+	utilruntime.Must(v1.AddToControlPlaneScheme(scheme))
+	utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion))
+}

pkg/apis/core/register_generic_control_plane.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package core
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	// GenericControlPlaneGroupName is the name of the group when installed in the generic control plane
+	GenericControlPlaneGroupName = "core"
+
+	// GenericControlPlaneSchemeGroupVersion is group version used to register these objects
+	GenericControlPlaneSchemeGroupVersion = schema.GroupVersion{Group: GenericControlPlaneGroupName, Version: "v1"}
+
+	// GenericControlPlaneSchemeBuilder object to register various known types for the control plane
+	GenericControlPlaneSchemeBuilder = runtime.NewSchemeBuilder(addGenericControlPlaneKnownTypes)
+
+	// AddToGenericControlPlaneScheme represents a func that can be used to apply all the registered
+	// funcs in a scheme
+	AddToGenericControlPlaneScheme = GenericControlPlaneSchemeBuilder.AddToScheme
+)
+
+func addGenericControlPlaneKnownTypes(scheme *runtime.Scheme) error {
+	if err := scheme.AddIgnoredConversionType(&metav1.TypeMeta{}, &metav1.TypeMeta{}); err != nil {
+		return err
+	}
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&Event{},
+		&EventList{},
+		&List{},
+		&LimitRange{},
+		&LimitRangeList{},
+		&ResourceQuota{},
+		&ResourceQuotaList{},
+		&Namespace{},
+		&NamespaceList{},
+		&ServiceAccount{},
+		&ServiceAccountList{},
+		&Secret{},
+		&SecretList{},
+		&SerializedReference{},
+		&RangeAllocation{},
+		&ConfigMap{},
+		&ConfigMapList{},
+	)
+
+	scheme.AddKnownTypes(GenericControlPlaneSchemeGroupVersion,
+		&Event{},
+		&EventList{},
+		&List{},
+		&LimitRange{},
+		&LimitRangeList{},
+		&ResourceQuota{},
+		&ResourceQuotaList{},
+		&Namespace{},
+		&NamespaceList{},
+		&ServiceAccount{},
+		&ServiceAccountList{},
+		&Secret{},
+		&SecretList{},
+		&SerializedReference{},
+		&RangeAllocation{},
+		&ConfigMap{},
+		&ConfigMapList{},
+	)
+
+	return nil
+}

pkg/apis/core/v1/register_generic_control_plane.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+	v1 "k8s.io/api/core/v1"
+)
+
+var (
+	localGenericControlPlaneSchemeBuilder = &v1.GenericControlPlaneSchemeBuilder
+	AddToControlPlaneScheme        = localGenericControlPlaneSchemeBuilder.AddToScheme
+)
+
+func init() {
+	// We only register manually written functions here. The registration of the
+	// generated functions takes place in the generated files. The separation
+	// makes the code compile even when the generated files are missing.
+	localGenericControlPlaneSchemeBuilder.Register(addDefaultingFuncs, addConversionFuncs, RegisterConversions)
+}

pkg/controller/clusterroleaggregation/clusterroleaggregation_controller.go

@@ -126,7 +126,7 @@ func (c *ClusterRoleAggregationController) syncClusterRole(ctx context.Context,
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
-		err = c.applyClusterRoles(ctx, sharedClusterRole.Name, newPolicyRules)
+		err = c.applyClusterRoles(ctx, sharedClusterRole, newPolicyRules)
 		if errors.IsUnsupportedMediaType(err) { // TODO: Remove this fallback at least one release after ServerSideApply GA
 			// When Server Side Apply is not enabled, fallback to Update. This is required when running
 			// 1.21 since api-server can be 1.20 during the upgrade/downgrade.
@@ -140,8 +140,8 @@ func (c *ClusterRoleAggregationController) syncClusterRole(ctx context.Context,
 	return err
 }
 
-func (c *ClusterRoleAggregationController) applyClusterRoles(ctx context.Context, name string, newPolicyRules []rbacv1.PolicyRule) error {
-	clusterRoleApply := rbacv1ac.ClusterRole(name).
+func (c *ClusterRoleAggregationController) applyClusterRoles(ctx context.Context, sharedClusterRole *rbacv1.ClusterRole, newPolicyRules []rbacv1.PolicyRule) error {
+	clusterRoleApply := rbacv1ac.ClusterRole(sharedClusterRole.Name).WithClusterName(sharedClusterRole.ClusterName).
 		WithRules(toApplyPolicyRules(newPolicyRules)...)
 
 	opts := metav1.ApplyOptions{FieldManager: "clusterrole-aggregation-controller", Force: true}