Add OpenAPI Schema support to virtual workspace framework

Signed-off-by: Nabarun Pal <[email protected]>

pkg/virtual/framework/dynamic/apidefinition/types.go

@@ -24,6 +24,7 @@ import (
  "k8s.io/apimachinery/pkg/runtime/schema"
  "k8s.io/apiserver/pkg/endpoints/handlers"
  "k8s.io/apiserver/pkg/registry/rest"
+ "k8s.io/kube-openapi/pkg/spec3"
 
  dynamiccontext "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/context"
  apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1"
@@ -34,6 +35,9 @@ type APIDefinition interface {
  // GetAPIResourceSchema returns the API schema this definition serves.
  GetAPIResourceSchema() *apisv1alpha1.APIResourceSchema
 
+ // GetOpenAPIV3Schema fetches OpenAPI V3 Schema
+ GetOpenAPIV3Spec() *spec3.OpenAPI
+
  // GetClusterName provides the name of the logical cluster where the resource specification comes from.
  GetClusterName() logicalcluster.Name
 

pkg/virtual/framework/dynamic/apiserver/apiserver.go

@@ -177,14 +177,12 @@ func (c completedConfig) New(virtualWorkspaceName string, delegationTarget gener
 
  s.GenericAPIServer.Handler.GoRestfulContainer.Add(discovery.NewLegacyRootAPIHandler(c.GenericConfig.DiscoveryAddresses, s.GenericAPIServer.Serializer, "/api").WebService())
 
+ openAPIHandler := newOpenAPIHandler(s.APISetRetriever, delegateHandler)
+
  s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler)
  s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler)
  s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/api/v1", crdHandler)
  s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/api/v1/", crdHandler)
-
- // TODO(david): plug OpenAPI if necessary. For now, according to the various virtual workspace use-cases,
- // it doesn't seem necessary.
- // Of course this requires using the --validate=false argument with some kubectl command like kubectl apply.
-
+ s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/openapi", openAPIHandler)
  return s, nil
 }

pkg/virtual/framework/dynamic/apiserver/discovery.go

@@ -17,6 +17,7 @@ limitations under the License.
 package apiserver
 
 import (
+ "context"
  "fmt"
  "net/http"
  "sort"
@@ -33,6 +34,7 @@ import (
  "k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
  "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
  "k8s.io/apiserver/pkg/registry/rest"
+ "k8s.io/kube-openapi/pkg/handler3"
  "k8s.io/kubernetes/pkg/controlplane/apiserver/miniaggregator"
 
  "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/apidefinition"
@@ -330,3 +332,86 @@ func sortGroupDiscoveryByKubeAwareVersion(gd []metav1.GroupVersionForDiscovery)
  return version.CompareKubeAwareVersionStrings(gd[i].Version, gd[j].Version) > 0
  })
 }
+
+type openAPIHandler struct {
+ apiSetRetriever apidefinition.APIDefinitionSetGetter
+ delegate http.Handler
+
+ openAPIV3Service *handler3.OpenAPIService
+}
+
+func newOpenAPIHandler(apiSetRetriever apidefinition.APIDefinitionSetGetter, delegate http.Handler) *openAPIHandler {
+ openAPIV3Service := handler3.NewOpenAPIService()
+
+ return &openAPIHandler{
+ apiSetRetriever: apiSetRetriever,
+ delegate: delegate,
+
+ openAPIV3Service: openAPIV3Service,
+ }
+}
+
+func (h *openAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ // handle /openapi -> do nothing
+ // handle /openapi/v2 -> do nothing for now. TODO: Implement. Actually, let OpenAPIV2 die. Ref: https://github.com/kcp-dev/kcp/pull/3059#discussion_r1424317153
+ // handle /openapi/v3 -> discovery endpoint
+ // handle /openapi/v3/apis/<group>/<version> -> serve OpenAPIV3Schema for <group>/<version>
+
+ pathParts := splitPath(r.URL.Path)
+
+ // if request doesn't start with /openapi return
+ // this check is a safety measure
+ if len(pathParts) == 1 && pathParts[0] != "openapi" {
+ h.delegate.ServeHTTP(w, r)
+ return
+ }
+
+ if len(pathParts) == 1 && pathParts[0] == "openapi" {
+ return
+ }
+
+ if len(pathParts) == 2 && pathParts[1] == "v2" { // handle /openapi/v2
+ // TODO: Implement OpenAPI V2 handler. Actually, let OpenAPIV2 die. Ref: https://github.com/kcp-dev/kcp/pull/3059#discussion_r1424317153
+ return
+ } else if len(pathParts) == 2 && pathParts[1] == "v3" { // handle /openapi/v3
+ if err := h.fetchAndUpdate(r.Context(), ""); err != nil {
+ responsewriters.ErrorNegotiated(
+ apierrors.NewInternalError(fmt.Errorf("unable to set: %w", err)),
+ errorCodecs, schema.GroupVersion{},
+ w, r)
+ }
+
+ h.openAPIV3Service.HandleDiscovery(w, r)
+ return
+ } else if len(pathParts) == 5 && pathParts[1] == "v3" && pathParts[2] == "apis" {
+ requestedGroup := pathParts[3]
+ if err := h.fetchAndUpdate(r.Context(), requestedGroup); err != nil {
+ responsewriters.ErrorNegotiated(
+ apierrors.NewInternalError(fmt.Errorf("unable to set: %w", err)),
+ errorCodecs, schema.GroupVersion{},
+ w, r)
+ }
+
+ h.openAPIV3Service.HandleGroupVersion(w, r)
+ } else {
+ h.delegate.ServeHTTP(w, r)
+ return
+ }
+}
+
+func (h *openAPIHandler) fetchAndUpdate(ctx context.Context, group string) error {
+ apiDomainKey := dyncamiccontext.APIDomainKeyFrom(ctx)
+ apiSet, _, err := h.apiSetRetriever.GetAPIDefinitionSet(ctx, apiDomainKey)
+ if err != nil {
+ return err
+ }
+
+ for gvr, apiDefinition := range apiSet {
+ if group == "" || (group != "" && group == gvr.Group) {
+ spec := apiDefinition.GetOpenAPIV3Spec()
+ h.openAPIV3Service.UpdateGroupVersion(gvr.Group, spec)
+ }
+ }
+
+ return nil
+}

pkg/virtual/framework/dynamic/apiserver/handler_test.go

@@ -39,6 +39,7 @@ import (
  apirequest "k8s.io/apiserver/pkg/endpoints/request"
  "k8s.io/apiserver/pkg/registry/rest"
  utilopenapi "k8s.io/apiserver/pkg/util/openapi"
+ "k8s.io/kube-openapi/pkg/spec3"
  "sigs.k8s.io/yaml"
 
  "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/apidefinition"
@@ -58,6 +59,7 @@ type mockedAPIDefinition struct {
  apiResourceSchema *apisv1alpha1.APIResourceSchema
  store rest.Storage
  subresourcesStores map[string]rest.Storage
+ openAPIV3Spec *spec3.OpenAPI
 }
 
 var _ apidefinition.APIDefinition = (*mockedAPIDefinition)(nil)
@@ -71,6 +73,9 @@ func (apiDef *mockedAPIDefinition) GetClusterName() logicalcluster.Name {
 func (apiDef *mockedAPIDefinition) GetStorage() rest.Storage {
  return apiDef.store
 }
+func (apiDef *mockedAPIDefinition) GetOpenAPIV3Spec() *spec3.OpenAPI {
+ return apiDef.openAPIV3Spec
+}
 func (apiDef *mockedAPIDefinition) GetSubResourceStorage(subresource string) rest.Storage {
  return apiDef.subresourcesStores[subresource]
 }

pkg/virtual/framework/dynamic/apiserver/serving_info.go

@@ -45,6 +45,7 @@ import (
  genericapiserver "k8s.io/apiserver/pkg/server"
  utilopenapi "k8s.io/apiserver/pkg/util/openapi"
  "k8s.io/klog/v2"
+ "k8s.io/kube-openapi/pkg/spec3"
  "k8s.io/kube-openapi/pkg/validation/spec"
 
  kcpfeatures "github.com/kcp-dev/kcp/pkg/features"
@@ -108,6 +109,17 @@ func CreateServingInfoFor(genericConfig genericapiserver.CompletedConfig, apiRes
  return nil, err
  }
 
+ openAPIV3Spec, err := buildOpenAPIV3(
+ apiResourceSchema,
+ apiResourceVersion,
+ builder.Options{
+ V2: false,
+ SkipFilterSchemaForKubectlOpenAPIV2Validation: true,
+ AllowNonStructural: false})
+ if err != nil {
+ return nil, err
+ }
+
  var modelsByGKV openapi.ModelsByGKV
 
  openAPIModels, err := utilopenapi.ToProtoModels(s)
@@ -302,6 +314,7 @@ func CreateServingInfoFor(genericConfig genericapiserver.CompletedConfig, apiRes
  requestScope: requestScope,
  statusRequestScope: &statusScope,
  logicalClusterName: logicalcluster.From(apiResourceSchema),
+ openAPIV3Spec: openAPIV3Spec,
  }
 
  return ret, nil
@@ -317,6 +330,8 @@ type servingInfo struct {
 
  requestScope *handlers.RequestScope
  statusRequestScope *handlers.RequestScope
+
+ openAPIV3Spec *spec3.OpenAPI
 }
 
 // Implement APIDefinition interface
@@ -348,6 +363,10 @@ func (apiDef *servingInfo) GetSubResourceRequestScope(subresource string) *handl
 func (apiDef *servingInfo) TearDown() {
 }
 
+func (apiDef *servingInfo) GetOpenAPIV3Spec() *spec3.OpenAPI {
+ return apiDef.openAPIV3Spec
+}
+
 var _ runtime.ObjectConvertor = nopConverter{}
 
 type nopConverter struct{}
@@ -371,8 +390,7 @@ func (u nopConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, valu
  return label, value, nil
 }
 
-// buildOpenAPIV2 builds OpenAPI v2 for the given apiResourceSpec.
-func buildOpenAPIV2(apiResourceSchema *apisv1alpha1.APIResourceSchema, apiResourceVersion *apisv1alpha1.APIResourceVersion, opts builder.Options) (*spec.Swagger, error) {
+func getCRDFromSchema(apiResourceSchema *apisv1alpha1.APIResourceSchema, apiResourceVersion *apisv1alpha1.APIResourceVersion) (*apiextensionsv1.CustomResourceDefinition, error) {
  openapiSchema, err := apiResourceVersion.GetSchema()
  if err != nil {
  return nil, err
@@ -393,9 +411,30 @@ func buildOpenAPIV2(apiResourceSchema *apisv1alpha1.APIResourceSchema, apiResour
  Scope: apiResourceSchema.Spec.Scope,
  },
  }
+
+ return crd, nil
+}
+
+// buildOpenAPIV2 builds OpenAPI v2 for the given apiResourceSpec.
+func buildOpenAPIV2(apiResourceSchema *apisv1alpha1.APIResourceSchema, apiResourceVersion *apisv1alpha1.APIResourceVersion, opts builder.Options) (*spec.Swagger, error) {
+ crd, err := getCRDFromSchema(apiResourceSchema, apiResourceVersion)
+ if err != nil {
+ return nil, err
+ }
+
  return builder.BuildOpenAPIV2(crd, apiResourceVersion.Name, opts)
 }
 
+// buildOpenAPIV3 builds OpenAPI v3 for the given apiResourceSpec.
+func buildOpenAPIV3(apiResourceSchema *apisv1alpha1.APIResourceSchema, apiResourceVersion *apisv1alpha1.APIResourceVersion, opts builder.Options) (*spec3.OpenAPI, error) {
+ crd, err := getCRDFromSchema(apiResourceSchema, apiResourceVersion)
+ if err != nil {
+ return nil, err
+ }
+
+ return builder.BuildOpenAPIV3(crd, apiResourceVersion.Name, opts)
+}
+
 func findAPIResourceVersion(schema *apisv1alpha1.APIResourceSchema, version string) (*apisv1alpha1.APIResourceVersion, bool) {
  for i := range schema.Spec.Versions {
  if vs := &schema.Spec.Versions[i]; vs.Name == version {