Skip to content

Commit 16622f3

Browse files
author
Yehudit Kerido
committed
feat(ws): Notebooks 2.0 // Backend // Backend can read data served by envtest
Signed-off-by: Yehudit Kerido <[email protected]>
1 parent 19eca50 commit 16622f3

File tree

9 files changed

+1336
-38
lines changed

9 files changed

+1336
-38
lines changed

workspaces/backend/Makefile

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ build: fmt vet swag ## Build backend binary.
8686
run: fmt vet swag ## Run a backend from your host.
8787
go run ./cmd/main.go --port=$(PORT)
8888

89+
.PHONY: run-envtest
90+
run-envtest: fmt vet prepare-envtest-assets ## Run envtest.
91+
go run ./cmd/main.go --enable-envtest --port=$(PORT)
92+
8993
# If you wish to build the manager image targeting other platforms you can use the --platform flag.
9094
# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
9195
# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
@@ -132,6 +136,11 @@ ENVTEST_VERSION ?= release-0.19
132136
GOLANGCI_LINT_VERSION ?= v1.61.0
133137
SWAGGER_VERSION ?= v1.16.4
134138

139+
.PHONY: prepare-envtest-assets
140+
prepare-envtest-assets: envtest ## Download K8s control plane binaries directly into ./bin/k8s/
141+
@echo ">>>> Downloading envtest Kubernetes control plane binaries to ./bin/k8s/..."
142+
$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=$(LOCALBIN)
143+
135144
.PHONY: SWAGGER
136145
SWAGGER: $(SWAGGER)
137146
$(SWAGGER): $(LOCALBIN)

workspaces/backend/cmd/main.go

Lines changed: 62 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,22 @@ package main
1818

1919
import (
2020
"flag"
21+
"fmt"
2122
"log/slog"
2223
"os"
24+
"path/filepath"
25+
stdruntime "runtime"
2326
"strconv"
2427

28+
"github.com/go-logr/logr"
29+
2530
ctrl "sigs.k8s.io/controller-runtime"
26-
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
2731

2832
application "github.com/kubeflow/notebooks/workspaces/backend/api"
2933
"github.com/kubeflow/notebooks/workspaces/backend/internal/auth"
3034
"github.com/kubeflow/notebooks/workspaces/backend/internal/config"
3135
"github.com/kubeflow/notebooks/workspaces/backend/internal/helper"
36+
"github.com/kubeflow/notebooks/workspaces/backend/internal/k8sclientfactory"
3237
"github.com/kubeflow/notebooks/workspaces/backend/internal/server"
3338
)
3439

@@ -47,7 +52,7 @@ import (
4752
// @consumes application/json
4853
// @produces application/json
4954

50-
func main() {
55+
func run() error {
5156
// Define command line flags
5257
cfg := &config.EnvConfig{}
5358
flag.IntVar(&cfg.Port,
@@ -93,44 +98,61 @@ func main() {
9398
"Key of request header containing user groups",
9499
)
95100

96-
// Initialize the logger
97-
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
101+
var enableEnvTest bool
102+
flag.BoolVar(&enableEnvTest,
103+
"enable-envtest",
104+
getEnvAsBool("ENABLE_ENVTEST", false),
105+
"Enable envtest for local development without a real k8s cluster",
106+
)
107+
flag.Parse()
98108

99-
// Build the Kubernetes client configuration
100-
kubeconfig, err := ctrl.GetConfig()
101-
if err != nil {
102-
logger.Error("failed to get Kubernetes config", "error", err)
103-
os.Exit(1)
104-
}
105-
kubeconfig.QPS = float32(cfg.ClientQPS)
106-
kubeconfig.Burst = cfg.ClientBurst
109+
// Initialize the logger
110+
slogTextHandler := slog.NewTextHandler(os.Stdout, nil)
111+
logger := slog.New(slogTextHandler)
107112

108113
// Build the Kubernetes scheme
109114
scheme, err := helper.BuildScheme()
110115
if err != nil {
111116
logger.Error("failed to build Kubernetes scheme", "error", err)
112-
os.Exit(1)
117+
return err
113118
}
114119

115-
// Create the controller manager
116-
mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{
117-
Scheme: scheme,
118-
Metrics: metricsserver.Options{
119-
BindAddress: "0", // disable metrics serving
120-
},
121-
HealthProbeBindAddress: "0", // disable health probe serving
122-
LeaderElection: false,
123-
})
120+
// Defining CRD's path
121+
crdPath := os.Getenv("CRD_PATH")
122+
if crdPath == "" {
123+
_, currentFile, _, ok := stdruntime.Caller(0)
124+
if !ok {
125+
logger.Info("Failed to get current file path using stdruntime.Caller")
126+
}
127+
testFileDir := filepath.Dir(currentFile)
128+
crdPath = filepath.Join(testFileDir, "..", "..", "controller", "config", "crd", "bases")
129+
logger.Info("CRD_PATH not set, using guessed default", "path", crdPath)
130+
}
131+
132+
// ctx creates a context that listens for OS signals (e.g., SIGINT, SIGTERM) for graceful shutdown.
133+
ctx := ctrl.SetupSignalHandler()
134+
135+
logrlogger := logr.FromSlogHandler(slogTextHandler)
136+
137+
// factory creates a new Kubernetes client factory, configured for envtest if enabled.
138+
factory := k8sclientfactory.NewClientFactory(logrlogger, scheme, enableEnvTest, []string{crdPath}, cfg)
139+
140+
// Create the controller manager, build Kubernetes client configuration
141+
// envtestCleanupFunc is a function to clean envtest if it was created, otherwise it's an empty function.
142+
mgr, kubeconfig, envtestCleanupFunc, err := factory.GetManagerAndConfig(ctx)
143+
defer envtestCleanupFunc()
124144
if err != nil {
125-
logger.Error("unable to create manager", "error", err)
126-
os.Exit(1)
145+
logger.Error("Failed to get Kubernetes manager/config from factory", "error", err)
146+
return err
127147
}
148+
kubeconfig.QPS = float32(cfg.ClientQPS)
149+
kubeconfig.Burst = cfg.ClientBurst
128150

129151
// Create the request authenticator
130152
reqAuthN, err := auth.NewRequestAuthenticator(cfg.UserIdHeader, cfg.UserIdPrefix, cfg.GroupsHeader)
131153
if err != nil {
132154
logger.Error("failed to create request authenticator", "error", err)
133-
os.Exit(1)
155+
return err
134156
}
135157

136158
// Create the request authorizer
@@ -143,22 +165,30 @@ func main() {
143165
app, err := application.NewApp(cfg, logger, mgr.GetClient(), mgr.GetScheme(), reqAuthN, reqAuthZ)
144166
if err != nil {
145167
logger.Error("failed to create app", "error", err)
146-
os.Exit(1)
168+
return err
147169
}
148170
svr, err := server.NewServer(app, logger)
149171
if err != nil {
150172
logger.Error("failed to create server", "error", err)
151-
os.Exit(1)
173+
return err
152174
}
153175
if err := svr.SetupWithManager(mgr); err != nil {
154176
logger.Error("failed to setup server with manager", "error", err)
155-
os.Exit(1)
177+
return err
178+
}
179+
180+
logger.Info("Starting manager...")
181+
if err := mgr.Start(ctx); err != nil {
182+
logger.Error("Problem running manager", "error", err)
183+
return err
156184
}
157185

158-
// Start the controller manager
159-
logger.Info("starting manager")
160-
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
161-
logger.Error("problem running manager", "error", err)
186+
return nil
187+
}
188+
189+
func main() {
190+
if err := run(); err != nil {
191+
fmt.Fprintf(os.Stderr, "Application run failed: %v\n", err)
162192
os.Exit(1)
163193
}
164194
}

workspaces/backend/go.mod

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,10 @@ require (
3333
github.com/felixge/httpsnoop v1.0.4 // indirect
3434
github.com/fsnotify/fsnotify v1.7.0 // indirect
3535
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
36-
github.com/go-logr/logr v1.4.2 // indirect
3736
github.com/go-logr/stdr v1.2.2 // indirect
3837
github.com/go-logr/zapr v1.3.0 // indirect
3938
github.com/go-openapi/jsonpointer v0.21.0 // indirect
4039
github.com/go-openapi/jsonreference v0.21.0 // indirect
41-
github.com/go-openapi/spec v0.21.0 // indirect
4240
github.com/go-openapi/swag v0.23.0 // indirect
4341
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
4442
github.com/gogo/protobuf v1.3.2 // indirect
@@ -55,7 +53,6 @@ require (
5553
github.com/inconshreveable/mousetrap v1.1.0 // indirect
5654
github.com/josharian/intern v1.0.0 // indirect
5755
github.com/json-iterator/go v1.1.12 // indirect
58-
github.com/mailru/easyjson v0.9.0 // indirect
5956
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
6057
github.com/modern-go/reflect2 v1.0.2 // indirect
6158
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -67,7 +64,6 @@ require (
6764
github.com/spf13/cobra v1.8.1 // indirect
6865
github.com/spf13/pflag v1.0.5 // indirect
6966
github.com/stoewer/go-strcase v1.2.0 // indirect
70-
github.com/swaggo/files/v2 v2.0.2 // indirect
7167
github.com/x448/float16 v0.8.4 // indirect
7268
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
7369
go.opentelemetry.io/otel v1.28.0 // indirect
@@ -87,7 +83,6 @@ require (
8783
golang.org/x/term v0.29.0 // indirect
8884
golang.org/x/text v0.22.0 // indirect
8985
golang.org/x/time v0.3.0 // indirect
90-
golang.org/x/tools v0.30.0 // indirect
9186
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
9287
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
9388
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
@@ -103,5 +98,13 @@ require (
10398
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
10499
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
105100
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
106-
sigs.k8s.io/yaml v1.4.0 // indirect
101+
)
102+
103+
require (
104+
github.com/go-logr/logr v1.4.2
105+
github.com/go-openapi/spec v0.21.0 // indirect
106+
github.com/mailru/easyjson v0.9.0 // indirect
107+
github.com/swaggo/files/v2 v2.0.2 // indirect
108+
golang.org/x/tools v0.30.0 // indirect
109+
sigs.k8s.io/yaml v1.4.0
107110
)
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package k8sclientfactory
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
"github.com/go-logr/logr"
24+
"k8s.io/apimachinery/pkg/runtime"
25+
"sigs.k8s.io/controller-runtime/pkg/envtest"
26+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
27+
28+
"github.com/kubeflow/notebooks/workspaces/backend/internal/config"
29+
30+
"k8s.io/client-go/rest"
31+
ctrl "sigs.k8s.io/controller-runtime"
32+
"sigs.k8s.io/controller-runtime/pkg/client"
33+
34+
"github.com/kubeflow/notebooks/workspaces/backend/localdev"
35+
)
36+
37+
// ClientFactory responsible for providing a Kubernetes client and manager
38+
type ClientFactory struct {
39+
useEnvtest bool
40+
crdPaths []string
41+
logger logr.Logger
42+
scheme *runtime.Scheme
43+
clientQPS float64
44+
clientBurst int
45+
}
46+
47+
// NewClientFactory creates a new factory
48+
func NewClientFactory(
49+
logger logr.Logger,
50+
scheme *runtime.Scheme,
51+
useEnvtest bool,
52+
crdPaths []string,
53+
appCfg *config.EnvConfig, // Pass the application config here
54+
) *ClientFactory {
55+
return &ClientFactory{
56+
useEnvtest: useEnvtest,
57+
crdPaths: crdPaths,
58+
logger: logger.WithName("k8s-client-factory"),
59+
scheme: scheme,
60+
clientQPS: appCfg.ClientQPS,
61+
clientBurst: appCfg.ClientBurst,
62+
}
63+
}
64+
65+
// GetManagerAndConfig returns a configured Kubernetes manager and its rest.Config
66+
// It also returns a cleanup function for envtest if it was started.
67+
func (f *ClientFactory) GetManagerAndConfig(ctx context.Context) (ctrl.Manager, *rest.Config, func(), error) {
68+
var mgr ctrl.Manager
69+
var cfg *rest.Config
70+
var err error
71+
var cleanupFunc func() = func() {} // No-op cleanup by default
72+
73+
if f.useEnvtest {
74+
f.logger.Info("Using envtest mode: setting up local Kubernetes environment...")
75+
var testEnvInstance *envtest.Environment
76+
77+
cfg, mgr, testEnvInstance, err = localdev.StartLocalDevEnvironment(ctx, f.crdPaths, f.scheme)
78+
if err != nil {
79+
return nil, nil, nil, fmt.Errorf("could not start local dev environment: %w", err)
80+
}
81+
f.logger.Info("Local dev K8s API (envtest) is ready.", "host", cfg.Host)
82+
83+
if testEnvInstance != nil {
84+
cleanupFunc = func() {
85+
f.logger.Info("Stopping envtest environment...")
86+
if err := testEnvInstance.Stop(); err != nil {
87+
f.logger.Error(err, "Failed to stop envtest environment")
88+
}
89+
}
90+
}
91+
} else {
92+
f.logger.Info("Using real cluster mode: connecting to existing Kubernetes cluster...")
93+
cfg, err = ctrl.GetConfig()
94+
if err != nil {
95+
return nil, nil, nil, fmt.Errorf("failed to get Kubernetes config: %w", err)
96+
}
97+
f.logger.Info("Successfully connected to existing Kubernetes cluster.")
98+
99+
cfg.QPS = float32(f.clientQPS)
100+
cfg.Burst = f.clientBurst
101+
mgr, err = ctrl.NewManager(cfg, ctrl.Options{
102+
Scheme: f.scheme,
103+
Metrics: metricsserver.Options{
104+
BindAddress: "0", // disable metrics serving
105+
},
106+
HealthProbeBindAddress: "0", // disable health probe serving
107+
LeaderElection: false,
108+
})
109+
if err != nil {
110+
return nil, nil, nil, fmt.Errorf("unable to create manager for real cluster: %w", err)
111+
}
112+
f.logger.Info("Successfully configured manager for existing Kubernetes cluster.")
113+
}
114+
return mgr, cfg, cleanupFunc, nil
115+
}
116+
117+
// GetClient returns just the client.Client (useful if manager lifecycle is handled elsewhere or already started)
118+
func (f *ClientFactory) GetClient(ctx context.Context) (client.Client, func(), error) {
119+
mgr, _, cleanup, err := f.GetManagerAndConfig(ctx)
120+
if err != nil {
121+
return nil, cleanup, err
122+
}
123+
return mgr.GetClient(), cleanup, nil
124+
}

0 commit comments

Comments
 (0)