Merge rancher/steve branch 'release/v2.9' into cnrancher release/v2.9…

…-ent

go.mod

@@ -13,6 +13,7 @@ replace (
 require (
 	github.com/adrg/xdg v0.4.0
 	github.com/golang/mock v1.6.0
+	github.com/golang/protobuf v1.5.4
 	github.com/google/gnostic-models v0.6.8
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.1
@@ -32,17 +33,20 @@ require (
 	github.com/urfave/cli/v2 v2.27.1
 	golang.org/x/sync v0.7.0
 	gopkg.in/yaml.v3 v3.0.1
+	helm.sh/helm/v3 v3.13.0
 	k8s.io/api v0.30.1
 	k8s.io/apiextensions-apiserver v0.30.1
 	k8s.io/apimachinery v0.30.1
 	k8s.io/apiserver v0.30.1
 	k8s.io/client-go v0.30.1
+	k8s.io/helm v2.17.0+incompatible
 	k8s.io/klog v1.0.0
 	k8s.io/kube-aggregator v0.30.1
 	k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3
 )
 
 require (
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
@@ -60,13 +64,12 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -120,8 +122,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -378,12 +380,14 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+helm.sh/helm/v3 v3.13.0 h1:XPJKIU30K4JTQ6VX/6e0hFAmEIonYa8E7wx5aqv4xOc=
+helm.sh/helm/v3 v3.13.0/go.mod h1:2PBEKsMWKLVZTojUOqMS3Eadv5mP43FBWrRgLNkNm9Y=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8=
@@ -402,6 +406,8 @@ k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
 k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ=
 k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao=
+k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=

main.go

@@ -34,7 +34,7 @@ func main() {
 func run(_ *cli.Context) error {
 	ctx := signals.SetupSignalContext()
 	debugconfig.MustSetupDebug()
-	s, err := config.ToServer(ctx, false)
+	s, err := config.ToServer(ctx, debugconfig.SQLCache)
 	if err != nil {
 		return err
 	}

pkg/debug/cli.go

@@ -13,6 +13,7 @@ import (
 type Config struct {
 	Debug      bool
 	DebugLevel int
+	SQLCache   bool
 }
 
 func (c *Config) MustSetupDebug() {
@@ -54,6 +55,10 @@ func Flags(config *Config) []cli.Flag {
 			Value:       7,
 			Destination: &config.DebugLevel,
 		},
+		cli.BoolFlag{
+			Name:        "sql-cache",
+			Destination: &config.SQLCache,
+		},
 	}
 }
 
@@ -68,5 +73,9 @@ func FlagsV2(config *Config) []cliv2.Flag {
 			Value:       7,
 			Destination: &config.DebugLevel,
 		},
+		&cliv2.BoolFlag{
+			Name:        "sql-cache",
+			Destination: &config.SQLCache,
+		},
 	}
 }

pkg/resources/common/formatter.go

@@ -23,19 +23,18 @@ import (
 func DefaultTemplate(clientGetter proxy.ClientGetter,
 	summaryCache *summarycache.SummaryCache,
 	asl accesscontrol.AccessSetLookup,
-	namespaceCache corecontrollers.NamespaceCache,
-	sqlCache bool) schema.Template {
+	namespaceCache corecontrollers.NamespaceCache) schema.Template {
 	return schema.Template{
 		Store:     metricsStore.NewMetricsStore(proxy.NewProxyStore(clientGetter, summaryCache, asl, namespaceCache)),
-		Formatter: formatter(summaryCache, sqlCache),
+		Formatter: formatter(summaryCache),
 	}
 }
 
 // DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally.
-func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache, sqlCache bool) schema.Template {
+func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache) schema.Template {
 	return schema.Template{
 		Store:     store,
-		Formatter: formatter(summaryCache, sqlCache),
+		Formatter: formatter(summaryCache),
 	}
 }
 
@@ -72,7 +71,7 @@ func selfLink(gvr schema2.GroupVersionResource, meta metav1.Object) (prefix stri
 	return buf.String()
 }
 
-func formatter(summarycache *summarycache.SummaryCache, sqlCache bool) types.Formatter {
+func formatter(summarycache *summarycache.SummaryCache) types.Formatter {
 	return func(request *types.APIRequest, resource *types.RawResource) {
 		if resource.Schema == nil {
 			return
@@ -105,20 +104,19 @@ func formatter(summarycache *summarycache.SummaryCache, sqlCache bool) types.For
 		}
 
 		if unstr, ok := resource.APIObject.Object.(*unstructured.Unstructured); ok {
-			if !sqlCache {
-				// with the sql cache, these were already added by the indexer
-				s, rel := summarycache.SummaryAndRelationship(unstr)
-				data.PutValue(unstr.Object, map[string]interface{}{
-					"name":          s.State,
-					"error":         s.Error,
-					"transitioning": s.Transitioning,
-					"message":       strings.Join(s.Message, ":"),
-				}, "metadata", "state")
-				data.PutValue(unstr.Object, rel, "metadata", "relationships")
-
-				summary.NormalizeConditions(unstr)
+			// with the sql cache, these were already added by the indexer. However, the sql cache
+			// is only used for lists, so we need to re-add here for get/watch
+			s, rel := summarycache.SummaryAndRelationship(unstr)
+			data.PutValue(unstr.Object, map[string]interface{}{
+				"name":          s.State,
+				"error":         s.Error,
+				"transitioning": s.Transitioning,
+				"message":       strings.Join(s.Message, ":"),
+			}, "metadata", "state")
+			data.PutValue(unstr.Object, rel, "metadata", "relationships")
+
+			summary.NormalizeConditions(unstr)
 
-			}
 			includeFields(request, unstr)
 			excludeFields(request, unstr)
 			excludeValues(request, unstr)

pkg/resources/formatters/formatter.go

@@ -1,17 +1,59 @@
 package formatters
 
 import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/pkg/errors"
 	"github.com/rancher/apiserver/pkg/types"
 	"github.com/rancher/norman/types/convert"
+	"github.com/rancher/wrangler/v3/pkg/data"
+	"github.com/sirupsen/logrus"
+	"helm.sh/helm/v3/pkg/release"
+	rspb "k8s.io/helm/pkg/proto/hapi/release"
 )
 
-func DropHelmData(request *types.APIRequest, resource *types.RawResource) {
-	data := resource.APIObject.Data()
-	if data.String("metadata", "labels", "owner") == "helm" ||
-		data.String("metadata", "labels", "OWNER") == "TILLER" {
-		if data.String("data", "release") != "" {
-			delete(data.Map("data"), "release")
+var (
+	ErrNotHelmRelease = errors.New("not helm release") // error for when it's not a helm release
+	magicGzip         = []byte{0x1f, 0x8b, 0x08}       // gzip magic header
+)
+
+func HandleHelmData(request *types.APIRequest, resource *types.RawResource) {
+	objData := resource.APIObject.Data()
+	if q := request.Query.Get("includeHelmData"); q == "true" {
+		var helmReleaseData string
+		if resource.Type == "secret" {
+			b, err := base64.StdEncoding.DecodeString(objData.String("data", "release"))
+			if err != nil {
+				return
+			}
+			helmReleaseData = string(b)
+		} else {
+			helmReleaseData = objData.String("data", "release")
+		}
+		if objData.String("metadata", "labels", "owner") == "helm" {
+			rl, err := decodeHelm3(helmReleaseData)
+			if err != nil {
+				logrus.Errorf("Failed to decode helm3 release data: %v", err)
+				return
+			}
+			objData.SetNested(rl, "data", "release")
+		}
+		if objData.String("metadata", "labels", "OWNER") == "TILLER" {
+			rl, err := decodeHelm2(helmReleaseData)
+			if err != nil {
+				logrus.Errorf("Failed to decode helm2 release data: %v", err)
+				return
+			}
+			objData.SetNested(rl, "data", "release")
 		}
+
+	} else {
+		DropHelmData(objData)
 	}
 }
 
@@ -22,3 +64,78 @@ func Pod(request *types.APIRequest, resource *types.RawResource) {
 		data.SetNested(convert.LowerTitle(fields[2]), "metadata", "state", "name")
 	}
 }
+
+// decodeHelm3 receives a helm3 release data string, decodes the string data using the standard base64 library
+// and unmarshals the data into release.Release struct to return it.
+func decodeHelm3(data string) (*release.Release, error) {
+	b, err := base64.StdEncoding.DecodeString(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// Data is too small to be helm 3 release object
+	if len(b) <= 3 {
+		return nil, ErrNotHelmRelease
+	}
+
+	// For backwards compatibility with releases that were stored before
+	// compression was introduced we skip decompression if the
+	// gzip magic header is not found
+	if bytes.Equal(b[0:3], magicGzip) {
+		r, err := gzip.NewReader(bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+		b2, err := io.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+		b = b2
+	}
+
+	var rls release.Release
+	// unmarshal release object bytes
+	if err := json.Unmarshal(b, &rls); err != nil {
+		return nil, err
+	}
+	return &rls, nil
+}
+
+// decodeHelm2 receives a helm2 release data and returns the corresponding helm2 release proto struct
+func decodeHelm2(data string) (*rspb.Release, error) {
+	b, err := base64.StdEncoding.DecodeString(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// For backwards compatibility with releases that were stored before
+	// compression was introduced we skip decompression if the
+	// gzip magic header is not found
+	if bytes.Equal(b[0:3], magicGzip) {
+		r, err := gzip.NewReader(bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+		b2, err := io.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+		b = b2
+	}
+
+	var rls rspb.Release
+	// unmarshal protobuf bytes
+	if err := proto.Unmarshal(b, &rls); err != nil {
+		return nil, err
+	}
+	return &rls, nil
+}
+
+func DropHelmData(data data.Object) {
+	if data.String("metadata", "labels", "owner") == "helm" ||
+		data.String("metadata", "labels", "OWNER") == "TILLER" {
+		if data.String("data", "release") != "" {
+			delete(data.Map("data"), "release")
+		}
+	}
+}