wip: exporter: new gateway exporter

Signed-off-by: Justin Chadwell <me@jedevc.com>

Author: jedevc

.gitattributes

@@ -0,0 +1,2 @@
+*.pb.go linguist-generated=true
+*.pb.go -diff

.github/workflows/buildkit.yml

@@ -112,7 +112,7 @@ jobs:
       - binaries
     with:
       cache_scope: build-integration-tests
-      pkgs: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend
+      pkgs: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend ./exporter
       kinds: integration
       codecov_flags: core
       includes: |

TODO.md

@@ -0,0 +1,13 @@
+# TODO
+
+<https://github.com/moby/buildkit/issues/3037>
+
+- [x] Basic tests for current behavior
+- [x] exportprovider
+- [ ] Double release errors
+- XXXs
+- [x] Attestation support
+
+## Follow-ups
+
+- [ ] Secrets

api/services/control/control.pb.go

@@ -244,10 +244,20 @@ message BuildResultInfo {
 	map<int64, Descriptor> Results = 3;
 }
 
+enum ExporterTarget {
+	UNKNOWN = 0;
+	NONE = 1;
+	FILE = 2;
+	DIRECTORY = 3;
+	STORE = 4;
+}
+
 // Exporter describes the output exporter
 message Exporter {
 	// Type identifies the exporter
 	string Type = 1;
 	// Attrs specifies exporter configuration
 	map<string, string> Attrs = 2;
+	// Target indicates the target type of the exporter
+	ExporterTarget Target = 3;
 }

api/services/control/control.proto

@@ -0,0 +1,39 @@
+package moby_buildkit_v1 //nolint:revive,staticcheck
+
+import (
+	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+)
+
+func ExporterTargetFromPB(target ExporterTarget) exptypes.ExporterTarget {
+	switch target {
+	case ExporterTarget_UNKNOWN:
+		return exptypes.ExporterTargetUnknown
+	case ExporterTarget_NONE:
+		return exptypes.ExporterTargetNone
+	case ExporterTarget_FILE:
+		return exptypes.ExporterTargetFile
+	case ExporterTarget_DIRECTORY:
+		return exptypes.ExporterTargetDirectory
+	case ExporterTarget_STORE:
+		return exptypes.ExporterTargetStore
+	default:
+		return exptypes.ExporterTargetUnknown
+	}
+}
+
+func ExporterTargetToPB(target exptypes.ExporterTarget) ExporterTarget {
+	switch target {
+	case exptypes.ExporterTargetUnknown:
+		return ExporterTarget_UNKNOWN
+	case exptypes.ExporterTargetNone:
+		return ExporterTarget_NONE
+	case exptypes.ExporterTargetFile:
+		return ExporterTarget_FILE
+	case exptypes.ExporterTargetDirectory:
+		return ExporterTarget_DIRECTORY
+	case exptypes.ExporterTargetStore:
+		return ExporterTarget_STORE
+	default:
+		return ExporterTarget_UNKNOWN
+	}
+}

api/services/control/control_vtproto.pb.go

@@ -9,12 +9,14 @@ import (
 	"github.com/moby/buildkit/frontend/gateway/grpcclient"
 	gatewayapi "github.com/moby/buildkit/frontend/gateway/pb"
 	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/exporter"
+	"github.com/moby/buildkit/session/exporter/exporterprovider"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 )
 
-func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
+func (c *Client) BuildExport(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, exportFunc gateway.ExportFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
 	defer func() {
 		if statusChan != nil {
 			close(statusChan)
@@ -42,21 +44,35 @@ func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildF
 		})
 	}
 
+	var g grpcclient.GrpcClient
+	if exportFunc != nil {
+		opt.EnableSessionExporter = true
+		exporter := exporterprovider.New(func(ctx context.Context, _ map[string][]byte, _ []string) ([]*exporter.ExporterRequest, error) {
+			if g == nil {
+				return nil, errors.New("no build session found for export")
+			}
+			return nil, g.Export(ctx, c.conn, exportFunc)
+		})
+		opt.Session = append(opt.Session, exporter)
+	}
+
 	cb := func(ref string, s *session.Session, opts map[string]string) error {
 		if feOpts == nil {
 			feOpts = map[string]string{}
 		}
 		maps.Copy(feOpts, opts)
 		gwClient := c.gatewayClientForBuild(ref)
-		g, err := grpcclient.New(ctx, feOpts, s.ID(), product, gwClient, gworkers)
+		var err error
+		g, err = grpcclient.New(ctx, feOpts, s.ID(), product, gwClient, gworkers)
 		if err != nil {
 			return err
 		}
 
 		caps := g.BuildOpts().Caps
 		gwClient.caps = &caps
 
-		if err := g.Run(ctx, buildFunc); err != nil {
+		err = g.Build(ctx, buildFunc)
+		if err != nil {
 			return errors.Wrap(err, "failed to run Build function")
 		}
 		return nil
@@ -65,6 +81,10 @@ func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildF
 	return c.solve(ctx, nil, cb, opt, statusChan)
 }
 
+func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
+	return c.BuildExport(ctx, opt, product, buildFunc, nil, statusChan)
+}
+
 func (c *Client) gatewayClientForBuild(buildid string) *gatewayClientForBuild {
 	g := gatewayapi.NewLLBBridgeClient(c.conn)
 	return &gatewayClientForBuild{
@@ -138,6 +158,11 @@ func (g *gatewayClientForBuild) Evaluate(ctx context.Context, in *gatewayapi.Eva
 	return g.gateway.Evaluate(ctx, in, opts...)
 }
 
+func (g *gatewayClientForBuild) GetRemote(ctx context.Context, in *gatewayapi.GetRemoteRequest, opts ...grpc.CallOption) (*gatewayapi.GetRemoteResponse, error) {
+	ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
+	return g.gateway.GetRemote(ctx, in, opts...)
+}
+
 func (g *gatewayClientForBuild) Ping(ctx context.Context, in *gatewayapi.PingRequest, opts ...grpc.CallOption) (*gatewayapi.PongResponse, error) {
 	ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
 	return g.gateway.Ping(ctx, in, opts...)
@@ -148,6 +173,11 @@ func (g *gatewayClientForBuild) Return(ctx context.Context, in *gatewayapi.Retur
 	return g.gateway.Return(ctx, in, opts...)
 }
 
+func (g *gatewayClientForBuild) GetReturn(ctx context.Context, in *gatewayapi.GetReturnRequest, opts ...grpc.CallOption) (*gatewayapi.GetReturnResponse, error) {
+	ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
+	return g.gateway.GetReturn(ctx, in, opts...)
+}
+
 func (g *gatewayClientForBuild) Inputs(ctx context.Context, in *gatewayapi.InputsRequest, opts ...grpc.CallOption) (*gatewayapi.InputsResponse, error) {
 	if g.caps != nil {
 		if err := g.caps.Supports(gatewayapi.CapFrontendInputs); err != nil {

api/services/control/converters.go

@@ -1,9 +1,10 @@
 package client
 
 const (
-	ExporterImage  = "image"
-	ExporterLocal  = "local"
-	ExporterTar    = "tar"
-	ExporterOCI    = "oci"
-	ExporterDocker = "docker"
+	ExporterImage   = "image"
+	ExporterLocal   = "local"
+	ExporterTar     = "tar"
+	ExporterOCI     = "oci"
+	ExporterDocker  = "docker"
+	ExporterGateway = "gateway"
 )

client/build.go

@@ -168,6 +168,9 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 				if supportFile && supportStore {
 					return nil, errors.Errorf("both file and store output is not supported by %s exporter", ex.Type)
 				}
+			case ExporterGateway:
+				supportFile = ex.Output != nil
+				supportDir = ex.OutputDir != ""
 			}
 			if !supportFile && ex.Output != nil {
 				return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
@@ -274,9 +277,24 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 				exportDeprecated = exp.Type
 				exportAttrDeprecated = exp.Attrs
 			}
+			target := controlapi.ExporterTarget_NONE
+			switch {
+			case exp.Output != nil:
+				target = controlapi.ExporterTarget_FILE
+			case exp.OutputDir != "":
+				switch exp.Type {
+				case ExporterOCI, ExporterDocker:
+					target = controlapi.ExporterTarget_STORE
+				default:
+					target = controlapi.ExporterTarget_DIRECTORY
+				}
+			case exp.OutputStore != nil:
+				target = controlapi.ExporterTarget_STORE
+			}
 			exports = append(exports, &controlapi.Exporter{
-				Type:  exp.Type,
-				Attrs: exp.Attrs,
+				Type:   exp.Type,
+				Attrs:  exp.Attrs,
+				Target: target,
 			})
 		}