Skip to content

Commit 9a0d0c7

Browse files
committed
wip: exporter: new gateway exporter
Signed-off-by: Justin Chadwell <me@jedevc.com>
1 parent 877093a commit 9a0d0c7

File tree

48 files changed

+4338
-792
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+4338
-792
lines changed

.gitattributes

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.pb.go linguist-generated=true
2+
*.pb.go -diff

.github/workflows/buildkit.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ jobs:
112112
- binaries
113113
with:
114114
cache_scope: build-integration-tests
115-
pkgs: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend
115+
pkgs: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend ./exporter
116116
kinds: integration
117117
codecov_flags: core
118118
includes: |

api/services/control/control.pb.go

Lines changed: 200 additions & 127 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

api/services/control/control.proto

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,10 +244,20 @@ message BuildResultInfo {
244244
map<int64, Descriptor> Results = 3;
245245
}
246246

247+
enum ExporterTarget {
248+
UNKNOWN = 0;
249+
NONE = 1;
250+
FILE = 2;
251+
DIRECTORY = 3;
252+
STORE = 4;
253+
}
254+
247255
// Exporter describes the output exporter
248256
message Exporter {
249257
// Type identifies the exporter
250258
string Type = 1;
251259
// Attrs specifies exporter configuration
252260
map<string, string> Attrs = 2;
261+
// Target indicates the target type of the exporter
262+
ExporterTarget Target = 3;
253263
}

api/services/control/control_vtproto.pb.go

Lines changed: 31 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

api/services/control/converters.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package moby_buildkit_v1 //nolint:revive,staticcheck
2+
3+
import (
4+
"github.com/moby/buildkit/exporter/containerimage/exptypes"
5+
)
6+
7+
func ExporterTargetFromPB(target ExporterTarget) exptypes.ExporterTarget {
8+
switch target {
9+
case ExporterTarget_UNKNOWN:
10+
return exptypes.ExporterTargetUnknown
11+
case ExporterTarget_NONE:
12+
return exptypes.ExporterTargetNone
13+
case ExporterTarget_FILE:
14+
return exptypes.ExporterTargetFile
15+
case ExporterTarget_DIRECTORY:
16+
return exptypes.ExporterTargetDirectory
17+
case ExporterTarget_STORE:
18+
return exptypes.ExporterTargetStore
19+
default:
20+
return exptypes.ExporterTargetUnknown
21+
}
22+
}
23+
24+
func ExporterTargetToPB(target exptypes.ExporterTarget) ExporterTarget {
25+
switch target {
26+
case exptypes.ExporterTargetUnknown:
27+
return ExporterTarget_UNKNOWN
28+
case exptypes.ExporterTargetNone:
29+
return ExporterTarget_NONE
30+
case exptypes.ExporterTargetFile:
31+
return ExporterTarget_FILE
32+
case exptypes.ExporterTargetDirectory:
33+
return ExporterTarget_DIRECTORY
34+
case exptypes.ExporterTargetStore:
35+
return ExporterTarget_STORE
36+
default:
37+
return ExporterTarget_UNKNOWN
38+
}
39+
}

client/build.go

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"maps"
66

77
"github.com/moby/buildkit/client/buildid"
8+
"github.com/moby/buildkit/exporter/containerimage/exptypes"
89
gateway "github.com/moby/buildkit/frontend/gateway/client"
910
"github.com/moby/buildkit/frontend/gateway/grpcclient"
1011
gatewayapi "github.com/moby/buildkit/frontend/gateway/pb"
@@ -14,7 +15,7 @@ import (
1415
"google.golang.org/grpc"
1516
)
1617

17-
func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
18+
func (c *Client) BuildExport(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, exportFunc gateway.ExportFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
1819
defer func() {
1920
if statusChan != nil {
2021
close(statusChan)
@@ -56,15 +57,29 @@ func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildF
5657
caps := g.BuildOpts().Caps
5758
gwClient.caps = &caps
5859

59-
if err := g.Run(ctx, buildFunc); err != nil {
60+
result, err := g.Build(ctx, buildFunc)
61+
if err != nil {
6062
return errors.Wrap(err, "failed to run Build function")
6163
}
64+
if exportFunc != nil {
65+
handle := exptypes.ExportHandle{
66+
Target: exptypes.ExporterTargetUnknown,
67+
Conn: c.conn,
68+
}
69+
if err := exportFunc(ctx, g, handle, result); err != nil {
70+
return errors.Wrap(err, "failed to run Export function")
71+
}
72+
}
6273
return nil
6374
}
6475

6576
return c.solve(ctx, nil, cb, opt, statusChan)
6677
}
6778

79+
func (c *Client) Build(ctx context.Context, opt SolveOpt, product string, buildFunc gateway.BuildFunc, statusChan chan *SolveStatus) (*SolveResponse, error) {
80+
return c.BuildExport(ctx, opt, product, buildFunc, nil, statusChan)
81+
}
82+
6883
func (c *Client) gatewayClientForBuild(buildid string) *gatewayClientForBuild {
6984
g := gatewayapi.NewLLBBridgeClient(c.conn)
7085
return &gatewayClientForBuild{
@@ -138,6 +153,11 @@ func (g *gatewayClientForBuild) Evaluate(ctx context.Context, in *gatewayapi.Eva
138153
return g.gateway.Evaluate(ctx, in, opts...)
139154
}
140155

156+
func (g *gatewayClientForBuild) GetRemote(ctx context.Context, in *gatewayapi.GetRemoteRequest, opts ...grpc.CallOption) (*gatewayapi.GetRemoteResponse, error) {
157+
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
158+
return g.gateway.GetRemote(ctx, in, opts...)
159+
}
160+
141161
func (g *gatewayClientForBuild) Ping(ctx context.Context, in *gatewayapi.PingRequest, opts ...grpc.CallOption) (*gatewayapi.PongResponse, error) {
142162
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
143163
return g.gateway.Ping(ctx, in, opts...)
@@ -148,6 +168,11 @@ func (g *gatewayClientForBuild) Return(ctx context.Context, in *gatewayapi.Retur
148168
return g.gateway.Return(ctx, in, opts...)
149169
}
150170

171+
func (g *gatewayClientForBuild) GetReturn(ctx context.Context, in *gatewayapi.GetReturnRequest, opts ...grpc.CallOption) (*gatewayapi.GetReturnResponse, error) {
172+
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
173+
return g.gateway.GetReturn(ctx, in, opts...)
174+
}
175+
151176
func (g *gatewayClientForBuild) Inputs(ctx context.Context, in *gatewayapi.InputsRequest, opts ...grpc.CallOption) (*gatewayapi.InputsResponse, error) {
152177
if g.caps != nil {
153178
if err := g.caps.Supports(gatewayapi.CapFrontendInputs); err != nil {

client/exporters.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package client
22

33
const (
4-
ExporterImage = "image"
5-
ExporterLocal = "local"
6-
ExporterTar = "tar"
7-
ExporterOCI = "oci"
8-
ExporterDocker = "docker"
4+
ExporterImage = "image"
5+
ExporterLocal = "local"
6+
ExporterTar = "tar"
7+
ExporterOCI = "oci"
8+
ExporterDocker = "docker"
9+
ExporterGateway = "gateway"
910
)

client/solve.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,9 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
168168
if supportFile && supportStore {
169169
return nil, errors.Errorf("both file and store output is not supported by %s exporter", ex.Type)
170170
}
171+
case ExporterGateway:
172+
supportFile = ex.Output != nil
173+
supportDir = ex.OutputDir != ""
171174
}
172175
if !supportFile && ex.Output != nil {
173176
return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
@@ -274,9 +277,24 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
274277
exportDeprecated = exp.Type
275278
exportAttrDeprecated = exp.Attrs
276279
}
280+
target := controlapi.ExporterTarget_NONE
281+
switch {
282+
case exp.Output != nil:
283+
target = controlapi.ExporterTarget_FILE
284+
case exp.OutputDir != "":
285+
switch exp.Type {
286+
case ExporterOCI, ExporterDocker:
287+
target = controlapi.ExporterTarget_STORE
288+
default:
289+
target = controlapi.ExporterTarget_DIRECTORY
290+
}
291+
case exp.OutputStore != nil:
292+
target = controlapi.ExporterTarget_STORE
293+
}
277294
exports = append(exports, &controlapi.Exporter{
278-
Type: exp.Type,
279-
Attrs: exp.Attrs,
295+
Type: exp.Type,
296+
Attrs: exp.Attrs,
297+
Target: target,
280298
})
281299
}
282300

cmd/buildctl/build/output.go

Lines changed: 62 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,10 @@ func parseOutputCSV(s string) (client.ExportEntry, error) {
4242
if v, ok := ex.Attrs["output"]; ok {
4343
return ex, errors.Errorf("output=%s not supported for --output, you meant dest=%s?", v, v)
4444
}
45-
ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs["dest"], ex.Attrs)
45+
ex.Output, ex.OutputDir, err = resolveExporterDest(ex.Type, ex.Attrs)
4646
if err != nil {
4747
return ex, errors.Wrap(err, "invalid output option: output")
4848
}
49-
if ex.Output != nil || ex.OutputDir != "" {
50-
delete(ex.Attrs, "dest")
51-
}
5249
return ex, nil
5350
}
5451

@@ -66,55 +63,83 @@ func ParseOutput(exports []string) ([]client.ExportEntry, error) {
6663
}
6764

6865
// resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path).
69-
func resolveExporterDest(exporter, dest string, attrs map[string]string) (filesync.FileOutputFunc, string, error) {
70-
wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
71-
return func(m map[string]string) (io.WriteCloser, error) {
72-
return wc, nil
73-
}
74-
}
75-
76-
var supportFile bool
77-
var supportDir bool
66+
func resolveExporterDest(exporter string, attrs map[string]string) (destFile filesync.FileOutputFunc, destDir string, _ error) {
67+
var destFilePath string
7868
switch exporter {
7969
case client.ExporterLocal:
80-
supportDir = true
70+
var ok bool
71+
destDir, ok = attrs["dest"]
72+
if !ok {
73+
return nil, "", errors.Errorf("%s exporter requires dest=<path>", client.ExporterLocal)
74+
}
75+
delete(attrs, "dest")
8176
case client.ExporterTar:
82-
supportFile = true
77+
var ok bool
78+
destFilePath, ok = attrs["dest"]
79+
if !ok {
80+
return nil, "", errors.Errorf("%s exporter requires dest=<file>", client.ExporterTar)
81+
}
82+
delete(attrs, "dest")
8383
case client.ExporterOCI, client.ExporterDocker:
8484
tar, err := strconv.ParseBool(attrs["tar"])
8585
if err != nil {
8686
tar = true
8787
}
88-
supportFile = tar
89-
supportDir = !tar
88+
var ok bool
89+
if tar {
90+
destFilePath, ok = attrs["dest"]
91+
} else {
92+
destDir, ok = attrs["dest"]
93+
}
94+
if !ok {
95+
return nil, "", errors.Errorf("%s exporter requires dest=<file|path>", exporter)
96+
}
97+
delete(attrs, "dest")
98+
case client.ExporterGateway:
99+
destFilePath = attrs["file"]
100+
destDir = attrs["dir"]
101+
if destFilePath == "" && destDir == "" {
102+
return nil, "", errors.Errorf("%s exporter requires file=<file> or dir=<path>", exporter)
103+
}
104+
if destFilePath != "" && destDir != "" {
105+
return nil, "", errors.Errorf("%s exporter requires only one of file=<file> or dir=<path>", exporter)
106+
}
107+
delete(attrs, "file")
108+
delete(attrs, "dir")
109+
default:
110+
dest, ok := attrs["dest"]
111+
if ok {
112+
return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter)
113+
}
90114
}
91115

92-
if supportDir {
93-
if dest == "" {
94-
return nil, "", errors.Errorf("output directory is required for %s exporter", exporter)
116+
if destFilePath != "" {
117+
wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
118+
return func(m map[string]string) (io.WriteCloser, error) {
119+
return wc, nil
120+
}
95121
}
96-
return nil, dest, nil
97-
} else if supportFile {
98-
if dest != "" && dest != "-" {
99-
fi, err := os.Stat(dest)
122+
if destFilePath != "" && destFilePath != "-" {
123+
fi, err := os.Stat(destFilePath)
100124
if err != nil && !errors.Is(err, os.ErrNotExist) {
101-
return nil, "", errors.Wrapf(err, "invalid destination file: %s", dest)
125+
return nil, "", errors.Wrapf(err, "invalid destination file: %s", destFilePath)
102126
}
103127
if err == nil && fi.IsDir() {
104128
return nil, "", errors.Errorf("destination file is a directory")
105129
}
106-
w, err := os.Create(dest)
107-
return wrapWriter(w), "", err
108-
}
109-
// if no output file is specified, use stdout
110-
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
111-
return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter)
130+
w, err := os.Create(destFilePath)
131+
if err != nil {
132+
return nil, "", err
133+
}
134+
destFile = wrapWriter(w)
135+
} else {
136+
// if no output file is specified, use stdout
137+
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
138+
return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter)
139+
}
140+
destFile = wrapWriter(os.Stdout)
112141
}
113-
return wrapWriter(os.Stdout), "", nil
114142
}
115-
// e.g. client.ExporterImage
116-
if dest != "" {
117-
return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter)
118-
}
119-
return nil, "", nil
143+
144+
return destFile, destDir, nil
120145
}

0 commit comments

Comments
 (0)