Skip to content

Commit 334f71e

Browse files
committed
feat: add igzip/pigz compression
1 parent e799563 commit 334f71e

File tree

3 files changed

+120
-3
lines changed

3 files changed

+120
-3
lines changed

README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ Keys supported by image output:
288288
* `unpack=true`: unpack image after creation (for use with containerd)
289289
* `dangling-name-prefix=<value>`: name image with `prefix@<digest>`, used for anonymous images
290290
* `name-canonical=true`: add additional canonical name `name@<digest>`
291-
* `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz should be used with `oci-mediatypes=true`.
291+
* `compression=<uncompressed|gzip|igzip|pigz|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz should be used with `oci-mediatypes=true`.
292292
* `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)
293293
* `rewrite-timestamp=true`: rewrite the file timestamps to the `SOURCE_DATE_EPOCH` value.
294294
See [`docs/build-repro.md`](docs/build-repro.md) for how to specify the `SOURCE_DATE_EPOCH` value.
@@ -466,7 +466,7 @@ buildctl build ... \
466466
* `ref=<ref>`: specify repository reference to store cache, e.g. `docker.io/user/image:tag`
467467
* `image-manifest=<true|false>`: whether to export cache manifest as an OCI-compatible image manifest rather than a manifest list/index (default: `true` since BuildKit `v0.21`, must be used with `oci-mediatypes=true`)
468468
* `oci-mediatypes=<true|false>`: whether to use OCI mediatypes in exported manifests (default: `true`, since BuildKit `v0.8`)
469-
* `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`
469+
* `compression=<uncompressed|gzip|igzip|pigz|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`
470470
* `compression-level=<value>`: choose compression level for gzip, estargz (0-9) and zstd (0-22)
471471
* `force-compression=true`: forcibly apply `compression` option to all layers
472472
* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
@@ -493,7 +493,7 @@ The directory layout conforms to OCI Image Spec v1.0.
493493
* `tag=<tag>`: specify custom tag of image to write to local index (default: `latest`)
494494
* `image-manifest=<true|false>`: whether to export cache manifest as an OCI-compatible image manifest rather than a manifest list/index (default: `true` since BuildKit `v0.21`, must be used with `oci-mediatypes=true`)
495495
* `oci-mediatypes=<true|false>`: whether to use OCI mediatypes in exported manifests (default `true`, since BuildKit `v0.8`)
496-
* `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`.
496+
* `compression=<uncompressed|gzip|igzip|pigz|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`.
497497
* `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)
498498
* `force-compression=true`: forcibly apply `compression` option to all layers
499499
* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)

util/compression/compression.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ type Type interface {
3535
type (
3636
uncompressedType struct{}
3737
gzipType struct{}
38+
igzipType struct{}
39+
pigzType struct{}
3840
estargzType struct{}
3941
zstdType struct{}
4042
)
@@ -46,6 +48,12 @@ var (
4648
// Gzip is used for blob data.
4749
Gzip = gzipType{}
4850

51+
// Igzip is used for blob data.
52+
Igzip = igzipType{}
53+
54+
// Pigz is used for blob data.
55+
Pigz = pigzType{}
56+
4957
// EStargz is used for estargz data.
5058
EStargz = estargzType{}
5159

@@ -87,6 +95,10 @@ func parse(t string) (Type, error) {
8795
return Uncompressed, nil
8896
case Gzip.String():
8997
return Gzip, nil
98+
case Pigz.String():
99+
return Pigz, nil
100+
case Igzip.String():
101+
return Igzip, nil
90102
case EStargz.String():
91103
return EStargz, nil
92104
case Zstd.String():

util/compression/gzip.go

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
package compression
22

33
import (
4+
"bytes"
45
"compress/gzip"
56
"context"
7+
"errors"
8+
"fmt"
69
"io"
10+
"os/exec"
711

812
"github.com/containerd/containerd/v2/core/content"
913
"github.com/containerd/containerd/v2/core/images"
@@ -64,3 +68,104 @@ func gzipWriter(comp Config) func(io.Writer) (io.WriteCloser, error) {
6468
return gzip.NewWriterLevel(dest, level)
6569
}
6670
}
71+
72+
func (c igzipType) Compress(ctx context.Context, comp Config) (compressorFunc Compressor, finalize Finalizer) {
73+
return func(dest io.Writer, _ string) (io.WriteCloser, error) {
74+
return gzipCmdWriter(ctx, "igzip", comp)(dest)
75+
}, nil
76+
}
77+
78+
func (c igzipType) Decompress(ctx context.Context, cs content.Store, desc ocispecs.Descriptor) (io.ReadCloser, error) {
79+
return decompress(ctx, cs, desc)
80+
}
81+
82+
func (c igzipType) NeedsConversion(ctx context.Context, cs content.Store, desc ocispecs.Descriptor) (bool, error) {
83+
return gzipType{}.NeedsConversion(ctx, cs, desc)
84+
}
85+
86+
func (c igzipType) NeedsComputeDiffBySelf(comp Config) bool {
87+
return gzipType{}.NeedsComputeDiffBySelf(comp)
88+
}
89+
90+
func (c igzipType) OnlySupportOCITypes() bool {
91+
return gzipType{}.OnlySupportOCITypes()
92+
}
93+
94+
func (c igzipType) MediaType() string {
95+
return gzipType{}.MediaType()
96+
}
97+
98+
func (c igzipType) String() string {
99+
return "igzip"
100+
}
101+
102+
func (c pigzType) Compress(ctx context.Context, comp Config) (compressorFunc Compressor, finalize Finalizer) {
103+
return func(dest io.Writer, _ string) (io.WriteCloser, error) {
104+
return gzipCmdWriter(ctx, "pigz", comp)(dest)
105+
}, nil
106+
}
107+
108+
func (c pigzType) Decompress(ctx context.Context, cs content.Store, desc ocispecs.Descriptor) (io.ReadCloser, error) {
109+
return decompress(ctx, cs, desc)
110+
}
111+
112+
func (c pigzType) NeedsConversion(ctx context.Context, cs content.Store, desc ocispecs.Descriptor) (bool, error) {
113+
return gzipType{}.NeedsConversion(ctx, cs, desc)
114+
}
115+
116+
func (c pigzType) NeedsComputeDiffBySelf(comp Config) bool {
117+
return gzipType{}.NeedsComputeDiffBySelf(comp)
118+
}
119+
120+
func (c pigzType) OnlySupportOCITypes() bool {
121+
return gzipType{}.OnlySupportOCITypes()
122+
}
123+
124+
func (c pigzType) MediaType() string {
125+
return gzipType{}.MediaType()
126+
}
127+
128+
func (c pigzType) String() string {
129+
return "pigz"
130+
}
131+
132+
type writeCloserWrapper struct {
133+
io.Writer
134+
closer func() error
135+
}
136+
137+
func (w *writeCloserWrapper) Close() error {
138+
return w.closer()
139+
}
140+
141+
func gzipCmdWriter(ctx context.Context, cmd string, comp Config) func(io.Writer) (io.WriteCloser, error) {
142+
return func(dest io.Writer) (io.WriteCloser, error) {
143+
reader, writer := io.Pipe()
144+
args := []string{"-c"}
145+
if comp.Level != nil {
146+
args = append(args, fmt.Sprintf("-%d", *comp.Level))
147+
}
148+
command := exec.CommandContext(ctx, cmd, args...)
149+
command.Stdin = reader
150+
command.Stdout = dest
151+
152+
var errBuf bytes.Buffer
153+
command.Stderr = &errBuf
154+
155+
if err := command.Start(); err != nil {
156+
return nil, err
157+
}
158+
159+
return &writeCloserWrapper{
160+
Writer: writer,
161+
closer: func() error {
162+
closeErr := writer.Close()
163+
waitErr := command.Wait()
164+
if waitErr != nil {
165+
return fmt.Errorf("%s: %s", waitErr, errBuf.String())
166+
}
167+
return errors.Join(closeErr, waitErr)
168+
},
169+
}, nil
170+
}
171+
}

0 commit comments

Comments
 (0)