spool: add common "spool" package

This provides for "spool" files: temporary buffers that are expected to
be too big to fit into memory comfortably.

This is a pattern that crops up relatively often via copy-pasted code.

Signed-off-by: Hank Donnay <hdonnay@redhat.com>

Author: hdonnay

toolkit/go.mod

@@ -2,4 +2,7 @@ module github.com/quay/claircore/toolkit
 
 go 1.24
 
-require github.com/google/go-cmp v0.7.0
+require (
+	github.com/google/go-cmp v0.7.0
+	golang.org/x/sys v0.37.0
+)

toolkit/go.sum

@@ -1,2 +1,4 @@
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=

toolkit/spool/os_linux.go

@@ -0,0 +1,93 @@
+package spool
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"runtime"
+	"sync"
+
+	"golang.org/x/sys/unix"
+)
+
+// Init initializes [root].
+func init() {
+	var p string
+
+	// If the environment was explicitly set, use it.
+	var ok bool
+	if p, ok = os.LookupEnv("TMPDIR"); ok {
+		goto Open
+	}
+
+	// Try to honor file-hierarchy(7).
+	for _, name := range []string{`/var/tmp`, os.TempDir()} {
+		fi, err := os.Stat(name)
+		if err == nil && fi.IsDir() {
+			p = name
+			goto Open
+		}
+	}
+
+Open:
+	var err error
+	root, err = os.OpenRoot(p)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func checkRootTmpFile() bool {
+	f, err := root.OpenFile(".", os.O_WRONLY|unix.O_TMPFILE, 0o600)
+	if err != nil {
+		return false
+	}
+	f.Close()
+	return true
+}
+
+var haveTmpFile = sync.OnceValue(checkRootTmpFile)
+
+func osAdjustName(name string) string {
+	if haveTmpFile() {
+		return "."
+	}
+	return name
+}
+
+func osAdjustFlag(flag int) int {
+	if haveTmpFile() && (flag&os.O_CREATE != 0) {
+		// If we can use tmp, do so.
+		flag &= ^os.O_CREATE
+		flag |= unix.O_TMPFILE
+	}
+	return flag
+}
+
+func osAddCleanup(f *os.File) {
+	// If not opened with O_TMPFILE (or there was an error), arrange for the
+	// file to be removed.
+	flags, err := unix.FcntlInt(f.Fd(), unix.F_GETFL, 0)
+	if err != nil || flags&unix.O_TMPFILE == 0 {
+		runtime.AddCleanup(f, func(name string) { root.Remove(name) }, f.Name())
+	}
+}
+
+// Reopen provides or emulates re-opening a file and obtaining an independent file description.
+//
+// The Linux implementation reopens files via [magic symlinks] in [proc].
+//
+// [magic symlinks]: https://www.man7.org/linux/man-pages/man7/symlink.7.html
+// [proc]: https://man7.org/linux/man-pages/man5/proc.5.html
+func Reopen(f *os.File, flag int) (*os.File, error) {
+	if flag&os.O_CREATE != 0 {
+		return nil, fmt.Errorf("spool: cannot pass O_CREATE to Reopen: %w", fs.ErrInvalid)
+	}
+	fd := int(f.Fd())
+	if fd == -1 {
+		return nil, fs.ErrClosed
+	}
+	p := fmt.Sprintf("/proc/self/fd/%d", fd)
+
+	return os.OpenFile(p, flag, 0)
+}

toolkit/spool/os_unix.go

@@ -0,0 +1,55 @@
+//go:build unix && !linux
+
+package spool
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"runtime"
+)
+
+// Init initializes [root].
+func init() {
+	var p string
+
+	// If the environment was explicitly set, use it.
+	var ok bool
+	if p, ok = os.LookupEnv("TMPDIR"); ok {
+		goto Open
+	}
+
+	// Try to honor file-hierarchy(7).
+	for _, name := range []string{`/var/tmp`, os.TempDir()} {
+		fi, err := os.Stat(name)
+		if err == nil && fi.IsDir() {
+			p = name
+			goto Open
+		}
+	}
+
+Open:
+	var err error
+	root, err = os.OpenRoot(p)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func osAdjustName(name string) string { return name }
+
+func osAdjustFlag(flag int) int {
+	return flag & os.O_CREATE
+}
+
+func osAddCleanup(f *os.File) {
+	runtime.AddCleanup(f, func(name string) { root.Remove(name) }, f.Name())
+}
+
+// Reopen provides or emulates re-opening a file and obtaining an independent file description.
+func Reopen(f *os.File, flag int) (*os.File, error) {
+	if flag&os.O_CREATE != 0 {
+		return nil, fmt.Errorf("spool: cannot pass O_CREATE to Reopen: %w", fs.ErrInvalid)
+	}
+	return root.OpenFile(f.Name(), flag, 0)
+}

toolkit/spool/os_windows.go

@@ -0,0 +1 @@
+package spool

toolkit/spool/spool.go

@@ -0,0 +1,94 @@
+// Package spool provides utilities for managing "spool files".
+//
+// Files returned by this package can be counted on to be removed when all open descriptors are closed.
+package spool
+
+import (
+	"cmp"
+	"io/fs"
+	"math/rand/v2"
+	"os"
+	"runtime"
+	"strconv"
+)
+
+// Root is the location all the spool files go into.
+//
+// This is initialized in os-specific files.
+var root *os.Root
+
+// Mkname generates a unique file name based on the provided prefix.
+func mkname(prefix string) string {
+	return cmp.Or(prefix, "tmp") + "." +
+		strconv.FormatUint(uint64(rand.Uint32()), 10)
+}
+
+// OpenFile opens a temporary file with the provided prefix.
+//
+// If "prefix" is not provided, "tmp" will be used.
+// Returned files cannot be opened by path. Callers should use [Reopen].
+func OpenFile(prefix string, flag int, perm fs.FileMode) (*os.File, error) {
+	name := osAdjustName(mkname(prefix))
+	flag = osAdjustFlag(flag)
+	f, err := root.OpenFile(name, flag, perm)
+	if f != nil {
+		osAddCleanup(f)
+	}
+	return f, err
+}
+
+// Create returns an [*os.File] that cannot be opened by path and will be
+// removed when closed.
+func Create() (*os.File, error) {
+	return OpenFile("", os.O_CREATE|os.O_RDWR, 0o600)
+}
+
+// Mkdir creates a directory with the provided prefix.
+//
+// The directory will have its contents removed when the returned [*os.Root] is
+// garbage collected.
+func Mkdir(prefix string, perm fs.FileMode) (*os.Root, error) {
+	name := mkname(prefix)
+	if err := root.Mkdir(name, perm); err != nil {
+		return nil, err
+	}
+	r, err := root.OpenRoot(name)
+	if err == nil { // NB If successful
+		runtime.AddCleanup(r, func(name string) {
+			root.RemoveAll(name)
+		}, name)
+	}
+	return r, err
+}
+
+/*
+This package needs a few parts implemented in OS-specific ways.
+Below is a quick rundown of them, along with a ready-to-use documentation comment.
+
+# Exported
+
+The documentation for these implementations should add additional paragraphs explaining the OS-specific parts starting "The ${OS} implementation [...]".
+
+See the Linux implementations in os_linux.go for an example.
+
+	// Reopen provides or emulates re-opening a file and obtaining an independent file description.
+	func Reopen(f *os.File, flag int) (*os.File, error)
+
+The [Reopen] API is not possible with dup(2), which returns another file descriptor to the same file description.
+An implementation using dup(2) would not provide independent offsets.
+
+# Unexported
+
+	osAdjustName(string) string
+
+AdjustName should modify the passed file name as needed and return the result.
+
+	osAdjustFlag(int) int
+
+AdjustFlag should modify the passed flags as needed and return the result.
+
+	osAddCleanup(*os.File)
+
+AddCleanup should use [runtime.AddCleanup] to attach any needed cleanup functions to the passed [*os.File].
+An implementation will not be called with a nil pointer.
+*/