mkfs.btrfs: support reproducible image creation

Documentation/mkfs.btrfs.rst

@@ -708,6 +708,48 @@ by large metadata blocks and space reservation strategy that allocates more
 than can fit into the filesystem.
 
 
+ENVIRONMENT
+-----------
+
+:command:`mkfs.btrfs` honors two environment variables that make image
+creation reproducible, so that the same inputs produce a byte-for-byte
+identical image.
+
+SOURCE_DATE_EPOCH
+        A decimal count of seconds since the Unix epoch
+        (`reproducible-builds.org <https://reproducible-builds.org>`_),
+        used in place of the current time for the timestamps written into
+        the image: the root-item and inode times of the created trees.
+        With *--rootdir*, each copied file's *ctime* and *atime* are set to
+        this value. Its *mtime* is taken from the source but clamped to it,
+        so nothing in the image is newer than the source date.
+
+        An empty value is treated as unset. A value that is not a
+        non-negative integer fitting in the platform time type is a hard
+        error.
+
+DETERMINISTIC_SEED
+        When set to *1*, the internal UUIDs that are otherwise random (the
+        chunk-tree UUID, each device UUID and each subvolume UUID) are
+        instead derived deterministically from the filesystem UUID. A fixed
+        filesystem UUID must be supplied with *-U*; without it
+        :command:`mkfs.btrfs` exits with an error rather than emit a random
+        image. Any value other than *1* leaves UUID generation random.
+
+To create a reproducible image, pin the filesystem UUID with *-U*, set
+``SOURCE_DATE_EPOCH`` and ``DETERMINISTIC_SEED``, normalize source ownership,
+and start with a fresh target file. :command:`mkfs.btrfs` copies *uid* and
+*gid* from the source tree and, like other mkfs tools, does not zero space it
+does not write:
+
+.. code-block:: bash
+
+        $ chown -R 0:0 ./rootdir
+        $ truncate -s 1G image.btrfs
+        $ SOURCE_DATE_EPOCH=1700000000 DETERMINISTIC_SEED=1 \
+                mkfs.btrfs -U <uuid> --rootdir ./rootdir image.btrfs
+
+
 AVAILABILITY
 ------------
 

Makefile

@@ -233,6 +233,7 @@ objects = \
 	common/parse-utils.o	\
 	common/path-utils.o	\
 	common/rbtree-utils.o	\
+	common/reproducible.o	\
 	common/send-stream.o	\
 	common/send-utils.o	\
 	common/sort-utils.o	\

common/device-scan.c

@@ -48,6 +48,7 @@
 #include "common/path-utils.h"
 #include "common/device-scan.h"
 #include "common/messages.h"
+#include "common/reproducible.h"
 #include "common/utils.h"
 #include "common/defs.h"
 #include "common/open-utils.h"
@@ -155,7 +156,10 @@ int btrfs_add_to_fsid(struct btrfs_trans_handle *trans,
 	disk_super = (struct btrfs_super_block *)buf;
 	dev_item = &disk_super->dev_item;
 
-	uuid_generate(device->uuid);
+	reproducible_uuid_generate(super->fsid,
+				   REPRODUCIBLE_UUID_ROLE_DEVICE,
+				   btrfs_super_num_devices(super) + 1,
+				   device->uuid);
 	device->fs_info = fs_info;
 	device->devid = 0;
 	device->type = 0;

common/path-utils.c

@@ -20,7 +20,9 @@
 #include <linux/kdev_t.h>
 #include <linux/loop.h>
 #include <linux/limits.h>
+#include <dirent.h>
 #include <fcntl.h>
+#include <ftw.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -508,3 +510,141 @@ int path_readlink(char *dest, const char *src)
 	dest[ret] = 0;
 	return ret;
 }
+
+struct path_walk_entry {
+	char *name;
+};
+
+static int path_walk_entry_cmp(const void *a, const void *b)
+{
+	const struct path_walk_entry *ea = a;
+	const struct path_walk_entry *eb = b;
+
+	return strcmp(ea->name, eb->name);
+}
+
+/*
+ * Emulate nftw()'s depth-first, preorder, FTW_PHYS walk, but visit each
+ * directory's entries in sorted name order. The struct FTW fields are filled
+ * the way nftw() reports them so existing nftw() callbacks work unchanged.
+ */
+static int path_sorted_walk_recursive(char *path, int level, path_walk_cb cb)
+{
+	struct stat st;
+	struct FTW ftwbuf;
+	struct path_walk_entry *entries = NULL;
+	size_t count = 0, cap = 0;
+	DIR *dir = NULL;
+	struct dirent *de;
+	const char *slash;
+	int ret;
+
+	slash = strrchr(path, '/');
+	ftwbuf.base = slash ? (int)(slash - path + 1) : 0;
+	ftwbuf.level = level;
+
+	if (lstat(path, &st) != 0)
+		return cb(path, NULL, FTW_NS, &ftwbuf);
+
+	if (!S_ISDIR(st.st_mode)) {
+		if (S_ISLNK(st.st_mode))
+			return cb(path, &st, FTW_SL, &ftwbuf);
+		return cb(path, &st, FTW_F, &ftwbuf);
+	}
+
+	/*
+	 * Open the directory before reporting it: one we can lstat() but not
+	 * open is reported once as FTW_DNR, not FTW_D then FTW_DNR.
+	 */
+	dir = opendir(path);
+	if (!dir)
+		return cb(path, &st, FTW_DNR, &ftwbuf);
+
+	ret = cb(path, &st, FTW_D, &ftwbuf);
+	if (ret)
+		goto cleanup;
+
+	/*
+	 * readdir() returns NULL at both end-of-stream and error; clear
+	 * errno before each call to tell them apart after the loop.
+	 */
+	while (1) {
+		errno = 0;
+		de = readdir(dir);
+		if (!de)
+			break;
+		if (strcmp(de->d_name, ".") == 0 ||
+		    strcmp(de->d_name, "..") == 0)
+			continue;
+		if (count == cap) {
+			struct path_walk_entry *new_entries;
+
+			cap = cap ? cap * 2 : 16;
+			new_entries = reallocarray(entries, cap,
+						   sizeof(*entries));
+			if (!new_entries) {
+				ret = -ENOMEM;
+				goto cleanup;
+			}
+			entries = new_entries;
+		}
+		entries[count].name = strndup(de->d_name, NAME_MAX);
+		if (!entries[count].name) {
+			ret = -ENOMEM;
+			goto cleanup;
+		}
+		count++;
+	}
+	if (errno != 0) {
+		ret = -errno;
+		goto cleanup;
+	}
+	closedir(dir);
+	dir = NULL;
+
+	if (count)
+		qsort(entries, count, sizeof(*entries), path_walk_entry_cmp);
+
+	for (size_t i = 0; i < count; i++) {
+		char *child = malloc(PATH_MAX);
+
+		if (!child) {
+			ret = -ENOMEM;
+			goto cleanup;
+		}
+		ret = path_cat_out(child, path, entries[i].name);
+		if (!ret)
+			ret = path_sorted_walk_recursive(child, level + 1, cb);
+		free(child);
+		if (ret)
+			goto cleanup;
+	}
+	ret = 0;
+
+cleanup:
+	if (dir)
+		closedir(dir);
+	for (size_t i = 0; i < count; i++)
+		free(entries[i].name);
+	free(entries);
+	return ret;
+}
+
+int path_sorted_walk(const char *root, path_walk_cb cb)
+{
+	char path[PATH_MAX];
+	size_t root_len = strlen(root);
+
+	/* glibc nftw() rejects an empty path with ENOENT and no callback. */
+	if (root_len == 0)
+		return -ENOENT;
+	if (root_len >= sizeof(path))
+		return -ENAMETOOLONG;
+	memcpy(path, root, root_len + 1);
+
+	/* Strip trailing slashes (but keep a lone "/"), matching nftw. */
+	while (root_len > 1 && path[root_len - 1] == '/')
+		path[--root_len] = '\0';
+
+	return path_sorted_walk_recursive(path, 0, cb);
+}

common/path-utils.h

@@ -18,7 +18,9 @@
 #define __BTRFS_PATH_UTILS_H__
 
 #include <sys/types.h>
+#include <sys/stat.h>
 #include <linux/limits.h>
+#include <ftw.h>
 
 char *path_canonicalize_dm_name(const char *ptname);
 char *path_canonicalize(const char *path);
@@ -38,4 +40,30 @@ char *path_basename(char *path);
 char *path_dirname(char *path);
 int path_readlink(char *dest, const char *src);
 
+/*
+ * Callback for path_sorted_walk(), matching nftw()'s callback signature
+ * so existing nftw() callbacks can be reused unchanged.
+ *
+ *   st     : lstat() result, or NULL when type == FTW_NS
+ *   type   : FTW_F, FTW_D (pre-order), FTW_SL (never followed),
+ *            FTW_DNR (unreadable directory) or FTW_NS (lstat failed)
+ *   ftwbuf : level (depth, 0 at the root) and base (basename offset)
+ *
+ * Return 0 to continue the walk, non-zero to abort it (that value is
+ * then returned by path_sorted_walk()).
+ */
+typedef int (*path_walk_cb)(const char *path, const struct stat *st,
+			    int type, struct FTW *ftwbuf);
+
+/*
+ * Walk a directory tree depth-first, pre-order, visiting the entries of
+ * each directory in byte-wise (strcmp) name order. Symlinks are never
+ * followed (FTW_PHYS). Sorting makes the walk order independent of the
+ * filesystem's readdir() order.
+ *
+ * Returns 0 on success, the callback's non-zero return value if it
+ * aborted, or -errno on I/O failure.
+ */
+int path_sorted_walk(const char *root, path_walk_cb cb);
+
 #endif

common/reproducible.c

@@ -0,0 +1,76 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#include "kerncompat.h"
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <uuid/uuid.h>
+#include "common/messages.h"
+#include "common/parse-utils.h"
+#include "common/reproducible.h"
+
+time_t reproducible_now(void)
+{
+	const char *sde = getenv("SOURCE_DATE_EPOCH");
+	u64 val;
+
+	if (!sde || !*sde)
+		return time(NULL);
+
+	if (parse_u64(sde, &val) < 0 || val > (u64)LONG_MAX) {
+		error("invalid SOURCE_DATE_EPOCH: '%s' is not a non-negative integer fitting in time_t",
+		      sde);
+		exit(1);
+	}
+	return (time_t)val;
+}
+
+bool reproducible_has_source_date(void)
+{
+	const char *sde = getenv("SOURCE_DATE_EPOCH");
+
+	return sde && *sde;
+}
+
+bool reproducible_is_deterministic(void)
+{
+	const char *v = getenv("DETERMINISTIC_SEED");
+
+	return v && strcmp(v, "1") == 0;
+}
+
+bool reproducible_is_enabled(void)
+{
+	return reproducible_has_source_date() && reproducible_is_deterministic();
+}
+
+void reproducible_uuid_generate(const u8 fs_uuid[BTRFS_UUID_SIZE],
+				enum reproducible_uuid_role role, u64 key,
+				u8 out[BTRFS_UUID_SIZE])
+{
+	u8 name[sizeof(u32) + sizeof(u64)];
+
+	if (!reproducible_is_deterministic()) {
+		uuid_generate(out);
+		return;
+	}
+
+	put_unaligned_le32((u32)role, name);
+	put_unaligned_le64(key, name + sizeof(u32));
+	uuid_generate_sha1(out, fs_uuid, (const char *)name, sizeof(name));
+}