replay.cpp

/*                      R E P L A Y . C P P
 * BRL-CAD
 *
 * Published in 2020 by the United States Government.
 * This work is in the public domain.
 *
 */
/** @file replay.cpp
 *
 * Topological "replay" of a parsed fast-import history into a real Git
 * repository using libgit2.  Unlike the fast-import writing path (which
 * relies on git computing object ids at import time and therefore never
 * learns the new SHA1s), the replay engine builds every object itself and
 * therefore knows, for every original object, the id it maps to in the
 * rebuilt history.
 *
 * The stream order that repowork parses is already topological (fast-import
 * requires marks to be defined before they are used), so by the time we
 * build a commit every object it depends on - its blobs and its parent
 * commits - has already been rebuilt and recorded.  This is the property
 * that makes SHA1-reference rewriting possible: any id referenced by a
 * commit message or text blob necessarily belongs to an ancestor, whose new
 * id is already known.
 *
 * With no rewriting requested, replay reproduces the input history
 * bit-for-bit (new id == old id for every object); that invariant is what
 * the replay round-trip test checks.
 */

#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <set>
#include <unordered_map>
#include <vector>

#include <git2.h>
#include <git2/sys/mempack.h>

#include "repowork.h"

#if defined(_WIN32)
#  define WIN32_LEAN_AND_MEAN
#  define NOMINMAX
#  include <windows.h>
#elif defined(__linux__)
#  include <unistd.h>
#endif

namespace {

// Abort with a readable message if a libgit2 call failed.  (repowork uses
// exit() for fatal conditions throughout; matching that here keeps the error
// behavior consistent.  A library-friendly error strategy is a Phase 1 item.)
void
lg2_fatal(int err, const char *op)
{
    if (err < 0) {
	const git_error *e = git_error_last();
	std::cerr << "libgit2 error (" << op << "): "
		  << ((e && e->message) ? e->message : "unknown") << "\n";
	exit(1);
    }
}

// Best-effort available physical memory in bytes (0 = unknown).  Used to
// decide whether the in-memory pack backend is safe or we should fall back to
// writing loose objects.
uint64_t
available_memory_bytes()
{
#if defined(_WIN32)
    MEMORYSTATUSEX st;
    st.dwLength = sizeof(st);
    if (GlobalMemoryStatusEx(&st))
	return (uint64_t)st.ullAvailPhys;
    return 0;
#elif defined(__linux__)
    long pages = sysconf(_SC_AVPHYS_PAGES), ps = sysconf(_SC_PAGE_SIZE);
    if (pages > 0 && ps > 0)
	return (uint64_t)pages * (uint64_t)ps;
    return 0;
#else
    return 0;
#endif
}

std::string
oid_hex(const git_oid *o)
{
    char buf[GIT_OID_MAX_HEXSIZE + 1];
    git_oid_tostr(buf, sizeof(buf), o);
    return std::string(buf);
}

// Produce the "<name> <email> <timestamp> <tz>" portion of an author/
// committer/tagger line.  By default the original bytes are preserved exactly
// (so the object id is reproduced).  When normalize is set, the identity is
// canonicalized the way git itself would (trimming stray whitespace such as
// "Name  <email>" -> "Name <email>") - an optional cleanup for callers who are
// reworking the history anyway.  The timestamp is left untouched.
std::string
format_ident(const std::string &ident, const std::string &tstamp, bool normalize)
{
    std::string verbatim = ident + " " + tstamp;
    if (!normalize)
	return verbatim;
    git_signature *sig = NULL;
    if (git_signature_from_buffer(&sig, verbatim.c_str()) != 0 || !sig)
	return verbatim;   // unparseable - leave it alone rather than mangle it
    std::string out = std::string(sig->name) + " <" + std::string(sig->email) + "> " + tstamp;
    git_signature_free(sig);
    return out;
}

// Remove the gpgsig header (and its continuation lines) from a verbatim block
// of trailing commit headers, leaving any other headers (e.g. encoding)
// intact.  Used when a commit changes and its signature would be invalid.
std::string
strip_gpgsig(const std::string &headers)
{
    std::string out;
    size_t pos = 0;
    bool skipping = false;
    while (pos < headers.size()) {
	size_t eol = headers.find('\n', pos);
	if (eol == std::string::npos) eol = headers.size();
	std::string line = headers.substr(pos, eol - pos);
	bool cont = (!line.empty() && line[0] == ' ');
	if (!cont)
	    skipping = (line.compare(0, 7, "gpgsig ") == 0);
	if (!skipping) {
	    out += line;
	    out += "\n";
	}
	pos = eol + 1;
    }
    return out;
}

// Format an armored signature as a "gpgsig" commit header: the first line gets
// the "gpgsig " prefix, every continuation line a single leading space (git's
// multi-line header convention).
std::string
make_gpgsig_header(const std::string &sig)
{
    std::string h;
    size_t pos = 0;
    bool first = true;
    while (pos < sig.size()) {
	size_t eol = sig.find('\n', pos);
	std::string line = (eol == std::string::npos) ? sig.substr(pos)
						      : sig.substr(pos, eol - pos);
	h += first ? "gpgsig " : " ";
	h += line;
	h += "\n";
	first = false;
	if (eol == std::string::npos)
	    break;
	pos = eol + 1;
    }
    return h;
}

// Produce an armored detached signature over a commit payload by invoking an
// external GPG program.  Temp files (payload in / signature out) keep this
// portable via the shell; the signing key and agent/passphrase config are the
// user's responsibility.
std::string
sign_payload(const std::string &payload, const std::string &gpg_program,
	     const std::string &keyid, const std::string &in_path,
	     const std::string &out_path)
{
    {
	std::ofstream o(in_path, std::ios::out | std::ios::binary | std::ios::trunc);
	o.write(payload.data(), (std::streamsize)payload.size());
    }
    std::remove(out_path.c_str());
    std::string cmd = "\"" + gpg_program + "\"" +
	" --batch --yes --armor --detach-sign --local-user \"" + keyid +
	"\" --output \"" + out_path + "\" \"" + in_path + "\"";
#if defined(_WIN32)
    // cmd.exe strips one outer quote pair; wrap the whole command so a program
    // path containing spaces (and the quoted args) parse correctly.
    cmd = "\"" + cmd + "\"";
#endif
    int rc = std::system(cmd.c_str());
    if (rc != 0) {
	std::cerr << "replay: signing failed (" << gpg_program << " rc=" << rc << ")\n";
	exit(1);
    }
    std::ifstream in(out_path, std::ios::in | std::ios::binary);
    std::string sig((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
    if (sig.empty()) {
	std::cerr << "replay: signer produced an empty signature\n";
	exit(1);
    }
    return sig;
}

bool is_hex(char c)   { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); }
bool is_alnum(char c) { return is_hex(c) || (c >= 'g' && c <= 'z') || (c >= 'A' && c <= 'Z'); }

// Resolve a (possibly abbreviated) hex token to the full original id it names,
// applying git's own reference criteria: the token must uniquely prefix exactly
// one id in the recognition set (the complete set of original object ids).
// Returns "" if it matches no id or is an ambiguous prefix of more than one -
// exactly the cases where git itself would refuse to resolve the abbreviation.
std::string
resolve_prefix(const std::string &tok, const std::set<std::string> &ids)
{
    if (tok.size() >= 40)
	return ids.count(tok) ? tok : std::string();

    auto it = ids.lower_bound(tok);
    if (it == ids.end() || it->compare(0, tok.size(), tok) != 0)
	return std::string();
    auto nx = std::next(it);
    if (nx != ids.end() && nx->compare(0, tok.size(), tok) == 0)
	return std::string();   // ambiguous prefix - leave it alone
    return *it;
}

// The shortest abbreviation of new_hex that is unambiguous in the repository
// as it currently stands, but never shorter than the reference's original
// length (so we preserve the author's style and only ever lengthen).
//
// NOTE: uniqueness is evaluated against the objects that exist at the moment
// of rewriting (the referenced object plus its ancestors).  Objects created
// later could in principle collide with a chosen prefix; a final verification
// pass reports any residual ambiguities.  A hard guarantee would require a
// fixpoint, since the abbreviation text is itself hashed into the containing
// object - that is deferred.
size_t
unique_abbrev_len(git_repository *repo, const std::string &new_hex, size_t desired)
{
    git_oid oid;
    if (git_oid_fromstr(&oid, new_hex.c_str()) != 0)
	return desired;
    git_object *obj = NULL;
    if (git_object_lookup(&obj, repo, &oid, GIT_OBJECT_ANY) != 0)
	return desired;
    git_buf sb = GIT_BUF_INIT;
    size_t minlen = desired;
    if (git_object_short_id(&sb, obj) == 0 && sb.size > minlen)
	minlen = sb.size;
    git_buf_dispose(&sb);
    git_object_free(obj);
    return (minlen > 40) ? 40 : minlen;
}

// Rewrite SHA1 references (full or abbreviated) found in a text buffer, mapping
// each one from its original id to the rebuilt id.  Only tokens that uniquely
// resolve to an already-rebuilt object whose id actually changed are touched,
// which keeps false positives (random hex, unrelated ids) from being altered.
// An abbreviated reference is replaced with an abbreviation of the new id that
// is at least as long as the original and re-expanded as needed to stay
// unambiguous in the rebuilt repo (see unique_abbrev_len).
std::string
rewrite_shas(git_repository *repo, const std::string &msg,
	     const std::map<std::string, std::string> &old_to_new,
	     const std::set<std::string> &recognition_ids,
	     size_t minlen, int &nrepl, std::set<std::string> &emitted)
{
    std::string out;
    out.reserve(msg.size());
    size_t i = 0, n = msg.size();
    while (i < n) {
	if (!is_hex(msg[i])) {
	    out += msg[i++];
	    continue;
	}
	size_t j = i;
	while (j < n && is_hex(msg[j]))
	    j++;
	size_t runlen = j - i;
	std::string token = msg.substr(i, runlen);
	bool left_ok  = (i == 0) || !is_alnum(msg[i - 1]);
	bool right_ok = (j == n) || !is_alnum(msg[j]);
	std::string rep;
	if (left_ok && right_ok && runlen >= minlen && runlen <= 40) {
	    // Recognize against the objects seen so far (temporal awareness): a
	    // reference resolves to the object it named when it was written, even
	    // if a later commit would make the abbreviation ambiguous in the
	    // finished repo.
	    std::string full = resolve_prefix(token, recognition_ids);
	    auto mit = full.size() ? old_to_new.find(full) : old_to_new.end();
	    if (mit != old_to_new.end() && mit->second != full) {
		const std::string &nf = mit->second;   // only rewrite ids that moved
		size_t alen = (runlen >= 40) ? 40 : unique_abbrev_len(repo, nf, runlen);
		rep = nf.substr(0, alen);
		if (alen < 40)
		    emitted.insert(rep);   // for final uniqueness verification
		nrepl++;
	    }
	}
	out += rep.size() ? rep : token;
	i = j;
    }
    return out;
}

// Resolve the blob id referenced by a filemodify op, preferring the mark
// (which is what fast-export emits) and falling back to a recorded SHA1.
bool
resolve_dataref(const git_op &o,
		const std::unordered_map<long, git_oid> &mark_to_oid,
		const std::map<std::string, std::string> &old_to_new,
		git_oid *out)
{
    if (o.dataref.mark != -1) {
	auto it = mark_to_oid.find(o.dataref.mark);
	if (it != mark_to_oid.end()) {
	    git_oid_cpy(out, &it->second);
	    return true;
	}
    }
    if (o.dataref.sha1.length()) {
	auto it = old_to_new.find(o.dataref.sha1);
	if (it != old_to_new.end())
	    return git_oid_fromstr(out, it->second.c_str()) == 0;
	// Not one of our objects - assume the SHA1 already names an object in
	// the target repo (e.g. a blob imported separately via --blob-map).
	return git_oid_fromstr(out, o.dataref.sha1.c_str()) == 0;
    }
    return false;
}

// Apply one fast-import file operation to the in-memory index that is
// accumulating a commit's tree.
void
apply_fileop(git_index *idx, const git_op &o,
	     const std::unordered_map<long, git_oid> &mark_to_oid,
	     const std::map<std::string, std::string> &old_to_new)
{
    switch (o.type) {
	case filemodify: {
	    git_oid boid;
	    if (!resolve_dataref(o, mark_to_oid, old_to_new, &boid)) {
		std::cerr << "replay: could not resolve blob for " << o.path << "\n";
		exit(1);
	    }
	    git_index_entry e;
	    memset(&e, 0, sizeof(e));
	    e.mode = (uint32_t)strtol(o.mode.c_str(), NULL, 8);
	    e.path = o.path.c_str();
	    git_oid_cpy(&e.id, &boid);
	    lg2_fatal(git_index_add(idx, &e), "index_add");
	    break;
	}
	case filedelete:
	    // The path may be a single file or a whole subtree; try the file
	    // form first, then fall back to removing a directory.  Both may
	    // legitimately "fail" (path already absent), so return codes are
	    // intentionally not fatal here.
	    if (git_index_remove(idx, o.path.c_str(), 0) < 0)
		git_index_remove_directory(idx, o.path.c_str(), 0);
	    break;
	case filedeleteall:
	    lg2_fatal(git_index_clear(idx), "index_clear");
	    break;
	case filerename:
	case filecopy: {
	    const git_index_entry *se = git_index_get_bypath(idx, o.path.c_str(), 0);
	    if (!se) {
		std::cerr << "replay: rename/copy of directory '" << o.path
			  << "' not yet supported\n";
		exit(1);
	    }
	    git_index_entry ne = *se;      // copy value before the index mutates
	    ne.path = o.dest_path.c_str();
	    lg2_fatal(git_index_add(idx, &ne), "index_add(copy)");
	    if (o.type == filerename)
		git_index_remove(idx, o.path.c_str(), 0);
	    break;
	}
	case notemodify:
	    std::cerr << "replay: notemodify is not supported\n";
	    exit(1);
    }
}

} // anonymous namespace

int
git_replay(git_fi_data *s, const std::string &infile_path,
	   const std::string &out_repo, const std::string &map_file,
	   const git_replay_opts &opts)
{
    // Unpack options into locals (keeps the body below unchanged).
    bool rewrite_refs = opts.rewrite_refs;
    bool rewrite_blob_refs = opts.rewrite_blob_refs;
    size_t ref_min_len = opts.ref_min_len;
    bool force_loose = opts.force_loose;
    bool normalize_identity = opts.normalize_identity;
    size_t pack_memory_mb = opts.pack_memory_mb;
    const std::string &sign_key = opts.sign_key;
    const std::string &gpg_program = opts.gpg_program;

    git_libgit2_init();

    git_repository *repo = NULL;
    git_repository_init_options iopts = GIT_REPOSITORY_INIT_OPTIONS_INIT;
    iopts.flags = GIT_REPOSITORY_INIT_MKPATH | GIT_REPOSITORY_INIT_BARE;
    lg2_fatal(git_repository_init_ext(&repo, out_repo.c_str(), &iopts),
	      "repository_init");

    // Route object writes into an in-memory pack backend rather than writing
    // hundreds of thousands of individual loose object files.  Loose writes are
    // pathologically slow at scale on Windows (per-file fsync + antivirus scan
    // + NTFS fanout); accumulating in memory and writing packfiles is what git
    // fast-import does and is far faster.  Reads during replay are served from
    // the backend too, so read-after-write (parent/tree lookups) still works.
    //
    // To keep peak RAM bounded for arbitrarily large histories, the accumulated
    // objects are flushed to a packfile whenever they exceed a memory budget
    // (staged packing), then the in-memory store is reset and replay resumes.
    // git_mempack_write_thin_pack packs exactly the current in-memory objects
    // (no reachability walk), so already-flushed objects are never duplicated
    // across packs.  --loose-objects forces the old loose path.
    uint64_t est_bytes = 0;
    for (const git_blob_data &b : s->blobs)
	est_bytes += b.length;
    est_bytes += (uint64_t)s->commits.size() * 512;   // rough commit/tree overhead
    uint64_t avail = available_memory_bytes();
    bool use_mempack = !force_loose;
    // Per-flush budget: an explicit --pack-memory value, else half of available
    // RAM, else a 1 GiB default when RAM is unknown.
    uint64_t flush_threshold = pack_memory_mb ? ((uint64_t)pack_memory_mb << 20)
			      : (avail ? avail / 2 : (1ULL << 30));

    std::cerr << "  object store: "
	      << (use_mempack ? "in-memory pack (staged)" : "loose (forced)")
	      << " (est " << (est_bytes >> 20) << " MiB";
    if (avail)
	std::cerr << ", avail " << (avail >> 20) << " MiB";
    if (use_mempack)
	std::cerr << ", flush at " << (flush_threshold >> 20) << " MiB";
    std::cerr << ")\n";

    git_odb *odb = NULL;
    lg2_fatal(git_repository_odb(&odb, repo), "repository_odb");
    git_odb_backend *mempack = NULL;
    if (use_mempack) {
	lg2_fatal(git_mempack_new(&mempack), "mempack_new");
	lg2_fatal(git_odb_add_backend(odb, mempack, 999), "odb_add_backend");
    }

    // Staged-flush bookkeeping.
    uint64_t pending_bytes = 0;
    size_t n_packs = 0;
    auto flush_pack = [&]() {
	size_t cnt = 0;
	git_mempack_object_count(&cnt, mempack);
	if (!cnt)
	    return;
	git_packbuilder *pb = NULL;
	lg2_fatal(git_packbuilder_new(&pb, repo), "packbuilder_new");
	git_packbuilder_set_threads(pb, 0);   // auto
	// Pack exactly the objects currently in memory (blobs, trees, commits,
	// tags) - no tree walk, so nothing already flushed is duplicated.
	lg2_fatal(git_mempack_write_thin_pack(mempack, pb), "mempack_write_thin_pack");
	lg2_fatal(git_packbuilder_write(pb, NULL, 0, NULL, NULL), "packbuilder_write");
	git_packbuilder_free(pb);
	git_mempack_reset(mempack);
	// Make the just-written pack visible so later parent/tree reads resolve.
	lg2_fatal(git_odb_refresh(odb), "odb_refresh");
	pending_bytes = 0;
	n_packs++;
    };

    // Blob content comes from either the source git repository (repo-read
    // mode) or the original fast-import file (stream mode).
    bool from_repo = !s->src_repo_path.empty();
    git_repository *src_repo = NULL;
    git_odb *src_odb = NULL;
    std::ifstream infile;
    if (from_repo) {
	lg2_fatal(git_repository_open(&src_repo, s->src_repo_path.c_str()),
		  "source repository_open");
	lg2_fatal(git_repository_odb(&src_odb, src_repo), "source repository_odb");
    } else {
	infile.open(infile_path, std::ifstream::binary);
	if (!infile.good()) {
	    std::cerr << "replay: could not reopen input file " << infile_path << "\n";
	    return -1;
	}
    }

    // Original object id (hex) -> rebuilt object id (hex).  This is the
    // artifact that makes reference rewriting possible.
    std::map<std::string, std::string> old_to_new;
    // Commit and tag ids rebuilt SO FAR, in topological (stream) order.
    // Reference recognition resolves against this incrementally-grown set, not
    // the whole repo, so an abbreviation resolves to the object it named when
    // it was written even if a later commit shares its prefix (temporal
    // awareness).  Blobs are intentionally excluded: references target commits
    // and tags in practice, and indexing every blob would bloat this set.
    std::set<std::string> old_ids;
    // fast-import mark -> rebuilt object id, for resolving blob/parent refs.
    std::unordered_map<long, git_oid> mark_to_oid;

    size_t n_blobs = 0, n_commits = 0, n_tags = 0, n_identical = 0, n_total = 0;
    int n_rewrites = 0;
    size_t n_idx_reuse = 0, n_idx_rebuild = 0;   // index fast-path diagnostics
    size_t n_signed = 0;
    // Abbreviated references we emitted, for a final uniqueness check.
    std::set<std::string> emitted_abbrevs;

    // Temp files used to hand commit payloads to the external signer.
    std::string sign_in, sign_out;
    if (!sign_key.empty()) {
#if defined(_WIN32)
	unsigned long pid = GetCurrentProcessId();
#else
	unsigned long pid = (unsigned long)getpid();
#endif
	std::filesystem::path td = std::filesystem::temp_directory_path();
	sign_in  = (td / ("repowork-sign-" + std::to_string(pid) + ".in")).string();
	sign_out = (td / ("repowork-sign-" + std::to_string(pid) + ".sig")).string();
    }

    // ---- Blob creation -----------------------------------------------
    // Blobs are created interleaved with commits (in mark == stream order)
    // rather than all up front, so that when a text blob's SHA1 references
    // are rewritten the ancestor commits it names have already been rebuilt
    // and are therefore present in old_to_new.
    size_t bi = 0;
    auto create_blob = [&](git_blob_data &b) {
	std::vector<char> data;
	if (from_repo) {
	    git_oid boid;
	    git_odb_object *srcobj = NULL;
	    if (git_oid_fromstr(&boid, b.id.sha1.c_str()) == 0 &&
		git_odb_read(&srcobj, src_odb, &boid) == 0) {
		const char *d = (const char *)git_odb_object_data(srcobj);
		data.assign(d, d + git_odb_object_size(srcobj));
		git_odb_object_free(srcobj);
	    } else {
		std::cerr << "replay: could not read blob " << b.id.sha1
			  << " from source repo\n";
		exit(1);
	    }
	} else {
	    data.resize(b.length);
	    if (b.cbuffer) {
		memcpy(data.data(), b.cbuffer, b.length);
	    } else if (b.length) {
		infile.clear();
		infile.seekg(b.offset);
		infile.read(data.data(), b.length);
	    }
	}

	// Optionally rewrite SHA1 references inside text blobs.  Binary blobs
	// (images, terrain, etc.) are skipped using git's own heuristic (NUL
	// bytes plus a printable/non-printable ratio over the first 8000
	// bytes) so we never scribble on random binary content.  The
	// uniqueness guard in rewrite_shas() is a second line of defense.
	if (rewrite_blob_refs && b.length &&
	    !git_blob_data_is_binary(data.data(), b.length)) {
	    std::string content(data.data(), b.length);
	    std::string nc = rewrite_shas(repo, content, old_to_new, old_ids, ref_min_len, n_rewrites, emitted_abbrevs);
	    if (nc != content)
		data.assign(nc.begin(), nc.end());
	}

	git_oid oid;
	lg2_fatal(git_blob_create_from_buffer(&oid, repo, data.data(), data.size()),
		  "blob_create_from_buffer");
	mark_to_oid[b.id.mark] = oid;
	pending_bytes += data.size();
	if (b.id.sha1.length()) {
	    std::string nh = oid_hex(&oid);
	    old_to_new[b.id.sha1] = nh;   // blobs recorded in the map, but not
	    n_total++;                    // added to the recognition set above
	    if (b.id.sha1 == nh) n_identical++;
	}
	n_blobs++;
    };
    auto flush_blobs_before = [&](long mark) {
	while (bi < s->blobs.size() && s->blobs[bi].id.mark < mark)
	    create_blob(s->blobs[bi++]);
    };

    // ---- Commits (and resets, in stream order) -----------------------
    std::map<std::string, git_oid> branch_tips;
    // Persistent index reused across linear commit chains.  fast-export emits
    // each commit's fileops as a diff from its first parent, so when that
    // parent is the commit we just built we can apply the diff in place and
    // skip re-reading the whole parent tree (the dominant cost).  cur_idx_mark
    // is the mark of the commit whose tree the index currently holds.
    git_index *cur_idx = NULL;
    long cur_idx_mark = -1;

    // Periodic progress reporting so long replays are not silent.
    const double progress_interval_s = 30.0;
    const size_t commits_total = s->commits.size();
    const auto t_start = std::chrono::steady_clock::now();
    auto t_last = t_start;
    auto report_progress = [&](bool force) {
	auto now = std::chrono::steady_clock::now();
	if (!force &&
	    std::chrono::duration<double>(now - t_last).count() < progress_interval_s)
	    return;
	t_last = now;
	double el = std::chrono::duration<double>(now - t_start).count();
	double orate = el > 0 ? (double)(n_blobs + n_commits) / el : 0.0;
	double crate = el > 0 ? (double)n_commits / el : 0.0;
	double eta = crate > 0 ? (commits_total - n_commits) / crate : 0.0;
	int pct = commits_total ? (int)(100.0 * n_commits / commits_total) : 0;
	std::cerr << "  replay: " << n_commits << "/" << commits_total
		  << " commits (" << pct << "%), " << n_blobs << " blobs, "
		  << (int)el << "s elapsed, ~" << (int)orate << " obj/s, ETA ~"
		  << (int)eta << "s\n";
    };

    for (size_t i = 0; i < s->commits.size(); i++) {
	git_commit_data *c = &s->commits[i];
	if (c->skip_commit)
	    continue;

	// A reset is just a ref assignment (used for e.g. lightweight tags).
	if (c->reset_commit) {
	    if (c->from.mark != -1 && mark_to_oid.count(c->from.mark)) {
		git_oid o = mark_to_oid[c->from.mark];
		git_reference *ref = NULL;
		lg2_fatal(git_reference_create(&ref, repo, c->branch.c_str(), &o, 1, NULL),
			  "reference_create(reset)");
		if (ref) git_reference_free(ref);
	    }
	    continue;
	}
	if (c->notes_commit)
	    continue;

	// Create any blobs that precede this commit in the stream, so their
	// (possibly rewritten) ids are available for this commit's tree.
	flush_blobs_before(c->id.mark);

	// Parents: first "from", then any merges.
	std::vector<git_commit *> parents;
	git_commit *first_parent = NULL;
	long fp_mark = -1;
	if (c->from.mark != -1 && mark_to_oid.count(c->from.mark)) {
	    git_oid po = mark_to_oid[c->from.mark];
	    lg2_fatal(git_commit_lookup(&first_parent, repo, &po), "commit_lookup(from)");
	    parents.push_back(first_parent);
	    fp_mark = c->from.mark;
	}
	for (size_t m = 0; m < c->merges.size(); m++) {
	    if (c->merges[m].mark == -1 || !mark_to_oid.count(c->merges[m].mark))
		continue;
	    git_oid mo = mark_to_oid[c->merges[m].mark];
	    git_commit *mc = NULL;
	    lg2_fatal(git_commit_lookup(&mc, repo, &mo), "commit_lookup(merge)");
	    parents.push_back(mc);
	}

	// Build this commit's tree.  Fast path: if the persistent index already
	// holds the first parent's tree (linear history), apply the diff in
	// place.  Otherwise (root, merge whose first parent isn't the previous
	// commit, or a branch switch) rebuild the base by reading the first
	// parent's tree.
	if (cur_idx && fp_mark != -1 && cur_idx_mark == fp_mark) {
	    // reuse: index already holds the first parent's tree
	    n_idx_reuse++;
	} else {
	    n_idx_rebuild++;
	    if (!cur_idx)
		lg2_fatal(git_index_new(&cur_idx), "index_new");
	    else
		git_index_clear(cur_idx);
	    if (first_parent) {
		git_tree *base_tree = NULL;
		lg2_fatal(git_commit_tree(&base_tree, first_parent), "commit_tree(from)");
		lg2_fatal(git_index_read_tree(cur_idx, base_tree), "index_read_tree");
		git_tree_free(base_tree);
	    }
	}
	for (size_t k = 0; k < c->fileops.size(); k++)
	    apply_fileop(cur_idx, c->fileops[k], mark_to_oid, old_to_new);

	git_oid tree_oid;
	lg2_fatal(git_index_write_tree_to(&tree_oid, cur_idx, repo), "index_write_tree_to");
	cur_idx_mark = c->id.mark;   // the index now holds THIS commit's tree

	std::string msg = commit_msg(c);
	if (rewrite_refs)
	    msg = rewrite_shas(repo, msg, old_to_new, old_ids, ref_min_len, n_rewrites, emitted_abbrevs);

	// Assemble the commit object with the ORIGINAL author/committer bytes
	// preserved verbatim.  git_signature + git_commit_create would normalize
	// identity whitespace (e.g. "Name  <email>" -> "Name <email>") and change
	// the commit id, breaking byte-exact reproduction.
	std::string header = "tree " + oid_hex(&tree_oid) + "\n";
	for (git_commit *p : parents)
	    header += "parent " + oid_hex(git_commit_id(p)) + "\n";
	const std::string &aid = c->author.length() ? c->author : c->committer;
	const std::string &ats = c->author.length() ? c->author_timestamp : c->committer_timestamp;
	header += "author " + format_ident(aid, ats, normalize_identity) + "\n";
	header += "committer " + format_ident(c->committer, c->committer_timestamp, normalize_identity) + "\n";

	std::string trailer;   // headers after "committer" (encoding, gpgsig, ...)
	if (!sign_key.empty()) {
	    // Re-sign: drop any existing signature, sign the payload (the commit
	    // as it would be unsigned), and append a fresh gpgsig header.  git
	    // verifies by removing the gpgsig header and checking the remainder,
	    // so the signed payload must equal header + trailer + "\n" + msg.
	    trailer = strip_gpgsig(c->extra_headers);
	    std::string payload = header + trailer + "\n" + msg;
	    std::string sig = sign_payload(payload, gpg_program, sign_key, sign_in, sign_out);
	    trailer += make_gpgsig_header(sig);
	    n_signed++;
	} else if (!c->extra_headers.empty()) {
	    // Preserve trailing headers verbatim only when the commit is
	    // unchanged; otherwise the signature would be invalid, so drop just
	    // the gpgsig and keep the rest.
	    bool unchanged = (c->orig_tree_sha1.length() &&
			      oid_hex(&tree_oid) == c->orig_tree_sha1);
	    size_t pi = 0;
	    if (unchanged && c->from.mark != -1 && pi < parents.size()) {
		if (oid_hex(git_commit_id(parents[pi])) != c->from.sha1)
		    unchanged = false;
		pi++;
	    }
	    for (size_t m = 0; unchanged && m < c->merges.size(); m++, pi++) {
		if (pi >= parents.size() ||
		    oid_hex(git_commit_id(parents[pi])) != c->merges[m].sha1)
		    unchanged = false;
	    }
	    trailer = unchanged ? c->extra_headers : strip_gpgsig(c->extra_headers);
	}
	std::string cbuf = header + trailer + "\n" + msg;

	git_oid new_oid;
	lg2_fatal(git_odb_write(&new_oid, odb, cbuf.data(), cbuf.size(), GIT_OBJECT_COMMIT),
		  "odb_write(commit)");

	mark_to_oid[c->id.mark] = new_oid;
	pending_bytes += cbuf.size();
	if (c->id.sha1.length()) {
	    std::string nh = oid_hex(&new_oid);
	    old_to_new[c->id.sha1] = nh;
	    old_ids.insert(c->id.sha1);
	    n_total++;
	    if (c->id.sha1 == nh) n_identical++;
	}
	if (c->branch.length())
	    branch_tips[c->branch] = new_oid;
	n_commits++;
	report_progress(false);

	for (size_t p = 0; p < parents.size(); p++)
	    git_commit_free(parents[p]);
	// cur_idx persists across iterations; freed after the loop.

	// Staged flush: bound peak memory by writing a packfile once the
	// in-memory objects exceed the budget.  Done at a commit boundary so
	// only complete objects are flushed.  The reused index (cur_idx) is
	// unaffected: after the flush its cached subtree oids resolve from the
	// written pack via the refreshed odb.
	if (use_mempack && pending_bytes >= flush_threshold)
	    flush_pack();
    }
    if (cur_idx)
	git_index_free(cur_idx);

    // Any blobs that follow the last commit in the stream.
    while (bi < s->blobs.size())
	create_blob(s->blobs[bi++]);

    // ---- Branch refs -------------------------------------------------
    // Stream mode: branch tip per commit's branch field.
    for (auto &bt : branch_tips) {
	std::string ref = std::string("refs/heads/") + bt.first;
	git_reference *r = NULL;
	lg2_fatal(git_reference_create(&r, repo, ref.c_str(), &bt.second, 1, NULL),
		  "reference_create(branch)");
	if (r) git_reference_free(r);
    }
    // Repo-read mode: full ref names (branches + lightweight tags) -> commit.
    for (auto &rm : s->ref_to_mark) {
	if (!mark_to_oid.count(rm.second))
	    continue;
	git_oid o = mark_to_oid[rm.second];
	git_reference *r = NULL;
	lg2_fatal(git_reference_create(&r, repo, rm.first.c_str(), &o, 1, NULL),
		  "reference_create(ref)");
	if (r) git_reference_free(r);
    }
    // Symbolic refs (HEAD, refs/remotes/*/HEAD, ...) mirrored from the source.
    for (auto &sr : s->symbolic_refs) {
	git_reference *r = NULL;
	if (git_reference_symbolic_create(&r, repo, sr.first.c_str(), sr.second.c_str(),
					  1, NULL) == 0 && r)
	    git_reference_free(r);
    }

    // ---- Annotated tags ----------------------------------------------
    for (size_t i = 0; i < s->tags.size(); i++) {
	git_tag_data *t = &s->tags[i];
	if (t->from.mark == -1 || !mark_to_oid.count(t->from.mark))
	    continue;
	git_oid to = mark_to_oid[t->from.mark];
	std::string tmsg = t->tag_msg;
	if (rewrite_refs)
	    tmsg = rewrite_shas(repo, tmsg, old_to_new, old_ids, ref_min_len, n_rewrites, emitted_abbrevs);

	// Assemble the tag object with the ORIGINAL tagger bytes preserved
	// verbatim (git_tag_create would normalize the tagger and change the id).
	std::string tbuf = "object " + oid_hex(&to) + "\n";
	tbuf += "type commit\n";
	tbuf += "tag " + t->tag + "\n";
	if (t->tagger.length())
	    tbuf += "tagger " + format_ident(t->tagger, t->tagger_timestamp, normalize_identity) + "\n";
	tbuf += "\n";
	tbuf += tmsg;
	git_oid tag_oid;
	lg2_fatal(git_odb_write(&tag_oid, odb, tbuf.data(), tbuf.size(), GIT_OBJECT_TAG),
		  "odb_write(tag)");
	pending_bytes += tbuf.size();

	std::string tref = "refs/tags/" + t->tag;
	git_reference *tr = NULL;
	lg2_fatal(git_reference_create(&tr, repo, tref.c_str(), &tag_oid, 1, NULL),
		  "reference_create(tag)");
	if (tr) git_reference_free(tr);

	if (t->id.sha1.length()) {
	    std::string nh = oid_hex(&tag_oid);
	    old_to_new[t->id.sha1] = nh;
	    old_ids.insert(t->id.sha1);
	    n_total++;
	    if (t->id.sha1 == nh) n_identical++;
	}
	n_tags++;
    }

    // Final flush of any remaining in-memory objects (including tags), so the
    // verification below and all later reads resolve from on-disk packs.
    if (use_mempack)
	flush_pack();

    // ---- Verify emitted abbreviations are unambiguous in the FINAL repo ---
    // unique_abbrev_len() only sees objects that existed when each reference
    // was rewritten; objects added afterwards could, in rare cases, collide.
    // Re-check every distinct abbreviation against the completed object db.
    size_t n_ambiguous = 0;
    if (!emitted_abbrevs.empty()) {
	for (const std::string &ab : emitted_abbrevs) {
	    git_oid shid, full;
	    if (git_oid_fromstrn(&shid, ab.c_str(), ab.size()) != 0)
		continue;
	    if (git_odb_exists_prefix(&full, odb, &shid, ab.size()) == GIT_EAMBIGUOUS)
		n_ambiguous++;
	}
    }

    // ---- Write the old->new map --------------------------------------
    if (map_file.length()) {
	std::ofstream mf(map_file, std::ios::out | std::ios::binary);
	if (!mf.good()) {
	    std::cerr << "replay: could not open map file " << map_file << " for writing\n";
	} else {
	    for (auto &kv : old_to_new)
		mf << kv.first << ";" << kv.second << "\n";
	    mf.close();
	}
    }

    if (!sign_key.empty()) {
	std::remove(sign_in.c_str());
	std::remove(sign_out.c_str());
    }
    if (src_odb) git_odb_free(src_odb);
    if (src_repo) git_repository_free(src_repo);
    git_odb_free(odb);
    git_repository_free(repo);
    git_libgit2_shutdown();

    std::cout << "Replayed " << n_blobs << " blobs, " << n_commits
	      << " commits, " << n_tags << " tags into " << out_repo << "\n";
    std::cout << "Tree build: " << n_idx_reuse << " incremental (fast path), "
	      << n_idx_rebuild << " full reloads\n";
    if (use_mempack)
	std::cout << "Wrote " << n_packs << " packfile(s)\n";
    if (n_signed)
	std::cout << "Signed " << n_signed << " commit(s) with key '" << sign_key << "'\n";
    if (n_total)
	std::cout << n_identical << " of " << n_total
		  << " objects with original ids kept the same id "
		  << (n_identical == n_total ? "(history preserved bit-for-bit)" : "(history changed)")
		  << "\n";
    if (rewrite_refs || rewrite_blob_refs) {
	std::cout << "Rewrote " << n_rewrites
		  << " SHA1 reference(s) in "
		  << (rewrite_refs && rewrite_blob_refs ? "messages and text blobs"
		      : rewrite_refs ? "commit/tag messages" : "text blobs")
		  << "\n";
	if (n_ambiguous)
	    std::cout << "WARNING: " << n_ambiguous
		      << " abbreviated reference(s) are ambiguous in the rebuilt repo "
			 "(consider a longer --ref-min-len)\n";
	else if (!emitted_abbrevs.empty())
	    std::cout << "Verified " << emitted_abbrevs.size()
		      << " distinct abbreviated reference(s) resolve uniquely\n";
    }
    if (map_file.length())
	std::cout << "Wrote old->new SHA1 map: " << map_file << "\n";

    return 0;
}

// Local Variables:
// tab-width: 8
// mode: C++
// c-basic-offset: 4
// indent-tabs-mode: t
// c-file-style: "stroustrup"
// End:
// ex: shiftwidth=4 tabstop=8