repo_read.cpp

/*                    R E P O _ R E A D . C P P
 * BRL-CAD
 *
 * Published in 2020 by the United States Government.
 * This work is in the public domain.
 *
 */
/** @file repo_read.cpp
 *
 * Read a git repository directly into repowork's in-memory model using
 * libgit2, as an alternative to parsing a fast-import stream.  Reading the
 * repository directly avoids the git-version limitations and quoting/encoding
 * fragility of `git fast-export`, and - crucially - gives byte-exact access to
 * the raw commit/tag objects, including GPG signatures, so signed commits can
 * be reproduced exactly (or have their now-invalid signatures dropped when the
 * commit is rewritten).
 *
 * Commits are enumerated in topological (parent-before-child) order.  Each
 * commit's tree change is expressed as fast-import-style fileops (a diff from
 * its first parent), which is exactly what the replay engine already consumes;
 * blob content is fetched from the source repository on demand during replay.
 */

#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <map>
#include <string>
#include <vector>

#include <git2.h>

#include "repowork.h"

namespace {

void
lg2r_fatal(int err, const char *op)
{
    if (err < 0) {
	const git_error *e = git_error_last();
	std::cerr << "libgit2 error (" << op << "): "
		  << ((e && e->message) ? e->message : "unknown") << "\n";
	exit(1);
    }
}

std::string
oid_to_hex(const git_oid *o)
{
    char buf[GIT_OID_MAX_HEXSIZE + 1];
    git_oid_tostr(buf, sizeof(buf), o);
    return std::string(buf);
}

std::string
mode_str(uint32_t m)
{
    char buf[16];
    snprintf(buf, sizeof(buf), "%o", (unsigned)m);
    return std::string(buf);
}

// Split "Name <email> <timestamp> <tz>" into ident ("Name <email>") and the
// timestamp remainder, matching how the fast-import parser stores them.
void
split_ident(const std::string &s, std::string &ident, std::string &tstamp)
{
    size_t gt = s.find_last_of('>');
    if (gt != std::string::npos) {
	ident = s.substr(0, gt + 1);
	tstamp = (gt + 2 <= s.size()) ? s.substr(gt + 2) : "";
    } else {
	ident = s;
	tstamp = "";
    }
}

} // anonymous namespace

int
git_read_repo(git_fi_data *s, const std::string &repo_path)
{
    git_libgit2_init();

    git_repository *repo = NULL;
    lg2r_fatal(git_repository_open(&repo, repo_path.c_str()), "repository_open");

    s->src_repo_path = repo_path;
    s->have_sha1s = true;

    git_odb *odb = NULL;
    lg2r_fatal(git_repository_odb(&odb, repo), "repository_odb");

    std::map<std::string, long> commit_mark;   // commit oid hex -> mark
    std::map<std::string, long> blob_mark;      // blob oid hex   -> mark

    // Ensure a blob (by oid) has a mark; content is read from the source repo
    // during replay, so only the id and size are recorded here (the size is
    // read cheaply from the object header for the replay memory estimate).
    auto ensure_blob = [&](const git_oid *boid) -> long {
	std::string h = oid_to_hex(boid);
	auto it = blob_mark.find(h);
	if (it != blob_mark.end())
	    return it->second;
	git_blob_data b;
	b.s = s;
	b.id.index = (long)s->blobs.size();
	b.id.sha1 = h;
	b.id.mark = s->next_mark(-1);
	b.offset = 0;
	b.length = 0;
	b.cbuffer = NULL;
	size_t blen = 0;
	git_object_t btype;
	if (git_odb_read_header(&blen, &btype, odb, boid) == 0)
	    b.length = blen;
	s->mark_to_index[b.id.mark] = b.id.index;
	s->sha1_to_mark[b.id.sha1] = b.id.mark;
	s->mark_to_sha1[b.id.mark] = b.id.sha1;
	s->blobs.push_back(b);
	blob_mark[h] = b.id.mark;
	return b.id.mark;
    };

    // ---- Commits, in topological (oldest-first) order ------------------
    git_revwalk *walk = NULL;
    lg2r_fatal(git_revwalk_new(&walk, repo), "revwalk_new");
    git_revwalk_sorting(walk, GIT_SORT_TOPOLOGICAL | GIT_SORT_REVERSE);
    {
	git_reference_iterator *rit = NULL;
	lg2r_fatal(git_reference_iterator_new(&rit, repo), "reference_iterator_new");
	git_reference *ref = NULL;
	while (git_reference_next(&ref, rit) == 0) {
	    git_object *peeled = NULL;
	    if (git_reference_peel(&peeled, ref, GIT_OBJECT_COMMIT) == 0) {
		git_revwalk_push(walk, git_object_id(peeled));
		git_object_free(peeled);
	    }
	    git_reference_free(ref);
	}
	git_reference_iterator_free(rit);
    }

    size_t n_signed = 0;
    const auto t_start = std::chrono::steady_clock::now();
    auto t_last = t_start;
    git_oid oid;
    while (git_revwalk_next(&oid, walk) == 0) {
	if (std::chrono::duration<double>(std::chrono::steady_clock::now() - t_last).count() >= 30.0) {
	    t_last = std::chrono::steady_clock::now();
	    double el = std::chrono::duration<double>(t_last - t_start).count();
	    std::cerr << "  reading repo: " << s->commits.size() << " commits, "
		      << s->blobs.size() << " blobs, " << (int)el << "s elapsed\n";
	}
	git_commit *commit = NULL;
	lg2r_fatal(git_commit_lookup(&commit, repo, &oid), "commit_lookup");

	git_commit_data gcd;
	gcd.s = s;
	gcd.id.index = (long)s->commits.size();
	gcd.id.sha1 = oid_to_hex(&oid);

	git_tree *ctree = NULL;
	lg2r_fatal(git_commit_tree(&ctree, commit), "commit_tree");
	gcd.orig_tree_sha1 = oid_to_hex(git_tree_id(ctree));

	// Parents: first -> from, remainder -> merges.  All parents precede
	// this commit in reverse-topological order, so their marks exist.
	unsigned int np = git_commit_parentcount(commit);
	git_tree *ptree = NULL;
	for (unsigned int p = 0; p < np; p++) {
	    const git_oid *poid = git_commit_parent_id(commit, p);
	    std::string ph = oid_to_hex(poid);
	    git_commitish pc;
	    pc.sha1 = ph;
	    pc.mark = commit_mark.count(ph) ? commit_mark[ph] : -1;
	    if (pc.mark != -1 && s->mark_to_index.count(pc.mark))
		pc.index = s->mark_to_index[pc.mark];
	    if (p == 0) {
		gcd.from = pc;
		git_commit *pcommit = NULL;
		if (git_commit_lookup(&pcommit, repo, poid) == 0) {
		    git_commit_tree(&ptree, pcommit);
		    git_commit_free(pcommit);
		}
	    } else {
		gcd.merges.push_back(pc);
	    }
	}

	// Tree change relative to the first parent, as M/D fileops.
	git_diff *diff = NULL;
	git_diff_options dopts = GIT_DIFF_OPTIONS_INIT;
	lg2r_fatal(git_diff_tree_to_tree(&diff, repo, ptree, ctree, &dopts),
		   "diff_tree_to_tree");
	size_t nd = git_diff_num_deltas(diff);
	for (size_t d = 0; d < nd; d++) {
	    const git_diff_delta *delta = git_diff_get_delta(diff, d);
	    git_op op;
	    if (delta->status == GIT_DELTA_DELETED) {
		op.type = filedelete;
		op.path = std::string(delta->old_file.path);
	    } else {
		op.type = filemodify;
		op.mode = mode_str(delta->new_file.mode);
		op.path = std::string(delta->new_file.path);
		op.dataref.sha1 = oid_to_hex(&delta->new_file.id);
		if (delta->new_file.mode == GIT_FILEMODE_COMMIT) {
		    // Gitlink (submodule): the id names a commit in another
		    // repository, not a blob here.  Keep the oid in the tree
		    // entry but do not try to read or materialize it.
		    op.dataref.mark = -1;
		} else {
		    op.dataref.mark = ensure_blob(&delta->new_file.id);
		}
	    }
	    gcd.fileops.push_back(op);
	}
	git_diff_free(diff);
	if (ptree) git_tree_free(ptree);
	git_tree_free(ctree);

	// Author / committer / trailing headers, captured verbatim.
	std::string hdr(git_commit_raw_header(commit) ? git_commit_raw_header(commit) : "");
	std::string extra;
	bool seen_committer = false;
	size_t pos = 0;
	while (pos < hdr.size()) {
	    size_t eol = hdr.find('\n', pos);
	    if (eol == std::string::npos) eol = hdr.size();
	    std::string line = hdr.substr(pos, eol - pos);
	    if (seen_committer) {
		extra += line;
		extra += "\n";
	    } else if (line.compare(0, 7, "author ") == 0) {
		split_ident(line.substr(7), gcd.author, gcd.author_timestamp);
	    } else if (line.compare(0, 10, "committer ") == 0) {
		split_ident(line.substr(10), gcd.committer, gcd.committer_timestamp);
		seen_committer = true;
	    }
	    pos = eol + 1;
	}
	gcd.extra_headers = extra;
	if (extra.find("gpgsig ") != std::string::npos)
	    n_signed++;

	const char *msg = git_commit_message_raw(commit);
	gcd.commit_msg = std::string(msg ? msg : "");

	gcd.id.mark = s->next_mark(-1);
	s->mark_to_index[gcd.id.mark] = gcd.id.index;
	s->sha1_to_mark[gcd.id.sha1] = gcd.id.mark;
	commit_mark[gcd.id.sha1] = gcd.id.mark;

	s->commits.push_back(gcd);
	git_commit_free(commit);
    }
    git_revwalk_free(walk);

    // ---- Refs: branch/lightweight-tag -> ref_to_mark; annotated -> tags --
    {
	git_reference_iterator *rit = NULL;
	lg2r_fatal(git_reference_iterator_new(&rit, repo), "reference_iterator_new(refs)");
	git_reference *ref = NULL;
	while (git_reference_next(&ref, rit) == 0) {
	    if (git_reference_type(ref) == GIT_REFERENCE_SYMBOLIC) {
		const char *tgt = git_reference_symbolic_target(ref);
		if (tgt)
		    s->symbolic_refs[git_reference_name(ref)] = tgt;
		git_reference_free(ref);
		continue;
	    }
	    std::string name = git_reference_name(ref);
	    const git_oid *toid = git_reference_target(ref);
	    git_object *obj = NULL;
	    if (git_object_lookup(&obj, repo, toid, GIT_OBJECT_ANY) != 0) {
		git_reference_free(ref);
		continue;
	    }
	    git_object_t ot = git_object_type(obj);

	    if (ot == GIT_OBJECT_TAG) {
		// Annotated tag: reproduce the tag object from its raw bytes.
		git_tag_data td;
		td.s = s;
		td.id.index = (long)s->tags.size();
		td.id.sha1 = oid_to_hex(toid);

		// Target commit (peeled) -> mark.
		git_object *peeled = NULL;
		if (git_object_peel(&peeled, obj, GIT_OBJECT_COMMIT) == 0) {
		    std::string ch = oid_to_hex(git_object_id(peeled));
		    td.from.sha1 = ch;
		    td.from.mark = commit_mark.count(ch) ? commit_mark[ch] : -1;
		    git_object_free(peeled);
		}

		// Parse the raw tag object for verbatim tag name, tagger, and
		// message (which for signed tags includes the signature block).
		git_odb_object *raw = NULL;
		if (git_odb_read(&raw, odb, toid) == 0) {
		    std::string t((const char *)git_odb_object_data(raw),
				  git_odb_object_size(raw));
		    size_t blank = t.find("\n\n");
		    std::string thdr = (blank == std::string::npos) ? t : t.substr(0, blank);
		    td.tag_msg = (blank == std::string::npos) ? std::string()
							       : t.substr(blank + 2);
		    size_t lp = 0;
		    while (lp < thdr.size()) {
			size_t eol = thdr.find('\n', lp);
			if (eol == std::string::npos) eol = thdr.size();
			std::string line = thdr.substr(lp, eol - lp);
			if (line.compare(0, 4, "tag ") == 0)
			    td.tag = line.substr(4);
			else if (line.compare(0, 7, "tagger ") == 0)
			    split_ident(line.substr(7), td.tagger, td.tagger_timestamp);
			lp = eol + 1;
		    }
		    git_odb_object_free(raw);
		}
		td.id.mark = s->next_mark(-1);
		s->sha1_to_mark[td.id.sha1] = td.id.mark;
		s->tags.push_back(td);
	    } else if (ot == GIT_OBJECT_COMMIT) {
		std::string ch = oid_to_hex(toid);
		if (commit_mark.count(ch))
		    s->ref_to_mark[name] = commit_mark[ch];
	    }
	    git_object_free(obj);
	    git_reference_free(ref);
	}
	git_reference_iterator_free(rit);
    }

    // HEAD is not enumerated by the ref iterator; reproduce it if symbolic.
    {
	git_reference *head = NULL;
	if (git_reference_lookup(&head, repo, "HEAD") == 0) {
	    if (git_reference_type(head) == GIT_REFERENCE_SYMBOLIC) {
		const char *tgt = git_reference_symbolic_target(head);
		if (tgt)
		    s->symbolic_refs["HEAD"] = tgt;
	    }
	    git_reference_free(head);
	}
    }

    std::cout << "Read " << s->commits.size() << " commits, " << s->blobs.size()
	      << " blobs, " << s->tags.size() << " tags from " << repo_path
	      << " (" << n_signed << " signed commits)\n";

    git_odb_free(odb);
    git_repository_free(repo);
    // NOTE: git_libgit2_shutdown() is issued by the replay stage.
    return 0;
}

// Local Variables:
// tab-width: 8
// mode: C++
// c-basic-offset: 4
// indent-tabs-mode: t
// c-file-style: "stroustrup"
// End:
// ex: shiftwidth=4 tabstop=8