Support hash to location endpoint

Summary:
Enabling hash to location endpoint for git-over-slapi. The "signature" of the request is `/commit/lhash_to_location (CommitHashToLocationRequestBatch) -> CommitHashToLocationResponse`

where

```
pub struct CommitHashToLocationRequestBatch {
    pub master_heads: Vec<HgId>,
    pub hgids: Vec<HgId>,
    pub unfiltered: Option<bool>,
}

pub struct CommitHashToLocationResponse {
    pub hgid: HgId,
    pub result: Result<Option<Location<HgId>>, ServerError>,
}
```

we'll reuse `HgId` as git hashes for now. We'll simply put the 20B git hash in HgId, but later on we'll change it to something like Id20 type of structure that will hold both hg and git hashes.

Reviewed By: markbt

Differential Revision: D64134940

fbshipit-source-id: 763feed2198cc132602072e1fb47c77d5b91a8ae

eden/mononoke/bonsai_git_mapping/src/lib.rs

@@ -40,6 +40,7 @@ impl BonsaiGitMappingEntry {
  }
 }
 
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
 pub enum BonsaisOrGitShas {
  Bonsai(Vec<ChangesetId>),
  GitSha1(Vec<GitSha1>),

eden/mononoke/edenapi_service/src/handlers/commit.rs

@@ -72,6 +72,7 @@ use mononoke_api::CreateInfo;
 use mononoke_api::MononokeError;
 use mononoke_api::MononokeRepo;
 use mononoke_api::Repo;
+use mononoke_api::RepoContext;
 use mononoke_api::XRepoLookupExactBehaviour;
 use mononoke_api::XRepoLookupSyncBehaviour;
 use mononoke_api_hg::HgRepoContext;
@@ -198,13 +199,13 @@ impl SaplingRemoteApiHandler for LocationToHashHandler {
 }
 
 pub async fn hash_to_location(state: &mut State) -> Result<impl TryIntoResponse, HttpError> {
- async fn hash_to_location_chunk<R: MononokeRepo>(
+ async fn hg_hash_to_location_chunk<R: MononokeRepo>(
  hg_repo_ctx: HgRepoContext<R>,
  master_heads: Vec<HgChangesetId>,
  hg_cs_ids: Vec<HgChangesetId>,
  ) -> impl Stream<Item = CommitHashToLocationResponse> {
  let hgcsid_to_location = hg_repo_ctx
- .many_changeset_ids_to_locations(master_heads, hg_cs_ids.clone())
+ .many_hg_changeset_ids_to_locations(master_heads, hg_cs_ids.clone())
  .await;
  let responses = hg_cs_ids.into_iter().map(move |hgcsid| {
  let result = match hgcsid_to_location.as_ref() {
@@ -223,6 +224,31 @@ pub async fn hash_to_location(state: &mut State) -> Result<impl TryIntoResponse,
  stream::iter(responses)
  }
 
+ async fn git_hash_to_location_chunk<R: MononokeRepo>(
+ repo_ctx: RepoContext<R>,
+ master_heads: Vec<GitSha1>,
+ git_commit_ids: Vec<GitSha1>,
+ ) -> impl Stream<Item = CommitHashToLocationResponse> {
+ let git_commit_id_to_location = repo_ctx
+ .many_git_commit_ids_to_locations(master_heads, git_commit_ids.clone())
+ .await;
+ let responses = git_commit_ids.into_iter().map(move |git_commit_id| {
+ let result = match git_commit_id_to_location.as_ref() {
+ Ok(hsh) => match hsh.get(&git_commit_id) {
+ Some(Ok(l)) => Ok(Some(l.map_descendant(|x| HgId::from(x.into_inner())))),
+ Some(Err(e)) => Err(e.into()),
+ None => Ok(None),
+ },
+ Err(e) => Err(e.into()),
+ };
+ CommitHashToLocationResponse {
+ hgid: HgId::from(git_commit_id.into_inner()),
+ result,
+ }
+ });
+ stream::iter(responses)
+ }
+
  let params = HashToLocationParams::take_from(state);
 
  state.put(HandlerInfo::new(
@@ -248,16 +274,43 @@ pub async fn hash_to_location(state: &mut State) -> Result<impl TryIntoResponse,
  .into_iter()
  .map(|x| x.into())
  .collect::<Vec<_>>();
-
- let response = stream::iter(batch.hgids)
- .chunks(HASH_TO_LOCATION_BATCH_SIZE)
- .map(|chunk| chunk.into_iter().map(|x| x.into()).collect::<Vec<_>>())
- .map({
- let ctx = hg_repo_ctx.clone();
- move |chunk| hash_to_location_chunk(ctx.clone(), master_heads.clone(), chunk)
- })
- .buffer_unordered(3)
- .flatten();
+ let response = match SlapiCommitIdentityScheme::try_take_from(state) {
+ Some(SlapiCommitIdentityScheme::Hg) | None => stream::iter(batch.hgids)
+ .chunks(HASH_TO_LOCATION_BATCH_SIZE)
+ .map(|chunk| chunk.into_iter().map(|x| x.into()).collect::<Vec<_>>())
+ .map({
+ let ctx = hg_repo_ctx.clone();
+ move |chunk| hg_hash_to_location_chunk(ctx.clone(), master_heads.clone(), chunk)
+ })
+ .buffer_unordered(3)
+ .flatten()
+ .left_stream(),
+ Some(SlapiCommitIdentityScheme::Git) => {
+ // TODO(mbthomas): This is a working around HgId/HgChangesetId not being "generic".
+ // This should be cleaned up when we have a generic Id20 type
+ stream::iter(batch.hgids)
+ .chunks(HASH_TO_LOCATION_BATCH_SIZE)
+ .map(|chunk| {
+ chunk
+ .into_iter()
+ .map(|x| GitSha1::from(x.into_byte_array()))
+ .collect::<Vec<_>>()
+ })
+ .map({
+ let ctx = hg_repo_ctx.repo_ctx().clone();
+ let master_heads = master_heads
+ .into_iter()
+ .map(|x| GitSha1::from(x.into_nodehash().sha1().into_byte_array()))
+ .collect::<Vec<_>>();
+ move |chunk| {
+ git_hash_to_location_chunk(ctx.clone(), master_heads.clone(), chunk)
+ }
+ })
+ .buffer_unordered(3)
+ .flatten()
+ .right_stream()
+ }
+ };
  let cbor_response = custom_cbor_stream(super::monitor_request(state, response), |t| {
  t.result.as_ref().err()
  });

eden/mononoke/mononoke_api/src/repo.rs

@@ -22,6 +22,7 @@ use blobrepo_hg::BlobRepoHg;
 use blobstore::Loadable;
 use bonsai_git_mapping::BonsaiGitMapping;
 use bonsai_git_mapping::BonsaiGitMappingRef;
+use bonsai_git_mapping::BonsaisOrGitShas;
 use bonsai_globalrev_mapping::BonsaiGlobalrevMapping;
 use bonsai_globalrev_mapping::BonsaiGlobalrevMappingRef;
 use bonsai_hg_mapping::BonsaiHgMapping;
@@ -85,6 +86,7 @@ use futures::stream::StreamExt;
 use futures::stream::TryStreamExt;
 use futures::try_join;
 use futures::Future;
+use futures::TryFutureExt;
 use git_source_of_truth::GitSourceOfTruthConfig;
 use git_symbolic_refs::GitSymbolicRefs;
 use git_types::MappedGitCommitId;
@@ -1738,6 +1740,90 @@ impl<R: MononokeRepo> RepoContext<R> {
  .map_err(MononokeError::from)
  }
 
+ /// This provides the same functionality as
+ /// `mononoke_api::RepoContext::many_changeset_ids_to_locations`. It just translates to
+ /// and from Git types.
+ pub async fn many_git_commit_ids_to_locations(
+ &self,
+ git_master_heads: Vec<GitSha1>,
+ git_ids: Vec<GitSha1>,
+ ) -> Result<HashMap<GitSha1, Result<Location<GitSha1>, MononokeError>>, MononokeError> {
+ let all_git_ids: Vec<_> = git_ids
+ .iter()
+ .cloned()
+ .chain(git_master_heads.clone().into_iter())
+ .collect();
+ let git_to_bonsai: HashMap<GitSha1, ChangesetId> =
+ get_git_bonsai_mapping(self.ctx().clone(), self, all_git_ids)
+ .await?
+ .into_iter()
+ .collect();
+ let master_heads = git_master_heads
+ .iter()
+ .map(|master_id| {
+ git_to_bonsai.get(master_id).cloned().ok_or_else(|| {
+ MononokeError::InvalidRequest(format!(
+ "failed to find bonsai equivalent for client head {}",
+ master_id
+ ))
+ })
+ })
+ .collect::<Result<Vec<_>, MononokeError>>()?;
+
+ // We should treat git_ids as being absolutely any hash. It is perfectly valid for the
+ // server to have not encountered the hash that it was given to convert. Filter out the
+ // hashes that we could not convert to bonsai.
+ let cs_ids = git_ids
+ .iter()
+ .filter_map(|hg_id| git_to_bonsai.get(hg_id).cloned())
+ .collect::<Vec<ChangesetId>>();
+
+ let cs_to_blocations = self
+ .many_changeset_ids_to_locations(master_heads, cs_ids)
+ .await?;
+
+ let bonsai_to_git: HashMap<ChangesetId, GitSha1> = get_git_bonsai_mapping(
+ self.ctx().clone(),
+ self,
+ cs_to_blocations
+ .iter()
+ .filter_map(|(_, result)| match result {
+ Ok(l) => Some(l.descendant),
+ _ => None,
+ })
+ .collect::<Vec<_>>(),
+ )
+ .await?
+ .into_iter()
+ .map(|(git_id, cs_id)| (cs_id, git_id))
+ .collect();
+ let response = git_ids
+ .into_iter()
+ .filter_map(|git_id| git_to_bonsai.get(&git_id).map(|cs_id| (git_id, cs_id)))
+ .filter_map(|(git_id, cs_id)| {
+ cs_to_blocations
+ .get(cs_id)
+ .map(|cs_result| (git_id, cs_result.clone()))
+ })
+ .map(|(git_id, cs_result)| {
+ let cs_result = match cs_result {
+ Ok(cs_location) => cs_location.try_map_descendant(|descendant| {
+ bonsai_to_git.get(&descendant).cloned().ok_or_else(|| {
+ MononokeError::InvalidRequest(format!(
+ "failed to find git equivalent for bonsai {}",
+ descendant
+ ))
+ })
+ }),
+ Err(e) => Err(e),
+ };
+ (git_id, cs_result)
+ })
+ .collect::<HashMap<GitSha1, Result<Location<GitSha1>, MononokeError>>>();
+
+ Ok(response)
+ }
+
  pub async fn derive_bulk(
  &self,
  ctx: &CoreContext,
@@ -1807,6 +1893,83 @@ pub async fn derive_git_changeset(
  }
 }
 
+// TODO(mbthomas): This is temporary to allow us to derive git changesets
+// Returns only the mapping for valid changesets that are known to the server.
+// For Bonsai -> Git conversion, missing Git changesets will be derived (so all Bonsais will be
+// in the output).
+// For Git -> Bonsai conversion, missing Bonsais will not be returned, since they cannot be
+// derived from Git Changesets.
+async fn get_git_bonsai_mapping<'a, R>(
+ ctx: CoreContext,
+ repo_ctx: &RepoContext<R>,
+ bonsai_or_git_shas: impl Into<BonsaisOrGitShas> + 'a + Send,
+) -> Result<Vec<(GitSha1, ChangesetId)>, Error>
+where
+ //R: CommitGraphRef + RepoDerivedDataRef + BonsaiHgMappingRef,
+ R: MononokeRepo,
+{
+ // STATS::get_git_bonsai_mapping.add_value(1);
+
+ let bonsai_or_git_shas = bonsai_or_git_shas.into();
+ let git_bonsai_list = repo_ctx
+ .repo()
+ .bonsai_git_mapping()
+ .get(&ctx, bonsai_or_git_shas.clone())
+ .await?
+ .into_iter()
+ .map(|entry| (entry.git_sha1, entry.bcs_id))
+ .collect::<Vec<_>>();
+
+ use BonsaisOrGitShas::*;
+ match bonsai_or_git_shas {
+ Bonsai(bonsais) => {
+ // If a bonsai commit doesn't exist in the bonsai_git_mapping,
+ // that might mean two things: 1) Bonsai commit just doesn't exist
+ // 2) Bonsai commit exists but git changesets weren't generated for it
+ // Normally the callers of get_git_bonsai_mapping would expect that git
+ // changesets will be lazily generated, so the
+ // code below explicitly checks if a commit exists and if yes then
+ // generates git changeset for it.
+ let mapping: HashMap<_, _> = git_bonsai_list
+ .iter()
+ .map(|(git_id, bcs_id)| (bcs_id, git_id))
+ .collect();
+
+ let mut notfound = vec![];
+ for b in bonsais {
+ if !mapping.contains_key(&b) {
+ notfound.push(b);
+ }
+ }
+
+ let existing: HashSet<_> = repo_ctx
+ .commit_graph()
+ .known_changesets(&ctx, notfound.clone())
+ .await?
+ .into_iter()
+ .collect();
+
+ let mut newmapping: Vec<_> = stream::iter(
+ notfound
+ .into_iter()
+ .filter(|csid| existing.contains(csid))
+ .map(Ok),
+ )
+ .map_ok(|csid| {
+ derive_git_changeset(&ctx, repo_ctx.repo().repo_derived_data(), csid)
+ .map_ok(move |gitsha1| (gitsha1, csid))
+ })
+ .try_buffer_unordered(100)
+ .try_collect()
+ .await?;
+
+ newmapping.extend(git_bonsai_list);
+ Ok(newmapping)
+ }
+ GitSha1(_) => Ok(git_bonsai_list),
+ }
+}
+
 #[cfg(test)]
 mod tests {
  use std::str::FromStr;

eden/mononoke/mononoke_api_hg/src/repo.rs

@@ -538,7 +538,7 @@ impl<R: MononokeRepo> HgRepoContext<R> {
  /// This provides the same functionality as
  /// `mononoke_api::RepoContext::many_changeset_ids_to_locations`. It just translates to
  /// and from Mercurial types.
- pub async fn many_changeset_ids_to_locations(
+ pub async fn many_hg_changeset_ids_to_locations(
  &self,
  hg_master_heads: Vec<HgChangesetId>,
  hg_ids: Vec<HgChangesetId>,