[MRG] Add a Mercurial contentprovider

.github/workflows/test.yml

@@ -106,6 +106,7 @@ jobs:
           python setup.py bdist_wheel
           pip install dist/*.whl
           pip freeze
+          pip install mercurial hg-evolve
 
       - name: "Run tests"
         run: |

Dockerfile

@@ -1,7 +1,7 @@
-ARG ALPINE_VERSION=3.9.4
+ARG ALPINE_VERSION=3.12.0
 FROM alpine:${ALPINE_VERSION}
 
-RUN apk add --no-cache git python3 python3-dev
+RUN apk add --no-cache git python3 python3-dev py-pip
 
 # build wheels in first image
 ADD . /tmp/src
@@ -15,8 +15,11 @@ RUN mkdir /tmp/wheelhouse \
 
 FROM alpine:${ALPINE_VERSION}
 
-# install python, git, bash
-RUN apk add --no-cache git git-lfs python3 bash docker
+# install python, git, bash, mercurial
+RUN apk add --no-cache git git-lfs python3 py-pip bash docker mercurial
+
+# install hg-evolve (Mercurial extensions)
+RUN pip3 install hg-evolve --user --no-cache-dir
 
 # install repo2docker
 COPY --from=0 /tmp/wheelhouse /tmp/wheelhouse

docs/source/contributing/tasks.md

@@ -23,11 +23,15 @@ If you want to run a specific test, you can do so with:
 py.test -s tests/<path-to-test>
 ```
 
+To skip the tests related to Mercurial repositories (to avoid to install
+Mercurial or hg-evolve), one can use the environment variables
+``REPO2DOCKER_SKIP_HG_TESTS`` or ``REPO2DOCKER_SKIP_HG_EVOLVE_TESTS``.
+
 ### Troubleshooting Tests
 
 Some of the tests have non-python requirements for your development machine. They are:
 
-- `git-lfs` must be installed ([instructions](https://github.com/git-lfs/git-lfs)). It need not be activated -- there is no need to run the `git lfs install` command. It just needs to be available to the test suite. 
+- `git-lfs` must be installed ([instructions](https://github.com/git-lfs/git-lfs)). It need not be activated -- there is no need to run the `git lfs install` command. It just needs to be available to the test suite.
    - If your test failure messages include "`git-lfs filter-process: git-lfs: command not found`", this step should address the problem.
 
 - Minimum Docker Image size of 128GB is required. If you are not running docker on a linux OS, you may need to expand the runtime image size for your installation. See Docker's instructions for [macOS](https://docs.docker.com/docker-for-mac/space/) or [Windows 10](https://docs.docker.com/docker-for-windows/#resources) for more information.
@@ -218,7 +222,7 @@ files accordingly.
 ## Compare generated Dockerfiles between repo2docker versions
 
 For larger refactorings it can be useful to check that the generated Dockerfiles match
-between an older version of r2d and the current version. The following shell script 
+between an older version of r2d and the current version. The following shell script
 automates this test.
 
 ```bash
@@ -231,7 +235,7 @@ basename="dockerfilediff"
 diff_r2d_dockerfiles_with_version () {
     docker run --rm -t -v "$(pwd)":"$(pwd)" --user 1000 jupyterhub/repo2docker:"$1" jupyter-repo2docker --no-build --debug "$(pwd)" &> "$basename"."$1"
     jupyter-repo2docker --no-build --debug "$(pwd)" &> "$basename"."$current_version"
-    
+
     # remove first line logging the path
     sed -i '/^\[Repo2Docker\]/d' "$basename"."$1"
     sed -i '/^\[Repo2Docker\]/d' "$basename"."$current_version"

docs/source/install.rst

@@ -22,6 +22,17 @@ The `BinderHub <https://binderhub.readthedocs.io/>`_ helm chart uses version
 `helm chart <https://github.com/jupyterhub/binderhub/blob/master/helm-chart/binderhub/values.yaml#L167>`_
 for more details.
 
+Optional: Mercurial
+-------------------
+
+For `Mercurial <https://www.mercurial-scm.org>`_ repositories, `Mercurial needs
+to be installed <https://www.mercurial-scm.org/download>`_. For support of
+`Mercurial topics
+<https://www.mercurial-scm.org/doc/evolution/tutorials/topic-tutorial.html>`_,
+also install `hg-evolve <https://www.mercurial-scm.org/doc/evolution/>`_ which
+provides the topic extension (however, no need to explicitly enable it in a
+Mercurial configuration file).
+
 Installing with ``pip``
 -----------------------
 

repo2docker/app.py

@@ -148,6 +148,7 @@ def _default_log_level(self):
             contentproviders.Figshare,
             contentproviders.Dataverse,
             contentproviders.Hydroshare,
+            contentproviders.Mercurial,
             contentproviders.Git,
         ],
         config=True,

repo2docker/contentproviders/__init__.py

@@ -4,3 +4,4 @@
 from .figshare import Figshare
 from .dataverse import Dataverse
 from .hydroshare import Hydroshare
+from .mercurial import Mercurial

repo2docker/contentproviders/mercurial.py

@@ -0,0 +1,76 @@
+import subprocess
+
+from .base import ContentProvider, ContentProviderException
+from ..utils import execute_cmd
+
+args_enabling_topic = ["--config", "extensions.topic="]
+
+
+class Mercurial(ContentProvider):
+    """Provide contents of a remote Mercurial repository."""
+
+    def detect(self, source, ref=None, extra_args=None):
+        if "github.com/" in source or source.endswith(".git"):
+            return None
+        try:
+            subprocess.check_output(
+                ["hg", "identify", source, "--config", "extensions.hggit=!"]
+                + args_enabling_topic,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError:
+            return None
+
+        return {"repo": source, "ref": ref}
+
+    def fetch(self, spec, output_dir, yield_output=False):
+        repo = spec["repo"]
+        ref = spec.get("ref", None)
+
+        # make a clone of the remote repository
+        try:
+            cmd = [
+                "hg",
+                "clone",
+                repo,
+                output_dir,
+                "--config",
+                "phases.publish=False",
+            ] + args_enabling_topic
+            if ref is not None:
+                # don't update so the clone will include an empty working
+                # directory, the given ref will be updated out later
+                cmd.extend(["--noupdate"])
+            for line in execute_cmd(cmd, capture=yield_output):
+                yield line
+
+        except subprocess.CalledProcessError as error:
+            msg = f"Failed to clone repository from {repo}"
+            if ref is not None:
+                msg += f" (ref {ref})"
+            msg += "."
+            raise ContentProviderException(msg) from error
+
+        # check out the specific ref given by the user
+        if ref is not None:
+            try:
+                for line in execute_cmd(
+                    ["hg", "update", "--clean", ref] + args_enabling_topic,
+                    cwd=output_dir,
+                    capture=yield_output,
+                ):
+                    yield line
+            except subprocess.CalledProcessError:
+                self.log.error(
+                    "Failed to update to ref %s", ref, extra=dict(phase="failed")
+                )
+                raise ValueError("Failed to update to ref {}".format(ref))
+
+        cmd = ["hg", "identify", "-i"] + args_enabling_topic
+        sha1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=output_dir)
+        self._node_id = sha1.stdout.read().decode().strip()
+
+    @property
+    def content_id(self):
+        """A unique ID to represent the version of the content."""
+        return self._node_id

tests/unit/contentproviders/test_mercurial.py

@@ -0,0 +1,162 @@
+from pathlib import Path
+import subprocess
+from tempfile import TemporaryDirectory
+import os
+from distutils.util import strtobool
+
+import pytest
+
+from repo2docker.contentproviders import Mercurial
+from repo2docker.contentproviders.mercurial import args_enabling_topic
+
+SKIP_HG = strtobool(os.environ.get("REPO2DOCKER_SKIP_HG_TESTS", "False"))
+SKIP_HG_EVOLVE = SKIP_HG or strtobool(
+    os.environ.get("REPO2DOCKER_SKIP_HG_EVOLVE_TESTS", "False")
+)
+
+skip_if_no_hg_tests = pytest.mark.skipif(
+    SKIP_HG,
+    reason="REPO2DOCKER_SKIP_HG_TESTS",
+)
+skip_if_no_evolve_tests = pytest.mark.skipif(
+    SKIP_HG_EVOLVE,
+    reason="REPO2DOCKER_SKIP_HG_EVOLVE_TESTS",
+)
+
+if SKIP_HG_EVOLVE:
+    args_enabling_topic = []
+
+
+@skip_if_no_hg_tests
+def test_if_mercurial_is_available():
+    """
+    To skip the tests related to Mercurial repositories (to avoid to install
+    Mercurial), one can use the environment variable
+    REPO2DOCKER_SKIP_HG_TESTS.
+    """
+    subprocess.check_output(["hg", "version"])
+
+
+@skip_if_no_evolve_tests
+def test_if_topic_is_available():
+    """Check that the topic extension can be enabled"""
+    output = subprocess.getoutput("hg version -v --config extensions.topic=")
+    assert "failed to import extension topic" not in output
+
+
+def _add_content_to_hg(repo_dir):
+    """Add content to file 'test' in hg repository and commit."""
+    # use append mode so this can be called multiple times
+    with open(Path(repo_dir) / "test", "a") as f:
+        f.write("Hello")
+
+    subprocess.check_call(["hg", "add", "test"], cwd=repo_dir)
+    subprocess.check_call(["hg", "commit", "-m", "Test commit"], cwd=repo_dir)
+
+    if not SKIP_HG_EVOLVE:
+
+        def check_call(command):
+            subprocess.check_call(command + args_enabling_topic, cwd=repo_dir)
+
+        check_call(["hg", "topic", "test-topic"])
+        check_call(["hg", "commit", "-m", "Test commit in topic test-topic"])
+        check_call(["hg", "up", "default"])
+
+
+def _get_node_id(repo_dir):
+    """Get repository's current commit node ID (currently SHA1)."""
+    node_id = subprocess.Popen(
+        ["hg", "identify", "-i"] + args_enabling_topic,
+        stdout=subprocess.PIPE,
+        cwd=repo_dir,
+    )
+    return node_id.stdout.read().decode().strip()
+
+
+@pytest.fixture()
+def hg_repo():
+    """
+    Make a dummy hg repo in which user can perform hg operations
+
+    Should be used as a contextmanager, it will delete directory when done
+    """
+    with TemporaryDirectory() as hgdir:
+        subprocess.check_call(["hg", "init"], cwd=hgdir)
+        yield hgdir
+
+
+@pytest.fixture()
+def hg_repo_with_content(hg_repo):
+    """Create a hg repository with content"""
+    _add_content_to_hg(hg_repo)
+    node_id = _get_node_id(hg_repo)
+
+    yield hg_repo, node_id
+
+
+@skip_if_no_hg_tests
+def test_detect_mercurial(hg_repo_with_content, repo_with_content):
+    mercurial = Mercurial()
+    assert mercurial.detect("this-is-not-a-directory") is None
+    assert mercurial.detect("https://github.com/jupyterhub/repo2docker") is None
+
+    git_repo = repo_with_content[0]
+    assert mercurial.detect(git_repo) is None
+
+    hg_repo = hg_repo_with_content[0]
+    assert mercurial.detect(hg_repo) == {"repo": hg_repo, "ref": None}
+
+
+@skip_if_no_hg_tests
+def test_clone(hg_repo_with_content):
+    """Test simple hg clone to a target dir"""
+    upstream, node_id = hg_repo_with_content
+
+    with TemporaryDirectory() as clone_dir:
+        spec = {"repo": upstream}
+        mercurial = Mercurial()
+        for _ in mercurial.fetch(spec, clone_dir):
+            pass
+        assert (Path(clone_dir) / "test").exists()
+
+        assert mercurial.content_id == node_id
+
+
+@skip_if_no_hg_tests
+def test_bad_ref(hg_repo_with_content):
+    """
+    Test trying to update to a ref that doesn't exist
+    """
+    upstream, node_id = hg_repo_with_content
+    with TemporaryDirectory() as clone_dir:
+        spec = {"repo": upstream, "ref": "does-not-exist"}
+        with pytest.raises(ValueError):
+            for _ in Mercurial().fetch(spec, clone_dir):
+                pass
+
+
+@skip_if_no_evolve_tests
+def test_ref_topic(hg_repo_with_content):
+    """
+    Test trying to update to a topic
+
+    To skip this test (to avoid to install hg-evolve), one can use the
+    environment variable REPO2DOCKER_SKIP_HG_EVOLVE_TESTS.
+
+    """
+    upstream, node_id = hg_repo_with_content
+    node_id = subprocess.Popen(
+        ["hg", "identify", "-i", "-r", "topic(test-topic)"] + args_enabling_topic,
+        stdout=subprocess.PIPE,
+        cwd=upstream,
+    )
+    node_id = node_id.stdout.read().decode().strip()
+
+    with TemporaryDirectory() as clone_dir:
+        spec = {"repo": upstream, "ref": "test-topic"}
+        mercurial = Mercurial()
+        for _ in mercurial.fetch(spec, clone_dir):
+            pass
+        assert (Path(clone_dir) / "test").exists()
+
+        assert mercurial.content_id == node_id