Merge pull request #16 from igordertigor/feature/run-old-tests-on-new…

…-version

Feature/run old tests on new version

src/semv/commands.py

@@ -30,7 +30,7 @@ def version_string(config: Config) -> Version:
     )
     h = hooks.Hooks()
     for name in config.checks:
-        h.register(getattr(hooks, name))
+        h.register(getattr(hooks, name)(**config.checks[name]))
 
     current_version = vcs.get_current_version()
     commits_or_none = (

src/semv/config.py

@@ -22,7 +22,7 @@ class Config:
         }
     )
     invalid_commit_action: InvalidCommitAction = InvalidCommitAction.warning
-    checks: Set[str] = field(default_factory=set)
+    checks: Dict[str, Dict[str, Any]] = field(default_factory=dict)
 
     @classmethod
     def parse(cls, text: str):

src/semv/hooks.py

@@ -1,9 +1,21 @@
-from typing import Callable, List
+from typing import List, Union
+from abc import ABC, abstractmethod
 import sys
+import re
+import os
+import shutil
+import glob
+import subprocess
+import tomli
+from tempfile import TemporaryDirectory
 from operator import attrgetter
 from .types import Version, VersionIncrement
 
-VersionEstimator = Callable[[Version], VersionIncrement]
+
+class VersionEstimator(ABC):
+    @abstractmethod
+    def run(self, current_version: Version):
+        raise NotImplementedError
 
 
 class Hooks:
@@ -15,7 +27,7 @@ def __init__(self):
     def estimate_version_increment(
         self, current_version: Version
     ) -> VersionIncrement:
-        check_results = (check(current_version) for check in self.checks)
+        check_results = (check.run(current_version) for check in self.checks)
         return VersionIncrement(
             min(
                 (x for x in check_results),
@@ -28,19 +40,110 @@ def register(self, check: VersionEstimator):
         self.checks.append(check)
 
 
-def dummy_version_estimator_skip(
-    current_version: Version,
-) -> VersionIncrement:
-    sys.stderr.write(
-        f'Dummy version estimator called on version {current_version}\n'
-    )
-    return VersionIncrement.skip
+class DummyVersionEstimator(VersionEstimator):
+    increment: VersionIncrement
+
+    def __init__(self, increment: str):
+        self.increment = VersionIncrement(increment)
+
+    def run(
+        self,
+        current_version: Version,
+    ) -> VersionIncrement:
+        sys.stderr.write(
+            f'Dummy version estimator called on version {current_version},'
+            f' increment {self.increment.value}\n'
+        )
+        return self.increment
+
+
+class RunPreviousVersionsTestsTox(VersionEstimator):
+    testenv: List[str]
+    buildenv: str
+
+    def __init__(
+        self, testenv: Union[str, List[str]], buildenv: str = 'build'
+    ):
+        """Run tests from previous version on new version
+
+        This is an attempt to automatically detect breaking versions. If
+        the tests from the previous version fail, then we can assume
+        that the version most likely contains a breaking change.
 
+        Parameters:
+            testenv:
+                Name of the environment(s) that contain the actual tests to run.
+            buildenv:
+                To build the current version's package, we run this tox
+                environment. It should create a wheel file in the
+                `dist/` folder.
+        """
+        if isinstance(testenv, str):
+            self.testenv = [testenv]
+        else:
+            self.testenv = testenv
+        self.buildenv = buildenv
 
-def dummy_version_estimator_major(
-    current_version: Version,
-) -> VersionIncrement:
-    sys.stderr.write(
-        f'Dummy version estimator called on version {current_version}\n'
-    )
-    return VersionIncrement.major
+    def run(self, current_version: Version) -> VersionIncrement:
+        source_dir = os.path.abspath(os.path.curdir)
+        build_proc = subprocess.run(
+            f'tox -e {self.buildenv}',
+            shell=True,
+            capture_output=True,
+        )
+        build_proc.check_returncode()
+        package = os.path.join(source_dir, max(glob.glob('dist/*.whl')))
+        with TemporaryDirectory() as tempdir:
+            git_proc = subprocess.run(
+                f'git clone --depth 1 --branch {current_version}'
+                f' file://{source_dir}/.git .',
+                shell=True,
+                capture_output=True,
+                cwd=tempdir,
+            )
+            git_proc.check_returncode()
+
+            possible_misleading_imports = glob.glob(
+                os.path.join(tempdir, f'{get_package_name()}.*')
+            )
+            if possible_misleading_imports:
+                src_layout = os.path.join(tempdir, 'src')
+                os.makedirs(src_layout, exist_ok=True)
+                for fake_import in possible_misleading_imports:
+                    shutil.move(fake_import, src_layout)
+
+            envs = ','.join(self.testenv)
+            test_proc = subprocess.run(
+                f'tox --installpkg {package} -e "{envs}" -- -v',
+                shell=True,
+                cwd=tempdir,
+                capture_output=True,
+            )
+            if test_proc.returncode:
+                m = re.search(
+                    r'=+ FAILURES =+(.*?).=+ \d+ failed in',
+                    test_proc.stdout.decode('utf-8'),
+                    re.DOTALL,
+                )
+                if m:
+                    # This will trivially be true, because we already
+                    # know that there was output due to the returncode
+                    closing_line = (
+                        '==========================='
+                        '    end of test report   '
+                        '============================\n'
+                    )
+                    sys.stderr.write(m.group(1).strip() + '\n' + closing_line)
+                return VersionIncrement.major
+        return VersionIncrement.skip
+
+
+def get_package_name() -> str:
+    if os.path.exists('pyproject.toml'):
+        with open('pyproject.toml') as f:
+            cfg = tomli.loads(f.read())
+        return cfg['project']['name']
+    else:
+        raise NotImplementedError(
+            'Only supporting files configured through pyproject.toml at the moment'
+        )

tests/cram/test_hooks.t

@@ -15,17 +15,17 @@ Case 1: No hooks
   [1]
 
 Case 2: Hook returns skip
-  $ echo "[tool.semv]" > pyproject.toml
-  $ echo 'checks = ["dummy_version_estimator_skip"]' >> pyproject.toml
+  $ echo "[tool.semv.checks]" > pyproject.toml
+  $ echo 'DummyVersionEstimator = {increment="skip"}' >> pyproject.toml
   $ semv
-  Dummy version estimator called on version v0.0.0
+  Dummy version estimator called on version v0.0.0, increment skip
   WARNING: No changes for new version
   [1]
 
 Case 3: Hook returns major version ~> Failure
-  $ echo "[tool.semv]" > pyproject.toml
-  $ echo 'checks = ["dummy_version_estimator_major"]' >> pyproject.toml
+  $ echo "[tool.semv.checks]" > pyproject.toml
+  $ echo 'DummyVersionEstimator = {increment="major"}' >> pyproject.toml
   $ semv
-  Dummy version estimator called on version v0.0.0
+  Dummy version estimator called on version v0.0.0, increment major
   ERROR: Commits suggest skip increment, but checks imply major increment
   [3]

tests/cram/test_old_tests_on_new_version.t

@@ -0,0 +1,88 @@
+This test checks one of the main ideas of Stephan Bönnemann's talk: If the previous version's tests fail when running against a new version, then that version is likely a major version.
+
+  $ bash "$TESTDIR"/setup.sh
+  Initialized empty Git repository in /tmp/cramtests-*/test_old_tests_on_new_version.t/.git/ (glob)
+  [master (root-commit) *] docs(readme): Add readme (glob)
+   1 file changed, 1 insertion(+)
+   create mode 100644 README.md
+
+Create some setup:
+Tox
+  $ echo "[tox]" > tox.ini
+  $ echo "isolated_build = true" >> tox.ini
+  $ echo "envlist = unit" >> tox.ini
+  $ echo "" >> tox.ini
+  $ echo "[testenv:unit]" >> tox.ini
+  $ echo "deps = pytest" >> tox.ini
+  $ echo "commands = pytest {posargs} tests.py" >> tox.ini
+  $ echo "" >> tox.ini
+  $ echo "[testenv:build]" >> tox.ini
+  $ echo "deps = build" >> tox.ini
+  $ echo "skip_install = true" >> tox.ini
+  $ echo "skip_dist = true" >> tox.ini
+  $ echo "commands = python -m build" >> tox.ini
+
+pyproject
+  $ echo "[project]" > pyproject.toml
+  $ echo 'name = "mypack"' >> pyproject.toml
+  $ echo 'dynamic = ["version"]' >> pyproject.toml
+  $ echo "" >> pyproject.toml
+  $ echo "[build-system]" >> pyproject.toml
+  $ echo 'requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]' >> pyproject.toml
+  $ echo 'build-backend = "setuptools.build_meta"' >> pyproject.toml
+  $ echo "" >> pyproject.toml
+  $ echo "[tool.setuptools_scm]" >> pyproject.toml
+
+source
+  $ mkdir src
+  $ echo "def f(x, y):" > src/mypack.py
+  $ echo "    return x + y" >> src/mypack.py
+
+tests
+  $ echo "import mypack" > tests.py
+  $ echo "" >> tests.py
+  $ echo "def test_regular_sum():" >> tests.py
+  $ echo "    assert mypack.f(2, 1) == 3" >> tests.py
+
+Commit
+  $ git add src/mypack.py tests.py tox.ini pyproject.toml
+  $ git commit -m "feat(mypack): Initial implementation"
+  [master *] feat(mypack): Initial implementation (glob)
+   4 files changed, 28 insertions(+)
+   create mode 100644 pyproject.toml
+   create mode 100644 src/mypack.py
+   create mode 100644 tests.py
+   create mode 100644 tox.ini
+  $ git tag v1.0.0
+
+Now configure the checks
+  $ echo "" >> pyproject.toml
+  $ echo "[tool.semv.checks]" >> pyproject.toml
+  $ echo 'RunPreviousVersionsTestsTox = {testenv = "unit"}' >> pyproject.toml
+  $ git add pyproject.toml
+  $ git commit -m "chore(semv): Add semv config"
+  [master *] chore(semv): Add semv config (glob)
+   1 file changed, 3 insertions(+)
+
+Introduce a breaking change
+  $ echo "def f(x, y):" > src/mypack.py
+  $ echo "    return x * y" >> src/mypack.py
+  $ git add src/mypack.py
+  $ git commit -m 'feat(mypack): Multiplications are cooler'
+  [master *] feat(mypack): Multiplications are cooler (glob)
+   1 file changed, 1 insertion(+), 1 deletion(-)
+  $ semv
+  _______________________________ test_regular_sum _______________________________
+
+      def test_regular_sum():
+  *       assert mypack.f(2, 1) == 3 (glob)
+  E       assert 2 == 3
+  E        +  where 2 = <function f at *>(2, 1) (glob)
+  E        +    where <function f at *> = mypack.f (glob)
+
+  tests.py:4: AssertionError
+  =========================== short test summary info ============================
+  FAILED tests.py::test_regular_sum - assert 2 == 3
+  ===========================    end of test report   ============================
+  ERROR: Commits suggest minor increment, but checks imply major increment
+  [3]

tests/cram/test_old_tests_on_new_version_importable_module.t

@@ -0,0 +1,87 @@
+This test checks one of the main ideas of Stephan Bönnemann's talk: If the previous version's tests fail when running against a new version, then that version is likely a major version.
+
+  $ bash "$TESTDIR"/setup.sh
+  Initialized empty Git repository in /tmp/cramtests-*/test_old_tests_on_new_version_importable_module.t/.git/ (glob)
+  [master (root-commit) *] docs(readme): Add readme (glob)
+   1 file changed, 1 insertion(+)
+   create mode 100644 README.md
+
+Create some setup:
+Tox
+  $ echo "[tox]" > tox.ini
+  $ echo "isolated_build = true" >> tox.ini
+  $ echo "envlist = unit" >> tox.ini
+  $ echo "" >> tox.ini
+  $ echo "[testenv:unit]" >> tox.ini
+  $ echo "deps = pytest" >> tox.ini
+  $ echo "commands = pytest {posargs} tests.py" >> tox.ini
+  $ echo "" >> tox.ini
+  $ echo "[testenv:build]" >> tox.ini
+  $ echo "deps = build" >> tox.ini
+  $ echo "skip_install = true" >> tox.ini
+  $ echo "skip_dist = true" >> tox.ini
+  $ echo "commands = python -m build" >> tox.ini
+
+pyproject
+  $ echo "[project]" > pyproject.toml
+  $ echo 'name = "mypack"' >> pyproject.toml
+  $ echo 'dynamic = ["version"]' >> pyproject.toml
+  $ echo "" >> pyproject.toml
+  $ echo "[build-system]" >> pyproject.toml
+  $ echo 'requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]' >> pyproject.toml
+  $ echo 'build-backend = "setuptools.build_meta"' >> pyproject.toml
+  $ echo "" >> pyproject.toml
+  $ echo "[tool.setuptools_scm]" >> pyproject.toml
+
+source
+  $ echo "def f(x, y):" > mypack.py
+  $ echo "    return x + y" >> mypack.py
+
+tests
+  $ echo "import mypack" > tests.py
+  $ echo "" >> tests.py
+  $ echo "def test_regular_sum():" >> tests.py
+  $ echo "    assert mypack.f(2, 1) == 3" >> tests.py
+
+Commit
+  $ git add mypack.py tests.py tox.ini pyproject.toml
+  $ git commit -m "feat(mypack): Initial implementation"
+  [master *] feat(mypack): Initial implementation (glob)
+   4 files changed, 28 insertions(+)
+   create mode 100644 mypack.py
+   create mode 100644 pyproject.toml
+   create mode 100644 tests.py
+   create mode 100644 tox.ini
+  $ git tag v1.0.0
+
+Now configure the checks
+  $ echo "" >> pyproject.toml
+  $ echo "[tool.semv.checks]" >> pyproject.toml
+  $ echo 'RunPreviousVersionsTestsTox = {testenv = "unit"}' >> pyproject.toml
+  $ git add pyproject.toml
+  $ git commit -m "chore(semv): Add semv config"
+  [master *] chore(semv): Add semv config (glob)
+   1 file changed, 3 insertions(+)
+
+Introduce a breaking change
+  $ echo "def f(x, y):" > mypack.py
+  $ echo "    return x * y" >> mypack.py
+  $ git add mypack.py
+  $ git commit -m 'feat(mypack): Multiplications are cooler'
+  [master *] feat(mypack): Multiplications are cooler (glob)
+   1 file changed, 1 insertion(+), 1 deletion(-)
+  $ semv
+  _______________________________ test_regular_sum _______________________________
+
+      def test_regular_sum():
+  *       assert mypack.f(2, 1) == 3 (glob)
+  E       assert 2 == 3
+  E        +  where 2 = <function f at *>(2, 1) (glob)
+  E        +    where <function f at *> = mypack.f (glob)
+
+  tests.py:4: AssertionError
+  =========================== short test summary info ============================
+  FAILED tests.py::test_regular_sum - assert 2 == 3
+  ===========================    end of test report   ============================
+  ERROR: Commits suggest minor increment, but checks imply major increment
+  [3]