QA-260 add check for tests without spec (#29)

Author: annademina

src/pytest_allure_spec_coverage/matcher.py

@@ -13,7 +13,7 @@
 """Matcher of tests cases and scenarios"""
 import itertools
 import os
-from contextlib import suppress
+import warnings
 from dataclasses import dataclass, field
 from typing import (
     Callable,
@@ -28,7 +28,6 @@
     Tuple,
     Type,
 )
-
 import pytest
 from _pytest.config import ExitCode
 from _pytest.main import Session
@@ -46,6 +45,10 @@
 from .xdist_shared import XdistSharedStorage
 
 
+class SpecCollectorWarning(UserWarning):
+    """Warn for spec collector issues"""
+
+
 def is_xdist_first_worker():
     """True if running on first xdist worker"""
     return os.getenv("PYTEST_XDIST_WORKER") == "gw0"
@@ -183,6 +186,7 @@ class ScenariosMatcher:
     storage: Optional[XdistSharedStorage]
 
     scenarios: Collection[Scenario] = field(default_factory=list)
+    nonexistent: [PytestItems] = field(default_factory=list)
     matches: Mapping[Scenario, PytestItems] = field(default_factory=dict)
 
     @property
@@ -210,15 +214,17 @@ def spec_coverage_percent(self):
         """Coverage percent"""
         return int((len(self.scenarios) - len(tuple(self.missed))) / len(self.scenarios) * 100)
 
-    def match(self, items: List[pytest.Item], deselected=False) -> None:
+    def match(self, items: List[pytest.Item], deselected: bool = False) -> None:
         """Match collected tests items with its scenarios"""
 
         if not self.matches:
             self.matches = {sc: PytestItems() for sc in self.scenarios}
         sc_lookup = {sc.id: sc for sc in self.scenarios}
         for item in items:
             for key in scenario_ids(item):
-                with suppress(KeyError):
+                if key not in sc_lookup:
+                    self.nonexistent.append(item)
+                else:
                     if deselected:
                         self.matches[sc_lookup[key]].deselected.append(item)
                     else:
@@ -343,12 +349,25 @@ def pytest_collection_finish(self):
         Important that fail will be after terminal reporting hooks
         """
         yield
-        if self.config.fail_under and self.spec_coverage_percent < self.config.fail_under:
-            pytest.exit(
-                f"Spec coverage percent is {self.spec_coverage_percent}%, "
-                f"and it is less than target {self.config.fail_under}%",
-                returncode=ExitCode.NO_TESTS_COLLECTED,
-            )
+        warn_message = ""
+        if self.nonexistent:
+            tests_without_spec = "\n    ".join(item.name for item in self.nonexistent)
+            warn_message = f"The following tests linked with nonexistent spec:\n    {tests_without_spec}"
+        if not self.config.fail_under:
+            if warn_message:
+                warnings.warn(SpecCollectorWarning(warn_message))
+        else:
+            exit_message = ""
+            if self.spec_coverage_percent < self.config.fail_under:
+                exit_message += (
+                    f"Spec coverage percent is {self.spec_coverage_percent}%, "
+                    f"and it is less than target {self.config.fail_under}%\n"
+                )
+            if exit_message or warn_message:
+                pytest.exit(
+                    exit_message + warn_message,
+                    returncode=ExitCode.NO_TESTS_COLLECTED,
+                )
 
     def pytest_terminal_summary(self, terminalreporter: TerminalReporter):
         """

src/pytest_allure_spec_coverage/xdist_shared.py

@@ -5,7 +5,7 @@
 
 
 class XdistSharedStorage:
-    """Xdist shared storage implementation """
+    """Xdist shared storage implementation"""
 
     @staticmethod
     def is_xdist_master(config):

tests/examples/matcher_pytester_test.py

@@ -31,11 +31,6 @@ def test_multiple_scenarios_case():
     """Case with multiple implemented scenarios"""
 
 
-@pytest.mark.scenario("non_existent")
-def test_non_existent_scenario_case():
-    """Case implemented non-existent scenario"""
-
-
 @pytest.mark.scenario("simple_scenario")
 @pytest.mark.scenario("simple_scenario")
 def test_duplicated_scenarios_case():

tests/examples/non_existent_test.py

@@ -0,0 +1,12 @@
+"""Non-existent scenario"""
+import pytest
+
+
+@pytest.mark.scenario("non_existent")
+def test_non_existent_scenario_case():
+    """Case implemented non-existent scenario"""
+
+
+@pytest.mark.scenario("simple_scenario")
+def test_single_scenario_case():
+    """Case with the single one scenario"""

tests/plugin/test_matcher.py

@@ -28,8 +28,8 @@
 from pytest_allure_spec_coverage.common import allure_listener
 from pytest_allure_spec_coverage.matcher import ScenariosMatcher, make_allure_labels
 from pytest_allure_spec_coverage.models.scenario import Scenario
-from ..examples.collector import scenarios
 from .common import run_tests, run_with_allure
+from ..examples.collector import scenarios
 
 
 @dataclass
@@ -49,7 +49,6 @@ def test_cases() -> Tuple[Collection[TestCase], Collection[Scenario], Collection
     return (
         (
             TestCase("test_abandoned_case"),
-            TestCase("test_non_existent_scenario_case"),
             TestCase("test_single_scenario_case", matches=[simple_scenario]),
             TestCase("test_multiple_scenarios_case", matches=[simple_scenario, nested_scenario]),
             TestCase("test_duplicated_scenarios_case", matches=[simple_scenario, simple_scenario]),
@@ -194,7 +193,9 @@ def test_matcher(
 
     with allure.step("Check summary for coverage percent"):
         percent = (len(scenarios) - len(not_implemented)) * 100 // len(scenarios)
-        assert f"{percent}%" in pytester_result.outlines[-1]
+        assert any(
+            f"{percent}%" in outline for outline in pytester_result.outlines
+        ), f'Should be "{percent}%" in outlines'
 
 
 @pytest.mark.usefixtures("_conftest")
@@ -216,7 +217,9 @@ def test_matcher_without_allure(
 
     with allure.step("Check summary for coverage percent"):
         # Last one line will be greetings while previous one with stats
-        assert f"{percent}%" in pytester_result.outlines[-2]
+        assert any(
+            f"{percent}%" in outline for outline in pytester_result.outlines
+        ), f'Should be "{percent}%" in outlines'
 
 
 @pytest.mark.usefixtures("_conftest")
@@ -231,13 +234,43 @@ def test_sc_only(pytester: Pytester):
             outcomes={"passed": 0},
         )
         assert pytester_result.ret == ExitCode.NO_TESTS_COLLECTED
-        assert "_pytest.outcomes.Exit" in pytester_result.outlines[-1]
-        assert "50% specification coverage" in pytester_result.outlines[-2]
+        assert any(
+            "_pytest.outcomes.Exit" in outline for outline in pytester_result.outlines
+        ), 'Should be "_pytest.outcomes.Exit" in outlines'
+        assert any(
+            "50% specification coverage" in outline for outline in pytester_result.outlines
+        ), 'Should be "50% specification coverage" in outlines'
     with allure.step("Assert that --sc-target less than coverage"):
         pytester_result, _ = run_with_allure(
             pytester=pytester,
             testfile_path="sc_only_test.py",
             additional_opts=["--sc-type", "test", "--sc-only", "--sc-target", "25"],
             outcomes={"passed": 0},
         )
-        assert "🎉🎉🎉" in pytester_result.outlines[-1]
+        assert any("🎉🎉🎉" in outline for outline in pytester_result.outlines), 'Should be "🎉🎉🎉" in outlines'
+
+
+@pytest.mark.usefixtures("_conftest")
+def test_non_existent(pytester: Pytester):
+    """Test that nonexistent spec warns or raise error"""
+    with allure.step("Assert that non_existent test has warning"):
+        pytester_result, _ = run_with_allure(
+            pytester=pytester,
+            testfile_path="non_existent_test.py",
+            additional_opts=["--sc-type", "test"],
+            outcomes={"passed": 2},
+        )
+        assert pytester_result.ret == ExitCode.OK
+        assert "      test_non_existent_scenario_case" in pytester_result.outlines
+        assert any("2 passed, 1 warning, 25% specification coverage" in outline for outline in pytester_result.outlines)
+    with allure.step("Assert that non_existent test raise error in sc_only mode"):
+        pytester_result, _ = run_with_allure(
+            pytester=pytester,
+            testfile_path="non_existent_test.py",
+            additional_opts=["--sc-type", "test", "--sc-only", "--sc-target", "25"],
+            outcomes={"passed": 0},
+        )
+        assert pytester_result.ret == ExitCode.NO_TESTS_COLLECTED
+        assert any(
+            "The following tests linked with nonexistent spec" in outline for outline in pytester_result.outlines
+        )