Add coverage testing

Author: lbdreyer

.github/workflows/ci-tests.yml

@@ -36,8 +36,12 @@ jobs:
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.10"]
-        session: ["tests", "doctest", "gallery", "linkcheck"]
+        session: ["doctest", "gallery", "linkcheck"]
         include:
+          - os: "ubuntu-latest"
+            python-version: "3.10"
+            session: "tests"
+            coverage: True
           - os: "ubuntu-latest"
             python-version: "3.9"
             session: "tests"
@@ -132,5 +136,10 @@ jobs:
       - name: "iris ${{ matrix.session }}"
         env:
           PY_VER: ${{ matrix.python-version }}
+          COVERAGE: ${{ matrix.coverage }}
         run: |
           nox --session ${{ matrix.session }} -- --verbose
+
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v3
+        if: ${{ matrix.coverage }}

docs/src/whatsnew/latest.rst

@@ -108,6 +108,12 @@ This document explains the changes made to Iris for this release
 #. `@rcomer`_ removed some old infrastructure that printed test timings.
    (:pull:`5101`)
 
+#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added coverage testing. This can
+   be enabled by setting the environment variable ``COVERAGE=True`` when running
+   the tests with nox, i.e. ``nox --session tests``. (:pull:`4765`)
+
+#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) removed the ``--coding-tests``
+   option from Iris' test runner. (:pull:`4765`)
 
 .. comment
     Whatsnew author names (@github name) in alphabetical order. Note that,

lib/iris/tests/runner/_runner.py

@@ -35,27 +35,22 @@ class TestRunner:
         ("system-tests", "s", "Run the limited subset of system tests."),
         ("gallery-tests", "e", "Run the gallery code tests."),
         ("default-tests", "d", "Run the default tests."),
-        (
-            "coding-tests",
-            "c",
-            "Run the coding standards tests. (These are a "
-            "subset of the default tests.)",
-        ),
         (
             "num-processors=",
             "p",
             "The number of processors used for running " "the tests.",
         ),
         ("create-missing", "m", "Create missing test result files."),
+        ("coverage", "c", "Enable coverage testing"),
     ]
     boolean_options = [
         "no-data",
         "system-tests",
         "stop",
         "gallery-tests",
         "default-tests",
-        "coding-tests",
         "create-missing",
+        "coverage",
     ]
 
     def initialize_options(self):
@@ -64,9 +59,9 @@ def initialize_options(self):
         self.system_tests = False
         self.gallery_tests = False
         self.default_tests = False
-        self.coding_tests = False
         self.num_processors = None
         self.create_missing = False
+        self.coverage = False
 
     def finalize_options(self):
         # These environment variables will be propagated to all the
@@ -84,8 +79,6 @@ def finalize_options(self):
             tests.append("system")
         if self.default_tests:
             tests.append("default")
-        if self.coding_tests:
-            tests.append("coding")
         if self.gallery_tests:
             tests.append("gallery")
         if not tests:
@@ -109,8 +102,6 @@ def run(self):
             tests.append("lib/iris/tests/system_test.py")
         if self.default_tests:
             tests.append("lib/iris/tests")
-        if self.coding_tests:
-            tests.append("lib/iris/tests/test_coding_standards.py")
         if self.gallery_tests:
             import iris.config
 
@@ -136,6 +127,9 @@ def run(self):
         if self.stop:
             args.append("-x")
 
+        if self.coverage:
+            args.extend(["--cov=lib/iris", "--cov-report=xml"])
+
         result = True
         for test in tests:
             args[0] = test

noxfile.py

@@ -28,6 +28,7 @@
 
 #: Cirrus-CI environment variable hook.
 PY_VER = os.environ.get("PY_VER", _PY_VERSIONS_ALL)
+COVERAGE = os.environ.get("COVERAGE", False)
 
 #: Default cartopy cache directory.
 CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy")
@@ -176,6 +177,9 @@ def tests(session: nox.sessions.Session):
     """
     Perform iris system, integration and unit tests.
 
+    Coverage testing is enabled if the COVERAGE environment variable is set to
+    True.
+
     Parameters
     ----------
     session: object
@@ -185,13 +189,15 @@ def tests(session: nox.sessions.Session):
     prepare_venv(session)
     session.install("--no-deps", "--editable", ".")
     session.env.update(ENV)
-    session.run(
+    run_args = [
         "python",
         "-m",
         "iris.tests.runner",
         "--default-tests",
-        "--system-tests",
-    )
+    ]
+    if COVERAGE:
+        run_args.append("--coverage")
+    session.run(*run_args)
 
 
 @nox.session(python=_PY_VERSION_DOCSBUILD, venv_backend="conda")

pyproject.toml

@@ -46,3 +46,20 @@ verbose = "False"
 [tool.pytest.ini_options]
 addopts = "-ra"
 testpaths = "lib/iris"
+
+[tool.coverage.run]
+branch = true
+source = [
+    "lib/iris",
+]
+omit = [
+    "lib/iris/tests/*",
+    "lib/iris/etc/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "if __name__ == .__main__.:"
+]

requirements/ci/nox.lock/py310-linux-64.lock

@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 234b47d943728b5abe70fba0fd74c6adc10e4f1e2a14b919344f8a693b5b3e6f
+# input_hash: f8af5f4aafcb766f463a1a897d3dab9e04f05f1494bced5931d78175ca0c66df
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda#ff9f73d45c4a07d6f424495288a26080
@@ -150,7 +150,7 @@ https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.4-py310hbf28c38_1
 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda#c2566c2ea5f153ddd6bf4acaf7547d97
 https://conda.anaconda.org/conda-forge/linux-64/libclang13-15.0.7-default_h3e3d535_1.conda#a3a0f7a6f0885f5e1e0ec691566afb77
 https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h36d4200_3.conda#c9f4416a34bc91e0eb029f912c68f81f
-https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.88.0-hdc1c0ab_0.conda#c44acb3847ff118c068b662aff858afd
+https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.88.1-hdc1c0ab_0.conda#81eaeb3b35163c8e90e57532bc93754d
 https://conda.anaconda.org/conda-forge/linux-64/libpq-15.2-hb675445_0.conda#4654b17eccaba55b8581d6b9c77f53cc
 https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-252-h2a991cd_0.tar.bz2#3c5ae9f61f663b3d5e1bf7f7da0c85f5
 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.4-h1daa5a0_1.conda#77003f63d1763c1e6569a02c1742c9f4
@@ -191,14 +191,15 @@ https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2#
 https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-h166bdaf_0.tar.bz2#c9b568bd804cb2903c6be6f5f68182e4
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2#536cc5db4d0a3ba0630541aec064b5e4
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2#f59c1242cc1dd93e72c2ee2b360979eb
-https://conda.anaconda.org/conda-forge/noarch/zipp-3.13.0-pyhd8ed1ab_0.conda#41b09d997939e83b231c4557a90c3b13
+https://conda.anaconda.org/conda-forge/noarch/zipp-3.14.0-pyhd8ed1ab_0.conda#01ea04980fa39d7b6dbdd6c67016d177
 https://conda.anaconda.org/conda-forge/noarch/babel-2.11.0-pyhd8ed1ab_0.tar.bz2#2ea70fde8d581ba9425a761609eed6ba
 https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.11.2-pyha770c72_0.conda#88b59f6989f0ed5ab3433af0b82555e1
 https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2#d1a88f3ed5b52e1024b80d4bcd26a7a0
 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py310h255011f_3.conda#800596144bb613cd7ac58b80900ce835
 https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py310hde88566_1.tar.bz2#94ce7a76b0c912279f6958e0b6b21d2b
 https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.0.7-py310hdf3cbec_0.conda#7bf9d8c765b6b04882c719509652c6d6
-https://conda.anaconda.org/conda-forge/linux-64/curl-7.88.0-hdc1c0ab_0.conda#5d9ac94ee84305ada32c3d287d0ec602
+https://conda.anaconda.org/conda-forge/linux-64/coverage-7.1.0-py310h1fa729e_0.conda#da7c45dbe780f5e162011a3af44e5009
+https://conda.anaconda.org/conda-forge/linux-64/curl-7.88.1-hdc1c0ab_0.conda#1968e4fef727858ac04746560e820928
 https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.38.0-py310h5764c6d_1.tar.bz2#12ebe92a8a578bc903bd844744f4d040
 https://conda.anaconda.org/conda-forge/linux-64/glib-2.74.1-h6239696_1.tar.bz2#f3220a9e9d3abcbfca43419a219df7e4
 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-mpi_mpich_h5d83325_1.conda#811c4d55cf17b42336ffa314239717b0
@@ -234,6 +235,7 @@ https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.3-py310h9b08913_0.con
 https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.0.0-pyhd8ed1ab_0.conda#c34694044915d7f291ef257029f2e2af
 https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py310h15e2413_1.conda#5be35366687def87437d210fd673100c
 https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.11.0-py310heca2aa9_3.conda#3b1946b676534472ce65181dda0b9554
+https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.0.0-pyhd8ed1ab_0.tar.bz2#c9e3f8bfdb9bfc34aa1836a6ed4b25d7
 https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.2.0-pyhd8ed1ab_0.conda#70ab87b96126f35d1e68de2ad9fb6423
 https://conda.anaconda.org/conda-forge/noarch/setuptools-scm-7.1.0-pyhd8ed1ab_0.conda#6613dbb3b25cc648a107f33ca9f80fc1
 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-napoleon-0.7-py_0.tar.bz2#0bc25ff6f2e34af63ded59692df5f749
@@ -242,8 +244,8 @@ https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.22.0-h4243ec0
 https://conda.anaconda.org/conda-forge/noarch/identify-2.5.18-pyhd8ed1ab_0.conda#e07a5691c27e65d8d3d9278c578c7771
 https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369
 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.0-mpi_mpich_h1e13492_2.conda#d4ed7704f0fa589e4d7656780fa87557
-https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.0-nompi_py310h0a86a1f_103.conda#7f69695b684f2595d9ba1ce26d693b7d
-https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.12-hd33c08f_1.conda#667dc93c913f0156e1237032e3a22046
+https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.2-nompi_py310h55e1e36_100.tar.bz2#4dd7aa28fb7d9a6de061c9579a30e7dd
+https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.13-hd33c08f_0.conda#e3b13445b8ee9d6a3d53a714f89ccd76
 https://conda.anaconda.org/conda-forge/linux-64/parallelio-2.5.10-mpi_mpich_h862c5c2_100.conda#56e43c5226670aa0943fae9a2628a934
 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.0.0-pyhd8ed1ab_0.conda#d41957700e83bbb925928764cb7f8878
 https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.19.0-pyhd8ed1ab_0.conda#afaa9bf6992f67a82d75fad47a93ec84

requirements/ci/py310.yml

@@ -39,6 +39,7 @@ dependencies:
   - pre-commit
   - psutil
   - pytest
+  - pytest-cov
   - pytest-xdist
   - requests