Separate benchmarks and coverage from tests (#108)

* Remove benchmarks from regular tests

* Change benchmarks targets to benchmark

* Add CI for coverage and benchmarks

Author: btjanaka

.github/workflows/testing.yml

@@ -31,16 +31,44 @@ jobs:
           pip install tox tox-gh-actions
       - name: Test with tox
         run: tox
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Test with tox
+        run: tox -e coverage
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Test with tox
+        run: tox -e benchmarks
   deploy:
     runs-on: ubuntu-latest
     needs: test
     if: startsWith(github.ref, 'refs/tags')
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.7
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

Makefile

@@ -1,5 +1,4 @@
 # Convenient commands. Run `make help` for command info.
-.PHONY: clean clean-test clean-pyc clean-build docs help
 .DEFAULT_GOAL := help
 
 define BROWSER_PYSCRIPT
@@ -18,57 +17,75 @@ help:
 	@grep -E '^[.a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[34;1m%-30s\033[0m %s\n", $$1, $$2}'
 
 clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+.PHONY: clean
 
 clean-build: ## remove build artifacts
 	rm -fr build/
 	rm -fr dist/
 	rm -fr .eggs/
 	find . -name '*.egg-info' -exec rm -fr {} +
 	find . -name '*.egg' -exec rm -f {} +
+.PHONY: clean-build
 
 clean-pyc: ## remove Python file artifacts
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
 	find . -name '__pycache__' -exec rm -fr {} +
+.PHONY: clean-pyc
 
 clean-test: ## remove test and coverage artifacts
 	rm -fr .tox/
 	rm -f .coverage
 	rm -fr htmlcov/
 	rm -fr .pytest_cache
+.PHONY: clean-test
 
 lint: ## check style with pylint
 	pylint ribs tests examples benchmarks
+.PHONY: lint
 
 test: ## run tests with the default Python
 	pytest tests
+.PHONY: test
+
 test-core: ## only test the core of ribs
 	pytest tests/core
+.PHONY: test-core
+
 test-extras: ## only test the extras of ribs
 	pytest tests/extras
-test-failed: ## run only tests that filed
-	pytest --last-failed
-test-only: ## run tests without benchmarks (which take a while)
-	pytest -c pytest_no_benchmark.ini tests
+.PHONY: test-extras
+
 test-coverage: ## get better test coverage by running without numba on
-	NUMBA_DISABLE_JIT=1 pytest -c pytest_no_benchmark.ini tests
+	NUMBA_DISABLE_JIT=1 pytest tests
+.PHONY: test-coverage
+
 test-all: ## run tests on every Python version with tox
 	tox
+.PHONY: test-all
 
-NUM_CPUS=4
-xtest: ## run tests distributed with 4 workers
-	pytest -n $(NUM_CPUS) tests
-xtest-only: ## run tests without benchmarks distributed over 4 workers
-	pytest -n $(NUM_CPUS) -c pytest_no_benchmark.ini tests
+benchmark: ## run benchmarks (may take a while)
+	pytest -c pytest_benchmark.ini tests
+.PHONY: benchmark
 
-examples-test: ## test examples are working
+xtest: ## run tests with n workers (e.g. make xtest n=4)
+	pytest -n $(n) tests
+.PHONY: xtest
+
+xbenchmark: ## run benchmarks with n workers (e.g. make xbenchmarks n=4)
+	pytest -n $(n) -c pytest_no_benchmark.ini tests
+.PHONY: xbenchmark
+
+test-examples: ## test examples are working
 	bash tests/examples.sh
+.PHONY: test-examples
 
 docs: ## generate Sphinx HTML documentation, including API docs
 	$(MAKE) -C docs clean
 	$(MAKE) -C docs html
 	$(BROWSER) docs/_build/html/index.html
+.PHONY: docs
 
 servedocs: ## compile the docs watching for changes
 	DOCS_MODE=dev sphinx-autobuild \
@@ -77,17 +94,18 @@ servedocs: ## compile the docs watching for changes
 		--watch examples/ \
 		docs/ \
 		docs/_build/html
+.PHONY: servedocs
 
 release-test: dist ## package and upload a release to TestPyPI
 	twine upload --repository testpypi dist/*
+.PHONY: release-test
+
 release: dist ## package and upload a release
 	twine upload dist/*
+.PHONY: release
 
 dist: clean ## builds source and wheel package
 	python setup.py sdist
 	python setup.py bdist_wheel
 	ls -l dist
 	check-wheel-contents dist/*.whl
-
-install: clean ## install the package to the active Python's site-packages
-	python setup.py install

pytest_benchmark.ini

@@ -0,0 +1,7 @@
+# Config for pytest benchmarks. Pass this file to pytest with the -c option.
+
+[pytest]
+python_files =  *_benchmark.py
+python_functions = benchmark_*
+addopts = -v --cov-report term-missing --cov=ribs --benchmark-sort=name --benchmark-name=long
+markers = style

pytest_no_benchmark.ini

@@ -22,8 +22,7 @@ indent_width = 4
 [aliases]
 
 [tool:pytest]
-python_files = *_test.py *_benchmark.py
-python_functions = test_* benchmark_*
-addopts = -v --cov-report term-missing --cov=ribs --benchmark-sort=name --benchmark-name=long
+python_files = *_test.py
+python_functions = test_*
+addopts = -v --cov-report term-missing --cov=ribs
 markers = style
-

setup.cfg

@@ -1,10 +1,11 @@
 # Tests for CI.
-# - core: ribs core (e.g. archive, emitters, optimizers)
-# - extras: ribs extras (e.g. visualize)
+# - py*-ribs_core: ribs core (e.g. archive, emitters, optimizers)
+# - py*-ribs_extras: ribs extras (e.g. visualize)
 # - coverage: everything, but with Numba disabled so we can check test coverage
+# - benchmarks: all pytest benchmarks under the tests directory
 
 [tox]
-envlist = {py36,py37,py38,py39}-ribs_{core,extras,coverage}, pylint
+envlist = {py36,py37,py38,py39}-ribs_{core,extras}, coverage, benchmarks
 
 [gh-actions]
 python =
@@ -13,25 +14,19 @@ python =
     3.8: py38
     3.9: py39
 
-[testenv:pylint]
-basepython = python
-deps =
-    .[all]
-    pylint
-    pytest
-commands = pylint ribs tests
-
 [testenv]
 setenv =
     PYTHONPATH = {toxinidir}
-    ribs_coverage: NUMBA_DISABLE_JIT=1
+    coverage: NUMBA_DISABLE_JIT=1
 deps =
     .[dev]
     ribs_core: .
     ribs_extras: .[all]
-    ribs_coverage: .[all]
+    coverage: .[all]
+    benchmarks: .[all]
 commands =
     pip install -U pip
     ribs_core: pytest --basetemp={envtmpdir} tests/core
     ribs_extras: pytest --basetemp={envtmpdir} tests/extras
-    ribs_coverage: pytest --basetemp={envtmpdir} -c pytest_no_benchmark.ini tests
+    coverage: pytest --basetemp={envtmpdir} tests
+    benchmarks: pytest --basetemp={envtmpdir} -c pytest_benchmark.ini tests