now using the unified building process offered by pycommons

Makefile

@@ -51,170 +51,25 @@ init: clean
 	pip freeze &&\
 	echo "$(NOW): Finished printing all installed packages."
 
-
-# Run the unit tests.
 test: init
-	echo "$(NOW): Erasing old coverage data." &&\
-	coverage erase &&\
-	echo "$(NOW): The original value of PATH is '${PATH}'." &&\
-	export PATH="${PATH}:${PYTHON_PACKAGE_BINARIES}" &&\
-	echo "$(NOW): PATH is now '${PATH}'." &&\
-	echo "$(NOW): Running pytest with doctests." && \
-	timeout --kill-after=15s 90m coverage run -a --include="moptipyapps/*" -m pytest --strict-config --doctest-modules --ignore=tests --ignore=examples && \
-	echo "$(NOW): Running pytest tests." && \
-	timeout --kill-after=15s 90m coverage run -a --include="moptipyapps/*" -m pytest --strict-config tests --ignore=examples && \
-	echo "$(NOW): Finished running pytest tests."
+	echo "$(NOW): Now performing unit tests." &&\
+	python3 -m pycommons.dev.building.run_tests --package moptipyapps &&\
+	echo "$(NOW): Finished running unit tests."
 
-# Perform static code analysis.
 static_analysis: init
-	echo "$(NOW): The original value of PATH is '${PATH}'." &&\
-	export PATH="${PATH}:${PYTHON_PACKAGE_BINARIES}" &&\
-	echo "$(NOW): PATH is now '${PATH}'." &&\
-	echo "$(NOW): Running static code analysis, starting with flake8." && \
-	flake8 . --ignore=,B008,B009,B010,DUO102,TRY003,TRY101,W503 && \
-	echo "$(NOW): Finished running flake8, now applying pylint to package." &&\
-	pylint moptipyapps --disable=C0103,C0302,C0325,R0801,R0901,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1702,R1728,W0212,W0238,W0703 &&\
-	echo "$(NOW): Done with pylint, now trying mypy." &&\
-	mypy moptipyapps --no-strict-optional --check-untyped-defs &&\
-	echo "$(NOW): Done with mypy, now doing pyflakes." &&\
-	python3 -m pyflakes . &&\
-	echo "$(NOW): Done with pyflakes, now applying bandit to find security issues." &&\
-	bandit -r moptipyapps -s B311 &&\
-	bandit -r examples -s B311 &&\
-	bandit -r tests -s B311,B101 &&\
-	echo "$(NOW): Done with bandit, now using pyroma to check setup.py." &&\
-	pyroma . &&\
-	echo "$(NOW): Done with pyroma, now applying semgrep." &&\
-	(semgrep --error --strict --use-git-ignore --skip-unknown-extensions --optimizations all --config=auto || semgrep --error --strict --use-git-ignore --skip-unknown-extensions --optimizations all --config=auto --verbose) &&\
-	echo "$(NOW): Done with semgrep, now applying pydocstyle." &&\
-	pydocstyle --convention=pep257 &&\
-	echo "$(NOW): Done with pydocstype, now applying tryceratops." &&\
-	tryceratops -i TRY003 -i TRY101 moptipyapps &&\
-	tryceratops -i TRY003 -i TRY101 examples &&\
-	tryceratops -i TRY003 -i TRY101 tests &&\
-	echo "$(NOW): Done with tryceratops, now applying unimport." &&\
-	unimport moptipyapps &&\
-	unimport examples &&\
-	unimport tests &&\
-	echo "$(NOW): Done with unimport, now applying vulture." &&\
-	vulture . --min-confidence 61 &&\
-	echo "$(NOW): Done with vulture, now applying dodgy." &&\
-	dodgy &&\
-	echo "$(NOW): Done with dodgy, now running pycodestyle." &&\
-	pycodestyle moptipyapps &&\
-	pycodestyle --ignore=E731,W503 examples &&\
-	pycodestyle tests &&\
-	echo "$(NOW): Done with pycodestyle, now running ruff." &&\
-	ruff --target-version py310 --select A,ANN,B,C,C4,COM,D,DJ,DTZ,E,ERA,EXE,F,G,I,ICN,INP,ISC,N,NPY,PIE,PLC,PLE,PLR,PLW,PT,PYI,Q,RET,RSE,RUF,S,SIM,T,T10,T20,TID,TRY,UP,W,YTT --ignore=ANN001,ANN002,ANN003,ANN101,ANN204,ANN401,B008,B009,B010,C901,D203,D208,D212,D401,D407,D413,N801,PLR0911,PLR0912,PLR0913,PLR0915,PLR2004,PYI041,RUF100,TRY003,UP035 --line-length 79 moptipyapps &&\
-	ruff --target-version py310 --select A,ANN,B,C,C4,COM,D,DJ,DTZ,E,ERA,EXE,F,G,I,ICN,ISC,N,NPY,PIE,PLC,PLE,PLR,PLW,PT,PYI,Q,RET,RSE,RUF,S,SIM,T10,TID,TRY,UP,W,YTT --ignore=ANN001,ANN002,ANN003,ANN101,ANN204,ANN401,B008,B009,B010,C901,D203,D208,D212,D401,D407,D413,N801,PLR0911,PLR0912,PLR0913,PLR0915,PLR2004,PYI041,RUF100,TRY003,UP035 --line-length 79 examples &&\
-	ruff --target-version py310 --select A,ANN,B,C,C4,COM,D,DJ,DTZ,E,ERA,EXE,F,G,I,ICN,ISC,N,NPY,PIE,PLC,PLE,PLR,PLW,PYI,Q,RET,RSE,RUF,T,SIM,T10,T20,TID,TRY,UP,W,YTT --ignore=ANN001,ANN002,ANN003,ANN101,ANN204,ANN401,B008,B009,B010,C901,D203,D208,D212,D401,D407,D413,N801,PLR0911,PLR0912,PLR0913,PLR0915,PLR2004,PYI041,RUF100,TRY003,UP035 --line-length 79 tests &&\
-    echo "$(NOW): Done with ruff, now running autoflake." &&\
-    autoflake -c -r moptipyapps &&\
-    autoflake -c -r tests &&\
-    autoflake -c -r examples &&\
+	echo "$(NOW): Now performing static analysis." &&\
+	python3 -m pycommons.dev.building.static_analysis --package moptipyapps &&\
 	echo "$(NOW): Done: All static checks passed."
 
-# We use sphinx to generate the documentation.
-# This automatically checks the docstrings and such and such.
 create_documentation: static_analysis test
-	echo "$(NOW): The original value of PATH is '${PATH}'." &&\
-	export PATH="${PATH}:${PYTHON_PACKAGE_BINARIES}" &&\
-	echo "$(NOW): PATH is now '${PATH}'." &&\
-	echo "$(NOW): First creating the .rst files from the source code." && \
-	sphinx-apidoc -M --ext-autodoc -o docs/source ./moptipyapps && \
-	echo "$(NOW): Now creating the documentation build folder and building the documentation." && \
-	sphinx-build -W -a -E -b html docs/source docs/build && \
-	echo "$(NOW): Done creating HTML documentation, cleaning up documentation temp files." && \
-	rm -rf docs/source/*.rst && \
-	rm -rf docs/source/*.md && \
-	echo "$(NOW): Now we pygmentize all the examples in 'examples' to 'build/examples'." &&\
-	mkdir -p docs/build/examples &&\
-	for f in examples/*.py; do \
-		if [ -z "$$f" ]; then \
-			echo "$(NOW): Empty module '$$f'?"; \
-		else \
-			echo "$(NOW): Now pygmentizing example '$$f'." &&\
-			{ pygmentize -f html -l python3 -O full -O style=default -o docs/build/"$${f%.py}.html" "$$f" || exit 1; };\
-		fi \
-	done &&\
-	echo "$(NOW): Finished pygmentizing all examples, now copying LICENSE and other files." &&\
-	pygmentize -f html -l text -O full -O style=default -o docs/build/LICENSE.html LICENSE &&\
-	pygmentize -f html -l text -O full -O style=default -o docs/build/requirements.html requirements.txt &&\
-	pygmentize -f html -l text -O full -O style=default -o docs/build/requirements-dev.html requirements-dev.txt &&\
-	pygmentize -f html -l make -O full -O style=default -o docs/build/Makefile.html Makefile &&\
-	echo "$(NOW): Finished copying LICENSE, now creating coverage report." &&\
-	mkdir -p docs/build/tc &&\
-	coverage html -d docs/build/tc --include="moptipyapps/*" &&\
-	echo "$(NOW): Now creating coverage badge." &&\
-	coverage-badge -o docs/build/tc/badge.svg &&\
-	if [[ -f docs/build/tc/badge.svg ]];then \
-		echo "$(NOW): docs/build/tc/badge.svg exists."; \
-	else \
-		echo "$(NOW): docs/build/tc/badge.svg does not exist!"; exit 1; \
-	fi &&\
-	echo "$(NOW): Deleting .gitignore file." &&\
-	rm -f docs/build/tc/.gitignore &&\
-	echo "$(NOW): Deleting useless _sources." &&\
-	rm -rf docs/build/_sources &&\
-	echo "$(NOW): Now rendering additional files." &&\
-	export PART_A='<!DOCTYPE html><html><title>' &&\
-	export PART_B='</title><link href=_static/bizstyle.css rel=stylesheet><body style="background-image:none"><div class=document><div class=documentwrapper><div class=bodywrapper><div class=body role=main><section>' &&\
-	export PART_C='</section></div></div></div></div></body></html>' &&\
-	export BASE_URL='https\:\/\/thomasweise\.github\.io\/moptipyapps\/' &&\
-	echo "$${PART_A}Contributing to moptipyapps$${PART_B}$(shell (python3 -m markdown -o html ./CONTRIBUTING.md))$$PART_C" > ./docs/build/CONTRIBUTING.html &&\
-	sed -i "s/\"$$BASE_URL/\".\//g" ./docs/build/CONTRIBUTING.html &&\
-	sed -i "s/=$$BASE_URL/=.\//g" ./docs/build/CONTRIBUTING.html &&\
-	echo "$${PART_A}Security Policy of moptipyapps$${PART_B}$(shell (python3 -m markdown -o html ./SECURITY.md))$$PART_C" > ./docs/build/SECURITY.html &&\
-	sed -i "s/\"$$BASE_URL/\".\//g" ./docs/build/SECURITY.html &&\
-	sed -i "s/=$$BASE_URL/=.\//g" ./docs/build/SECURITY.html &&\
-	echo "$(NOW): Now minifying all html files." &&\
-	cd "docs/build/" &&\
-	find -type f -name "*.html" -not -path "./tc/*" -exec python3 -c "print('{}');import minify_html;f=open('{}','r');s=f.read();f.close();s=minify_html.minify(s,do_not_minify_doctype=True,ensure_spec_compliant_unquoted_attribute_values=True,keep_html_and_head_opening_tags=False,minify_css=True,minify_js=True,remove_bangs=True,remove_processing_instructions=True);f=open('{}','w');f.write(s);f.close()" \; &&\
-	cd "../../" &&\
-	echo "$(NOW): Done creating coverage data. Now creating .nojekyll files." &&\
-	cd "docs/build/" &&\
-	find -type d -exec touch "{}/.nojekyll" \;
-	cd "../../" &&\
-	echo "$(NOW): Done creating the documentation."
+	echo "$(NOW): Now building documentation." &&\
+	python3 -m pycommons.dev.building.make_documentation --root . --package moptipyapps &&\
+	echo "$(NOW): Done building documentation."
 
-# Create different distribution formats, also to check if there is any error.
 create_distribution: static_analysis test create_documentation
 	echo "$(NOW): Now building source distribution file." &&\
-	python3 setup.py check &&\
-	python3 -m build &&\
-	echo "$(NOW): Done with the build process, now checking result." &&\
-	python3 -m twine check dist/* &&\
-	echo "$(NOW): Now testing the tar.gz." &&\
-	export tempDir=`mktemp -d` &&\
-	echo "$(NOW): Created temp directory '$$tempDir'. Creating virtual environment." &&\
-	python3 -m venv "$$tempDir" &&\
-	echo "$(NOW): Created virtual environment, now activating it." &&\
-	source "$$tempDir/bin/activate" &&\
-	echo "$(NOW): Now installing tar.gz." &&\
-	python3 -m pip --no-input --timeout 360 --retries 100 --require-virtualenv install "$(CWD)/dist/moptipyapps-$(VERSION).tar.gz" && ## nosem \
-	echo "$(NOW): Installing tar.gz has worked. We now create the list of packages in this environment via pip freeze." &&\
-	pip freeze > "$(CWD)/dist/moptipyapps-$(VERSION)-requirements_frozen.txt" &&\
-	echo "$(NOW): Now fixing moptipyapps line in requirements file." &&\
-	sed -i "s/^moptipyapps.*/moptipyapps==$(VERSION)/" "$(CWD)/dist/moptipyapps-$(VERSION)-requirements_frozen.txt" &&\
-	echo "$(NOW): Now we deactivate the environment." &&\
-	deactivate &&\
-	rm -rf "$$tempDir" &&\
-	echo "$(NOW): Now testing the wheel." &&\
-	export tempDir=`mktemp -d` &&\
-	echo "$(NOW): Created temp directory '$$tempDir'. Creating virtual environment." &&\
-	python3 -m venv "$$tempDir" &&\
-	echo "$(NOW): Created virtual environment, now activating it." &&\
-	source "$$tempDir/bin/activate" &&\
-	echo "$(NOW): Now installing wheel." &&\
-	python3 -m pip --no-input --timeout 360 --retries 100 --require-virtualenv install "$(CWD)/dist/moptipyapps-$(VERSION)-py3-none-any.whl" && ## nosem \
-	echo "$(NOW): Now we deactivate the environment." &&\
-	deactivate &&\
-	echo "$(NOW): Finished, cleaning up." &&\
-	rm -rf "$$tempDir" &&\
-	echo "$(NOW): Now also packaging the documentation." &&\
-	cd docs/build &&\
-	tar --dereference --exclude=".nojekyll" -c * | xz -v -9e -c > "$(CWD)/dist/moptipyapps-$(VERSION)-documentation.tar.xz" &&\
-	cd $(CWD) &&\
+	export PYTHONPATH=".:${PYTHONPATH}" &&\
+	python3 -m pycommons.dev.building.make_dist --root . --package moptipyapps &&\
 	echo "$(NOW): Successfully finished building source distribution."
 
 # We install the package and see if that works out.

moptipyapps/binpacking2d/encodings/ibl_encoding_1.py

@@ -162,9 +162,8 @@ def __move_down(packing: np.ndarray, bin_start: int, i1: int) -> bool:
                 (packing[i0, IDX_LEFT_X] < packing_i1_right_x) and \
                 (packing[i0, IDX_BOTTOM_Y] < packing_i1_top_y):
             # The object would horizontally intersect with the current object
-            diff: int = int(packing_i1_bottom_y - packing[i0, IDX_TOP_Y])
-            if diff < min_down:  # we can move down only a shorter distance
-                min_down = diff  # ok, a shorter distance was found
+            min_down = min(min_down, int(
+                packing_i1_bottom_y - packing[i0, IDX_TOP_Y]))
     if min_down > 0:  # Can we move down? If yes, update box.
         packing[i1, IDX_BOTTOM_Y] = packing_i1_bottom_y - min_down
         packing[i1, IDX_TOP_Y] = packing_i1_top_y - min_down
@@ -239,14 +238,12 @@ def __move_left(packing: np.ndarray, bin_start: int, i1: int) -> bool:
             if packing[i0, IDX_TOP_Y] == packing_i1_bottom_y:
                 # only consider those the box *directly* below and move the
                 # right end of the new box to the left end of that box below
-                diff = int(packing_i1_right_x - packing[i0, IDX_LEFT_X])
-                if diff < min_left:
-                    min_left = diff
+                min_left = min(min_left, int(
+                    packing_i1_right_x - packing[i0, IDX_LEFT_X]))
         elif (packing_i1_top_y > packing[i0, IDX_BOTTOM_Y]) \
                 and (packing_i1_bottom_y < packing[i0, IDX_TOP_Y]):
-            diff = int(packing_i1_left_x - packing[i0, IDX_RIGHT_X])
-            if diff < min_left:
-                min_left = diff
+            min_left = min(min_left, int(
+                packing_i1_left_x - packing[i0, IDX_RIGHT_X]))
     if min_left > 0:
         # move the box to the left
         packing[i1, IDX_LEFT_X] = packing_i1_left_x - min_left

moptipyapps/binpacking2d/encodings/ibl_encoding_2.py

@@ -193,9 +193,8 @@ def __move_down(packing: np.ndarray, bin_id: int,
                 (packing[i0, IDX_LEFT_X] < packing_i1_right_x) and \
                 (packing[i0, IDX_BOTTOM_Y] < packing_i1_top_y):
             # The object would horizontally intersect with the current object
-            diff: int = int(packing_i1_bottom_y - packing[i0, IDX_TOP_Y])
-            if diff < min_down:  # we can move down only a shorter distance
-                min_down = diff  # ok, a shorter distance was found
+            min_down = min(min_down, int(
+                packing_i1_bottom_y - packing[i0, IDX_TOP_Y]))
     if min_down > 0:  # Can we move down? If yes, update box.
         packing[i1, IDX_BOTTOM_Y] = packing_i1_bottom_y - min_down
         packing[i1, IDX_TOP_Y] = packing_i1_top_y - min_down
@@ -278,14 +277,12 @@ def __move_left(packing: np.ndarray, bin_id: int,
             if packing[i0, IDX_TOP_Y] == packing_i1_bottom_y:
                 # only consider those the box *directly* below and move the
                 # right end of the new box to the left end of that box below
-                diff = int(packing_i1_right_x - packing[i0, IDX_LEFT_X])
-                if diff < min_left:
-                    min_left = diff
+                min_left = min(min_left, int(
+                    packing_i1_right_x - packing[i0, IDX_LEFT_X]))
         elif (packing_i1_top_y > packing[i0, IDX_BOTTOM_Y]) \
                 and (packing_i1_bottom_y < packing[i0, IDX_TOP_Y]):
-            diff = int(packing_i1_left_x - packing[i0, IDX_RIGHT_X])
-            if diff < min_left:
-                min_left = diff
+            min_left = min(min_left, int(
+                packing_i1_left_x - packing[i0, IDX_RIGHT_X]))
     if min_left > 0:
         # move the box to the left
         packing[i1, IDX_LEFT_X] = packing_i1_left_x - min_left

moptipyapps/binpacking2d/instance.py

@@ -123,7 +123,7 @@
 from moptipy.utils.logger import CSV_SEPARATOR, KeyValueLogSection
 from moptipy.utils.nputils import int_range_to_dtype
 from moptipy.utils.strings import sanitize_name
-from pycommons.io.path import Path, file_path
+from pycommons.io.path import UTF8, Path, file_path
 from pycommons.types import check_int_range, check_to_int_range, type_error
 
 #: the instances resource name
@@ -907,8 +907,8 @@ def from_resource(name: str) -> "Instance":
         if hasattr(container, text_attr):  # ok, we got the text already
             text = cast(list[str], getattr(container, text_attr))
         else:  # the first time we load the text
-            with resources.open_text(package=str(__package__),
-                                     resource=INSTANCES_RESOURCE) as stream:
+            with resources.files(__package__).joinpath(
+                    INSTANCES_RESOURCE).open("r", encoding=UTF8) as stream:
                 text = [line for line in (line_raw.strip() for line_raw
                                           in stream.readlines())
                         if len(line) > 0]

moptipyapps/binpacking2d/objectives/bin_count_and_empty.py

@@ -63,8 +63,7 @@ def bin_count_and_empty(y: np.ndarray, temp: np.ndarray) -> int:
     for i in range(n_items):  # iterate over all packed items
         bin_idx: int = int(y[i, IDX_BIN]) - 1  # get the bin index of the item
         temp[bin_idx] += 1  # increase number of items
-        if bin_idx > total_bins:  # it's a new biggest bin = new last bin?
-            total_bins = bin_idx  # and we remember it
+        total_bins = max(total_bins, bin_idx)
     return (n_items * total_bins) + temp[0:total_bins + 1].min()
 
 

moptipyapps/binpacking2d/objectives/bin_count_and_last_skyline.py

@@ -138,8 +138,7 @@ def bin_count_and_last_skyline(y: np.ndarray, bin_width: int,
             if cur_left < left < next_left:
                 next_left = left
 
-        if next_left < use_right:
-            use_right = next_left
+        use_right = min(use_right, next_left)
         area_under_skyline += (use_right - cur_left) * use_top
         cur_left = use_right
     return ((bins - 1) * bin_size) + area_under_skyline

moptipyapps/binpacking2d/objectives/bin_count_and_lowest_skyline.py

@@ -122,12 +122,11 @@ def bin_count_and_lowest_skyline(y: np.ndarray, bin_width: int,
                 if cur_left < left < next_left:
                     next_left = left
 
-            if next_left < use_right:
-                use_right = next_left
+            use_right = min(use_right, next_left)
             area_under_skyline += (use_right - cur_left) * use_top
             cur_left = use_right
-        if area_under_skyline <= min_area_under_skyline:
-            min_area_under_skyline = area_under_skyline
+        min_area_under_skyline = min(min_area_under_skyline,
+                                     area_under_skyline)
     return ((bins - 1) * bin_size) + min_area_under_skyline
 
 

moptipyapps/binpacking2d/objectives/bin_count_and_small.py

@@ -88,8 +88,7 @@ def bin_count_and_small(y: np.ndarray, bin_area: int,
         bin_idx: int = int(y[i, IDX_BIN]) - 1  # get the bin index of the item
         temp[bin_idx] += ((y[i, IDX_RIGHT_X] - y[i, IDX_LEFT_X])
                           * (y[i, IDX_TOP_Y] - y[i, IDX_BOTTOM_Y]))
-        if bin_idx > total_bins:  # is it a new last bin?
-            total_bins = bin_idx  # and we remember it
+        total_bins = max(total_bins, bin_idx)
     return (bin_area * total_bins) + temp[0:total_bins + 1].min()
 
 

moptipyapps/dynamic_control/ode.py

@@ -141,8 +141,7 @@ def f(self, t: float, state: np.ndarray) -> np.ndarray:
                 self.max_ok_t = t
         else:  # no: there was some error, either in state or controller
             self.is_ok = False  # then we are no longer OK
-            if t < self.min_error_t:  # update the minimum error time
-                self.min_error_t = t
+            self.min_error_t = min(self.min_error_t, t)
             if self.max_ok_t > t:  # what? an earlier error?
                 self.max_ok_t = np.nextafter(t, -inf)  # ok, reset ok time
 

moptipyapps/qap/instance.py

@@ -376,7 +376,7 @@ def from_resource(name: str) -> "Instance":
         if hasattr(container, inst_attr):  # instance loaded?
             return cast(Instance, getattr(container, inst_attr))
 
-        lb: int | None = _BOUNDS[name] if name in _BOUNDS else None
+        lb: int | None = _BOUNDS.get(name, None)
         with open_resource_stream(f"{name}.dat") as stream:
             inst: Final[Instance] = Instance.from_qaplib_stream(
                 stream, name=name, lower_bound=lb)

moptipyapps/qap/qaplib/__init__.py

@@ -11,7 +11,9 @@
 """
 
 from importlib import resources  # nosem
-from typing import TextIO
+from typing import TextIO, cast
+
+from pycommons.io.path import UTF8
 
 
 def open_resource_stream(file_name: str) -> TextIO:
@@ -21,4 +23,5 @@ def open_resource_stream(file_name: str) -> TextIO:
     :param file_name: the file name of the resource
     :return: the stream
     """
-    return resources.open_text(package=f"{__package__}", resource=file_name)
+    return cast(TextIO, resources.files(__package__).joinpath(
+        file_name).open("r", encoding=UTF8))