diff --git a/Makefile b/Makefile index 1b1e7b26..7266d959 100644 --- a/Makefile +++ b/Makefile @@ -37,9 +37,7 @@ clean: status rm -rf build && \ rm -rf dist && \ rm -rf docs/build && \ - mv docs/source/index.rst docs/source/index.x && \ rm -rf docs/source/*.rst && \ - mv docs/source/index.x docs/source/index.rst && \ rm -rf moptipyapps.egg-info && \ echo "$(NOW): Done cleaning up, moptipyapps is uninstalled and auto-generated stuff is deleted." @@ -64,7 +62,7 @@ test: init echo "$(NOW): Running pytest with doctests." && \ timeout --kill-after=15s 90m coverage run -a --include="moptipyapps/*" -m pytest --strict-config --doctest-modules --ignore=tests --ignore=examples && \ echo "$(NOW): Running pytest tests." && \ - timeout --kill-after=15s 90m coverage run --include="moptipyapps/*" -m pytest --strict-config tests --ignore=examples && \ + timeout --kill-after=15s 90m coverage run -a --include="moptipyapps/*" -m pytest --strict-config tests --ignore=examples && \ echo "$(NOW): Finished running pytest tests." # Perform static code analysis. @@ -127,10 +125,8 @@ create_documentation: static_analysis test echo "$(NOW): Now creating the documentation build folder and building the documentation." && \ sphinx-build -W -a -E -b html docs/source docs/build && \ echo "$(NOW): Done creating HTML documentation, cleaning up documentation temp files." && \ - mv docs/source/index.rst docs/source/index.tmp && \ rm -rf docs/source/*.rst && \ rm -rf docs/source/*.md && \ - mv docs/source/index.tmp docs/source/index.rst && \ echo "$(NOW): Now we pygmentize all the examples in 'examples' to 'build/examples'." &&\ mkdir -p docs/build/examples &&\ for f in examples/*.py; do \ diff --git a/docs/source/conf.py b/docs/source/conf.py index 166ae380..fef5fb2d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,136 +1,15 @@ """The configuration for sphinx to generate the documentation.""" -import datetime -import os -import re -import sys +from typing import Final -# the path of the documentation configuration -doc_path = os.path.abspath(os.path.dirname(__file__)) - -# get the path to the root directory of this project -root_path = os.path.abspath(os.path.join(doc_path, "..", "..")) -sys.path.insert(0, root_path) - -# set the base url -html_baseurl = "https://thomasweise.github.io/moptipyapps/" - -# We want to include the contents of our GitHub README.md file. -# So first, we need to load the README.md file. -old_lines: list[str] -with open(os.path.join(root_path, "README.md"), - encoding="utf-8-sig") as reader: - old_lines = reader.readlines() - -# Now, we need to fix the file contents. -# We discard the top-level heading as well as the badge for the build status. -# We need to move all sub-headings one step up. -# Furthermore, we can turn all absolute URLs starting with -# http://thomasweise.github.io/moptipyapps/xxx to local references, i.e., -# ./xxx. Finally, it seems that the myst parser now again drops the numerical -# prefixes of links, i.e., it tags `## 1.2. Hello` with id `hello` instead of -# `12-hello`. This means that we need to fix all references following the -# pattern `[xxx](#12-hello)` to `[xxx](#hello)`. We do this with a regular -# expression `regex_search`. -new_lines = [] -in_code: bool = False # we only process non-code lines -skip: bool = True -# detects strings of the form [xyz](#123-bla) and gives \1=xyz and \2=bla -regex_search = re.compile("(\\[.+?])\\(#\\d+-(.+?)\\)") -license_link: str = \ - "https://github.com/thomasWeise/moptipyapps/blob/main/LICENSE" -needs_newline: bool = False -can_add_anyway: bool = True -for line in old_lines: - if skip: # we skip everything until the introduction section - if line.lstrip().startswith("## 1. Introduction"): - skip = False - new_lines.append(line[1:]) - elif line.startswith("[![") and can_add_anyway: - needs_newline = True - new_lines.append(line) - continue - else: - can_add_anyway = False - continue - if needs_newline: - new_lines.append("") - needs_newline = False - continue - if in_code: - if line.startswith("```"): - in_code = False # toggle to non-code - else: - if line.startswith("```"): - in_code = True # toggle to code - elif line.startswith("#"): - line = line[1:] # move all sub-headings one step up - else: # fix all internal urls - # replace links of the form "#12-bla" to "#bla" - line = re.sub(regex_search, "\\1(#\\2)", line) - - line = line.replace(license_link, "./LICENSE.html") - for k in [html_baseurl, f"http{html_baseurl[5:]}"]: - line = line.replace(f"]({k}", "](./")\ - .replace(f' src="{k}', ' src="./')\ - .replace(f' href="{k}', ' href="./') - - new_lines.append(line) - -# write the post-processed README.md file -with open(os.path.join(doc_path, "README.md"), "w", - encoding="utf-8") as outf: - outf.writelines(new_lines) - -# enable myst header anchors -myst_heading_anchors = 6 - -# project information -project = "moptipyapps" -author = "Thomas Weise" -# noinspection PyShadowingBuiltins -copyright = f"2023-{datetime.datetime.now ().year}, {author}" +from pycommons.dev.doc.setup_doc import setup_doc +from pycommons.io.path import Path, file_path -# tell sphinx to go kaboom on errors -nitpicky = True -myst_all_links_external = True - -# The full version, including alpha/beta/rc tags. -release = {} -with open(os.path.abspath(os.path.sep.join([ - root_path, "moptipyapps", "version.py"]))) as fp: - exec(fp.read(), release) # nosec # nosemgrep # noqa: DUO105 -release = release["__version__"] - -# The Sphinx extension modules that we use. -extensions = ["myst_parser", # for processing README.md - "sphinx.ext.autodoc", # to convert docstrings to documentation - "sphinx.ext.doctest", # do the doc tests again - "sphinx.ext.intersphinx", # to link to numpy et al. - "sphinx_autodoc_typehints", # to infer types from hints - "sphinx.ext.viewcode", # add rendered source code - ] - -# Location of dependency documentation for cross-referencing. -intersphinx_mapping = { - "matplotlib": ("https://matplotlib.org/stable/", None), - "moptipy": ("https://thomasweise.github.io/moptipy/", None), - "numpy": ("https://numpy.org/doc/stable/", None), - "python": ("https://docs.python.org/3/", None), - 'scipy': ('https://docs.scipy.org/doc/scipy/', None), - "urllib3": ("https://urllib3.readthedocs.io/en/stable/", None), -} - -# inherit docstrings in autodoc -autodoc_inherit_docstrings = True - -# add default values after comma -typehints_defaults = "comma" - -# the sources to be processed -source_suffix = [".rst", ".md"] - -# The theme to use for HTML and HTML Help pages. -html_theme = "bizstyle" - -# Code syntax highlighting style: -pygments_style = "default" +# the path of the documentation configuration +doc_path: Final[Path] = file_path(__file__).up(1) +root_path: Final[Path] = doc_path.up(2) +setup_doc(doc_path, root_path, 2023, dependencies=( + "matplotlib", "moptipy", "numpy", "pycommons", "scipy", "sklearn", + "urllib3"), + full_urls={ + "https://github.com/thomasWeise/moptipyapps/blob/main/LICENSE": + "./LICENSE.html"}) diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index bb2fa97d..00000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -The *Applications of Metaheuristic Optimization in Python* Package. -============================================================================================ - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - -.. include:: README.md - :parser: myst_parser.sphinx_ - -7. Modules and Code --------------------- - -.. toctree:: - :maxdepth: 4 - - modules diff --git a/examples/binpacking2d_plot.py b/examples/binpacking2d_plot.py index afab844e..eac067da 100644 --- a/examples/binpacking2d_plot.py +++ b/examples/binpacking2d_plot.py @@ -24,7 +24,7 @@ from moptipy.utils.nputils import rand_generator from moptipy.utils.plot_utils import save_figure from moptipy.utils.sys_info import is_make_build -from moptipy.utils.temp import TempDir +from pycommons.io.temp import temp_dir from moptipyapps.binpacking2d.encodings.ibl_encoding_1 import ( ImprovedBottomLeftEncoding1, @@ -50,7 +50,8 @@ # can occur multiple times, i.e., may occur repeatedly. # So we include each of n_different_items item IDs exactly as often as # it should be repeated. We sometimes include it directly and sometimes -# in the negated form, which is interpreted as a 90° rotation by the encoding. +# in the negated form, which is interpreted as a 90deg rotation by the +# encoding. # Then, we convert the list of item IDs to a numpy array and, finally, shuffle # the array. The encoding then inserts the items in the same sequence they # appear in the array into bins. @@ -78,10 +79,10 @@ # We can now plot the packing. We create the figures in a temp directory. -# To keep the figures, you would put an existing directory path into `td` -# by doing `from moptipy.utils.path import Path; td = Path.directory("mydir")` -# and not use the `with` block. -with TempDir.create() as td: # create temporary directory `td` +# To keep the figures, you would put an existing directory path into `td` by +# doing `from pycommons.io.path import Path; td = Path("mydir")` and not use +# the `with` block. +with temp_dir() as td: # create temporary directory `td` files = [] # the collection of files # Plot the packings. The default plotting includes the item ID into each diff --git a/examples/binpacking2d_rls.py b/examples/binpacking2d_rls.py index a22fb7bb..b09832ef 100644 --- a/examples/binpacking2d_rls.py +++ b/examples/binpacking2d_rls.py @@ -27,7 +27,7 @@ ) from moptipy.spaces.signed_permutations import SignedPermutations from moptipy.utils.plot_utils import save_figure -from moptipy.utils.temp import TempDir +from pycommons.io.temp import temp_dir from moptipyapps.binpacking2d.encodings.ibl_encoding_2 import ( ImprovedBottomLeftEncoding2, @@ -72,10 +72,10 @@ process.get_copy_of_best_y(y) # We can now plot the best packing. We create the figures in a temp directory. -# To keep the figures, you would put an existing directory path into `td` -# by doing `from moptipy.utils.path import Path; td = Path.directory("mydir")` -# and not use the `with` block. -with TempDir.create() as td: # create temporary directory `td` +# To keep the figures, you would put an existing directory path into `td` by +# doing `from pycommons.io.path import Path; td = Path("mydir")` and not use +# the `with` block. +with temp_dir() as td: # create temporary directory `td` files = [] # the collection of files # Plot the packing. The default plotting includes the item ID into each diff --git a/examples/binpacking2d_timing.py b/examples/binpacking2d_timing.py index 8c121e77..cf2e05e8 100644 --- a/examples/binpacking2d_timing.py +++ b/examples/binpacking2d_timing.py @@ -7,7 +7,7 @@ import numpy as np import psutil from moptipy.utils.nputils import rand_generator -from moptipy.utils.types import type_name +from pycommons.types import type_name from moptipyapps.binpacking2d.encodings.ibl_encoding_1 import ( ImprovedBottomLeftEncoding1, diff --git a/examples/order1_from_dat.py b/examples/order1_from_dat.py index ad607a0b..5c41091b 100644 --- a/examples/order1_from_dat.py +++ b/examples/order1_from_dat.py @@ -62,18 +62,18 @@ from moptipy.api.execution import Execution from moptipy.operators.permutations.op0_shuffle import Op0Shuffle from moptipy.operators.permutations.op1_swap2 import Op1Swap2 -from moptipy.utils.console import logger -from moptipy.utils.help import argparser from moptipy.utils.nputils import rand_seeds_from_str -from moptipy.utils.path import Path from moptipy.utils.plot_defaults import distinct_colors, distinct_markers from moptipy.utils.plot_utils import create_figure, get_axes, save_figure -from moptipy.utils.types import check_to_int_range +from pycommons.io.console import logger +from pycommons.io.path import Path, directory_path +from pycommons.types import check_to_int_range from moptipyapps.order1d.distances import swap_distance from moptipyapps.order1d.instance import Instance from moptipyapps.order1d.space import OrderingSpace from moptipyapps.qap.objective import QAPObjective +from moptipyapps.shared import moptipyapps_argparser #: the impact of rank differences POWER: Final[float] = 2.0 @@ -100,7 +100,7 @@ def parse_data(path: str, collector: Callable[[ :param fitness_limit: the minimum acceptable fitness :param pattern: the file name pattern """ - the_path: Final[Path] = Path.path(path) + the_path: Final[Path] = Path(path) if isdir(the_path): # recursively parse directories logger(f"recursing into directory '{the_path}'.") for subpath in listdir(the_path): @@ -191,7 +191,7 @@ def run(source: str, dest: str, max_fes: int, fitness_limit: int, del data # free the now useless data # run the algorithm - dest_file: Final[Path] = Path.path(f"{dest}.txt") + dest_file: Final[Path] = Path(f"{dest}.txt") logger(f"finished constructing matrix with {instance.n} rows, " "now doing optimization for " f"{max_fes} FEs and, afterwards, writing result to '{dest_file}'.") @@ -211,7 +211,7 @@ def run(source: str, dest: str, max_fes: int, fitness_limit: int, files: Final[list[str]] = sorted({f[0][0] for f in instance.tags}) logger(f"now we begin plotting the data from {len(files)} files.") - dest_dir: Final[Path] = Path.directory(dirname(dest_file)) + dest_dir: Final[Path] = directory_path(dirname(dest_file)) figure: Final[Figure] = create_figure() colors: Final[tuple[tuple[float, float, float], ...]] \ = distinct_colors(len(files)) @@ -231,15 +231,15 @@ def run(source: str, dest: str, max_fes: int, fitness_limit: int, # Perform the optimization if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "One-Dimensional Ordering of Permutations", "Run the one-dimensional order of permutations experiment.") parser.add_argument( "source", help="the directory or file with the input data", - type=Path.path, nargs="?", default="./") + type=Path, nargs="?", default="./") parser.add_argument( "dest", help="the file to write the output to", - type=Path.path, nargs="?", default="./result.txt") + type=Path, nargs="?", default="./result.txt") parser.add_argument("fitnessLimit", help="the minimum acceptable fitness", type=int, nargs="?", default=1_000_000_000) parser.add_argument("maxFEs", help="the maximum FEs to perform", diff --git a/examples/qap_example_experiment_rls_rs.py b/examples/qap_example_experiment_rls_rs.py index f37f46f9..f901fef4 100644 --- a/examples/qap_example_experiment_rls_rs.py +++ b/examples/qap_example_experiment_rls_rs.py @@ -10,11 +10,11 @@ from moptipy.operators.permutations.op0_shuffle import Op0Shuffle from moptipy.operators.permutations.op1_swap2 import Op1Swap2 from moptipy.spaces.permutations import Permutations -from moptipy.utils.help import argparser -from moptipy.utils.path import Path +from pycommons.io.path import Path from moptipyapps.qap.instance import Instance from moptipyapps.qap.objective import QAPObjective +from moptipyapps.shared import moptipyapps_argparser def make_instances() -> Iterable[Callable[[], Instance]]: @@ -68,7 +68,7 @@ def run(base_dir: str, n_runs: int = 3) -> None: :param base_dir: the base directory :param n_runs: the number of runs """ - use_dir: Final[Path] = Path.path(base_dir) + use_dir: Final[Path] = Path(base_dir) use_dir.ensure_dir_exists() run_experiment( @@ -81,11 +81,11 @@ def run(base_dir: str, n_runs: int = 3) -> None: # Run the experiment from the command line if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "Quadratic Assignment Problem (QAP)", "Run the QAP experiment.") parser.add_argument( "dest", help="the directory to store the experimental results under", - type=Path.path, nargs="?", default="./results/") + type=Path, nargs="?", default="./results/") args: Final[argparse.Namespace] = parser.parse_args() run(args.dest) diff --git a/examples/ttp_example_experiment_mo.py b/examples/ttp_example_experiment_mo.py index bb758728..908a5bf5 100644 --- a/examples/ttp_example_experiment_mo.py +++ b/examples/ttp_example_experiment_mo.py @@ -24,9 +24,9 @@ Op2GeneralizedAlternatingPosition, ) from moptipy.spaces.permutations import Permutations -from moptipy.utils.help import argparser -from moptipy.utils.path import Path +from pycommons.io.path import Path +from moptipyapps.shared import moptipyapps_argparser from moptipyapps.ttp.errors import Errors from moptipyapps.ttp.game_encoding import GameEncoding from moptipyapps.ttp.game_plan_space import GamePlanSpace @@ -92,7 +92,7 @@ def run(base_dir: str, n_runs: int = 3) -> None: :param base_dir: the base directory :param n_runs: the number of runs """ - use_dir: Final[Path] = Path.path(base_dir) + use_dir: Final[Path] = Path(base_dir) use_dir.ensure_dir_exists() run_experiment( @@ -105,11 +105,11 @@ def run(base_dir: str, n_runs: int = 3) -> None: # Run the experiment from the command line if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "Traveling Tournament Problem (TTP)", "Run the Multi-Objective TTP experiment.") parser.add_argument( "dest", help="the directory to store the experimental results under", - type=Path.path, nargs="?", default="./results/") + type=Path, nargs="?", default="./results/") args: Final[argparse.Namespace] = parser.parse_args() run(args.dest) diff --git a/examples/ttp_example_experiment_rls_rs.py b/examples/ttp_example_experiment_rls_rs.py index ddafaa20..a0f63b11 100644 --- a/examples/ttp_example_experiment_rls_rs.py +++ b/examples/ttp_example_experiment_rls_rs.py @@ -44,9 +44,9 @@ from moptipy.operators.permutations.op0_shuffle import Op0Shuffle from moptipy.operators.permutations.op1_swap2 import Op1Swap2 from moptipy.spaces.permutations import Permutations -from moptipy.utils.help import argparser -from moptipy.utils.path import Path +from pycommons.io.path import Path +from moptipyapps.shared import moptipyapps_argparser from moptipyapps.ttp.errors import Errors from moptipyapps.ttp.game_encoding import GameEncoding from moptipyapps.ttp.game_plan_space import GamePlanSpace @@ -110,7 +110,7 @@ def run(base_dir: str, n_runs: int = 3) -> None: :param base_dir: the base directory :param n_runs: the number of runs """ - use_dir: Final[Path] = Path.path(base_dir) + use_dir: Final[Path] = Path(base_dir) use_dir.ensure_dir_exists() run_experiment( @@ -123,11 +123,11 @@ def run(base_dir: str, n_runs: int = 3) -> None: # Run the experiment from the command line if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "Traveling Tournament Problem (TTP)", "Run the TTP experiment.") parser.add_argument( "dest", help="the directory to store the experimental results under", - type=Path.path, nargs="?", default="./results/") + type=Path, nargs="?", default="./results/") args: Final[argparse.Namespace] = parser.parse_args() run(args.dest) diff --git a/moptipyapps/binpacking2d/encodings/ibl_encoding_1.py b/moptipyapps/binpacking2d/encodings/ibl_encoding_1.py index c5aa8019..12317f26 100644 --- a/moptipyapps/binpacking2d/encodings/ibl_encoding_1.py +++ b/moptipyapps/binpacking2d/encodings/ibl_encoding_1.py @@ -69,7 +69,7 @@ import numpy as np from moptipy.api.encoding import Encoding from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.binpacking2d.instance import ( IDX_HEIGHT, diff --git a/moptipyapps/binpacking2d/encodings/ibl_encoding_2.py b/moptipyapps/binpacking2d/encodings/ibl_encoding_2.py index b776041f..935a46f8 100644 --- a/moptipyapps/binpacking2d/encodings/ibl_encoding_2.py +++ b/moptipyapps/binpacking2d/encodings/ibl_encoding_2.py @@ -94,7 +94,7 @@ import numpy as np from moptipy.api.encoding import Encoding from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.binpacking2d.instance import ( IDX_HEIGHT, diff --git a/moptipyapps/binpacking2d/experiment.py b/moptipyapps/binpacking2d/experiment.py index 89b9e276..5f88909d 100644 --- a/moptipyapps/binpacking2d/experiment.py +++ b/moptipyapps/binpacking2d/experiment.py @@ -16,8 +16,7 @@ Op1Swap2OrFlip, ) from moptipy.spaces.signed_permutations import SignedPermutations -from moptipy.utils.help import argparser -from moptipy.utils.path import Path +from pycommons.io.path import Path from moptipyapps.binpacking2d.encodings.ibl_encoding_1 import ( ImprovedBottomLeftEncoding1, @@ -25,6 +24,7 @@ from moptipyapps.binpacking2d.instance import Instance from moptipyapps.binpacking2d.packing_result import DEFAULT_OBJECTIVES from moptipyapps.binpacking2d.packing_space import PackingSpace +from moptipyapps.shared import moptipyapps_argparser #: the maximum number of FEs MAX_FES: Final[int] = 1_000_000 @@ -91,7 +91,7 @@ def run(base_dir: str, n_runs: int = 23) -> None: :param base_dir: the base directory :param n_runs: the number of runs, by default `23` """ - use_dir: Final[Path] = Path.path(base_dir) + use_dir: Final[Path] = Path(base_dir) use_dir.ensure_dir_exists() encodings: Final[tuple[Callable[[Instance], Encoding], ...]] = ( @@ -128,10 +128,10 @@ def run(base_dir: str, n_runs: int = 23) -> None: # Run the experiment from the command line if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "2D Bin Packing", "Run the 2D Bin Packing experiment.") parser.add_argument( "dest", help="the directory to store the experimental results under", - type=Path.path, nargs="?", default="./results/") + type=Path, nargs="?", default="./results/") args: Final[argparse.Namespace] = parser.parse_args() run(args.dest) diff --git a/moptipyapps/binpacking2d/instance.py b/moptipyapps/binpacking2d/instance.py index 24f32b35..b2c01baf 100644 --- a/moptipyapps/binpacking2d/instance.py +++ b/moptipyapps/binpacking2d/instance.py @@ -122,9 +122,9 @@ from moptipy.api.component import Component from moptipy.utils.logger import CSV_SEPARATOR, KeyValueLogSection from moptipy.utils.nputils import int_range_to_dtype -from moptipy.utils.path import Path from moptipy.utils.strings import sanitize_name -from moptipy.utils.types import check_int_range, check_to_int_range, type_error +from pycommons.io.path import Path, file_path +from pycommons.types import check_int_range, check_to_int_range, type_error #: the instances resource name INSTANCES_RESOURCE: Final[str] = "instances.txt" @@ -682,12 +682,12 @@ def from_2dpacklib(file: str) -> "Instance": :param file: the file path :return: the instance """ - path: Final[Path] = Path.file(file) + path: Final[Path] = file_path(file) name: str = basename(path).lower() if name.endswith(".ins2d"): name = sanitize_name(name[:-6]) - lines: Final[list[str]] = path.read_all_list() + lines: Final[list[str]] = str.splitlines(path.read_all_str()) n_different_items: Final[int] = check_to_int_range( lines[0], "n_different_items", 1, 100_000_000) diff --git a/moptipyapps/binpacking2d/make_instances.py b/moptipyapps/binpacking2d/make_instances.py index 413e4790..7d14824d 100644 --- a/moptipyapps/binpacking2d/make_instances.py +++ b/moptipyapps/binpacking2d/make_instances.py @@ -25,9 +25,9 @@ # noinspection PyPackageRequirements import urllib3 # type: ignore -from moptipy.utils.path import Path -from moptipy.utils.temp import TempDir -from moptipy.utils.types import type_error +from pycommons.io.path import Path, directory_path, file_path, write_lines +from pycommons.io.temp import temp_dir +from pycommons.types import type_error import moptipyapps.binpacking2d.instance as inst_mod from moptipyapps.binpacking2d.instance import ( @@ -65,7 +65,7 @@ def download_2dpacklib_instances( :param http: the HTTP pool :return: the list of unpackaged files """ - dest: Final[Path] = Path.directory(dest_dir) + dest: Final[Path] = directory_path(dest_dir) if not isinstance(source_urls, Iterable): raise type_error(source_urls, "source_urls", Iterable) result: Final[list[Path]] = [] @@ -153,15 +153,16 @@ def join_instances_to_compact( raise type_error(binpacklib2d_files, "files", Iterable) if not callable(normalizer): raise type_error(normalizer, "normalizer", call=True) - dest_path = Path.path(dest_file) + dest_path = Path(dest_file) data: Final[list[tuple[str, str]]] = [] for file in binpacklib2d_files: - inst: Instance = Instance.from_2dpacklib(Path.file(file)) + inst: Instance = Instance.from_2dpacklib(file_path(file)) inst.name = normalizer(inst.name) data.append((inst.name, inst.to_compact_str())) append_almost_squares_strings(data.append) # add the asquas instances data.sort() - dest_path.write_all([content for _, content in data]) + with dest_path.open_for_write() as wd: + write_lines((content for _, content in data), wd) dest_path.enforce_file() return dest_path, [thename for thename, _ in data] @@ -181,10 +182,10 @@ def make_2dpacklib_resource( and/or transforms an instance name :return: the canonical path to the and the list of instance names stored """ - dest_path: Final[Path] = Path.directory(dirname(inst_mod.__file__))\ + dest_path: Final[Path] = directory_path(dirname(inst_mod.__file__))\ .resolve_inside(INSTANCES_RESOURCE) if dest_file is None \ - else Path.path(dest_file) - with TempDir.create() as temp: + else Path(dest_file) + with temp_dir() as temp: files: Iterable[Path] = download_2dpacklib_instances( dest_dir=temp, source_urls=source_urls) return join_instances_to_compact( diff --git a/moptipyapps/binpacking2d/objectives/bin_count.py b/moptipyapps/binpacking2d/objectives/bin_count.py index 8670f2d6..e3c89183 100644 --- a/moptipyapps/binpacking2d/objectives/bin_count.py +++ b/moptipyapps/binpacking2d/objectives/bin_count.py @@ -6,7 +6,7 @@ from typing import Final from moptipy.api.objective import Objective -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.binpacking2d.instance import Instance from moptipyapps.binpacking2d.packing import IDX_BIN diff --git a/moptipyapps/binpacking2d/packing.py b/moptipyapps/binpacking2d/packing.py index 41f1f3ce..e4a03a16 100644 --- a/moptipyapps/binpacking2d/packing.py +++ b/moptipyapps/binpacking2d/packing.py @@ -6,7 +6,7 @@ from moptipy.api.component import Component from moptipy.api.logging import SECTION_RESULT_Y, SECTION_SETUP from moptipy.evaluation.log_parser import LogParser -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.binpacking2d.instance import Instance diff --git a/moptipyapps/binpacking2d/packing_result.py b/moptipyapps/binpacking2d/packing_result.py index db9aa432..f66ada61 100644 --- a/moptipyapps/binpacking2d/packing_result.py +++ b/moptipyapps/binpacking2d/packing_result.py @@ -35,19 +35,19 @@ ) from moptipy.evaluation.end_results import EndResult from moptipy.evaluation.log_parser import LogParser -from moptipy.utils.console import logger -from moptipy.utils.help import argparser from moptipy.utils.logger import CSV_SEPARATOR -from moptipy.utils.path import Path -from moptipy.utils.strings import ( - intfloatnone_to_str, - intnone_to_str, +from pycommons.ds.immutable_map import immutable_mapping +from pycommons.io.console import logger +from pycommons.io.path import Path, file_path +from pycommons.strings.string_conv import ( + int_or_none_to_str, + num_or_none_to_str, num_to_str, - str_to_intfloat, - str_to_intfloatnone, - str_to_intnone, + str_to_int_or_none, + str_to_num, + str_to_num_or_none, ) -from moptipy.utils.types import check_int_range, immutable_mapping, type_error +from pycommons.types import check_int_range, type_error from moptipyapps.binpacking2d.instance import Instance, _lower_bound_damv from moptipyapps.binpacking2d.objectives.bin_count import ( @@ -73,6 +73,7 @@ BinCountAndSmall, ) from moptipyapps.binpacking2d.packing import Packing +from moptipyapps.shared import moptipyapps_argparser #: the number of items KEY_N_ITEMS: Final[str] = "nItems" @@ -372,15 +373,15 @@ def from_single_log( # pylint: disable=W0102 called repeatedly :return: the packing result """ - file_path = Path.file(file) + the_file_path = file_path(file) end_results: Final[list[EndResult]] = [] - EndResult.from_logs(file_path, end_results.append) + EndResult.from_logs(the_file_path, end_results.append) if len(end_results) != 1: raise ValueError( - f"needs one end result record in file {file_path!r}, " + f"needs one end result record in file {the_file_path!r}, " f"but got {end_results}.") - packing = Packing.from_log(file_path) + packing = Packing.from_log(the_file_path) if not isinstance(packing, Packing): raise type_error(packing, f"packing from {file!r}", Packing) return PackingResult.from_packing_and_end_result( @@ -414,9 +415,9 @@ def to_csv(results: Iterable["PackingResult"], file: str) -> Path: :param file: the path :return: the path of the file that was written """ - path: Final[Path] = Path.path(file) + path: Final[Path] = Path(file) logger(f"Writing packing results to CSV file {path!r}.") - Path.path(os.path.dirname(path)).ensure_dir_exists() + Path(os.path.dirname(path)).ensure_dir_exists() # get a nicely sorted view on the results use_results = sorted(results) @@ -492,11 +493,11 @@ def to_csv(results: Iterable["PackingResult"], file: str) -> Path: f"{e.total_fes}{CSV_SEPARATOR}" f"{e.total_time_millis}") if needs_goal_f: - line.append(intfloatnone_to_str(e.goal_f)) + line.append(num_or_none_to_str(e.goal_f)) if needs_max_fes: - line.append(intnone_to_str(e.max_fes)) + line.append(int_or_none_to_str(e.max_fes)) if needs_max_ms: - line.append(intnone_to_str(e.max_time_millis)) + line.append(int_or_none_to_str(e.max_time_millis)) for ob in objectives: line.append(num_to_str(pr.objectives[ob]) if ob in pr.objectives else "") @@ -516,7 +517,7 @@ def from_csv(file: str, :param file: the file to read from :param collector: the collector for the results """ - path = Path.file(file) + path = file_path(file) if not callable(collector): raise type_error(collector, "collector", call=True) @@ -679,25 +680,25 @@ def from_csv(file: str, splt[idx_objective].strip(), # objective encoding, # encoding int((splt[idx_seed])[2:], 16), # rand seed - str_to_intfloat(splt[idx_best_f]), # best_f + str_to_num(splt[idx_best_f]), # best_f int(splt[idx_li_fe]), # last_improvement_fe int(splt[idx_li_ms]), # last_improvement_time_millis int(splt[idx_tt_fe]), # total_fes int(splt[idx_tt_ms]), # total_time_millis None if idx_goal_f < 0 else - str_to_intfloatnone(splt[idx_goal_f]), # goal_f + str_to_num_or_none(splt[idx_goal_f]), # goal_f None if idx_max_fes < 0 else - str_to_intnone(splt[idx_max_fes]), # max_fes + str_to_int_or_none(splt[idx_max_fes]), # max_fes None if idx_max_ms < 0 else - str_to_intnone(splt[idx_max_ms])) # max_time_millis + str_to_int_or_none(splt[idx_max_ms])) # max_time_millis the_objectives: dict[str, int | float] = { - on: str_to_intfloat(splt[idx_objectives[on]]) + on: str_to_num(splt[idx_objectives[on]]) for on in objectives} the_bin_bounds: dict[str, int] = { on: int(splt[idx_bounds[on]]) for on in bounds} the_objective_bounds: dict[str, int | float] = { - on: str_to_intfloat(splt[idxx]) + on: str_to_num(splt[idxx]) for on, idxx in idx_objective_bounds.items()} collector(PackingResult( @@ -760,17 +761,17 @@ def parse_file(self, path: str) -> bool: # Run log files to end results if executed as script if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "Convert log files for the bin packing experiment to a CSV file.", "Re-evaluate all results based on different objective functions.") parser.add_argument( "source", nargs="?", default="./results", help="the location of the experimental results, i.e., the root folder " - "under which to search for log files", type=Path.path) + "under which to search for log files", type=Path) parser.add_argument( "dest", help="the path to the end results CSV file to be created", - type=Path.path, nargs="?", default="./evaluation/end_results.txt") + type=Path, nargs="?", default="./evaluation/end_results.txt") args: Final[argparse.Namespace] = parser.parse_args() packing_results: Final[list[PackingResult]] = [] diff --git a/moptipyapps/binpacking2d/packing_space.py b/moptipyapps/binpacking2d/packing_space.py index 9791aee7..38f98bb1 100644 --- a/moptipyapps/binpacking2d/packing_space.py +++ b/moptipyapps/binpacking2d/packing_space.py @@ -19,7 +19,7 @@ import numpy as np from moptipy.api.space import Space from moptipy.utils.logger import CSV_SEPARATOR, KeyValueLogSection -from moptipy.utils.types import check_int_range, type_error +from pycommons.types import check_int_range, type_error from moptipyapps.binpacking2d.instance import ( IDX_HEIGHT, diff --git a/moptipyapps/binpacking2d/packing_statistics.py b/moptipyapps/binpacking2d/packing_statistics.py index c653d659..68a8070d 100644 --- a/moptipyapps/binpacking2d/packing_statistics.py +++ b/moptipyapps/binpacking2d/packing_statistics.py @@ -37,14 +37,14 @@ EMPTY_CSV_ROW, Statistics, ) -from moptipy.utils.console import logger -from moptipy.utils.help import argparser from moptipy.utils.logger import CSV_SEPARATOR -from moptipy.utils.path import Path from moptipy.utils.strings import ( num_to_str, ) -from moptipy.utils.types import check_int_range, immutable_mapping, type_error +from pycommons.ds.immutable_map import immutable_mapping +from pycommons.io.console import logger +from pycommons.io.path import Path +from pycommons.types import check_int_range, type_error from moptipyapps.binpacking2d.objectives.bin_count import BIN_COUNT_NAME from moptipyapps.binpacking2d.packing_result import ( @@ -56,6 +56,7 @@ KEY_N_ITEMS, PackingResult, ) +from moptipyapps.shared import moptipyapps_argparser @dataclass(frozen=True, init=False, order=False, eq=False) @@ -304,9 +305,9 @@ def to_csv(results: Iterable["PackingStatistics"], file: str) -> Path: :param file: the path :return: the path of the file that was written """ - path: Final[Path] = Path.path(file) + path: Final[Path] = Path(file) logger(f"Writing packing results to CSV file {path!r}.") - Path.path(os.path.dirname(path)).ensure_dir_exists() + Path(os.path.dirname(path)).ensure_dir_exists() # get a nicely sorted view on the statistics use_stats = sorted(results) @@ -615,7 +616,7 @@ def h(p) -> None: # Run packing-results to stat file if executed as script if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "Build an end-results statistics CSV file.", "This program computes statistics over packing results") def_src: str = "./evaluation/end_results.txt" @@ -624,9 +625,9 @@ def h(p) -> None: parser.add_argument( "source", nargs="?", default=def_src, help="either the directory with moptipy log files or the path to the " - "end-results CSV file", type=Path.path) + "end-results CSV file", type=Path) parser.add_argument( - "dest", type=Path.path, nargs="?", + "dest", type=Path, nargs="?", default="./evaluation/end_statistics.txt", help="the path to the end results statistics CSV file to be created") args: Final[argparse.Namespace] = parser.parse_args() diff --git a/moptipyapps/binpacking2d/plot_packing.py b/moptipyapps/binpacking2d/plot_packing.py index 0b354a02..26916236 100644 --- a/moptipyapps/binpacking2d/plot_packing.py +++ b/moptipyapps/binpacking2d/plot_packing.py @@ -9,7 +9,7 @@ from matplotlib.figure import Figure # type: ignore from matplotlib.patches import Rectangle # type: ignore from matplotlib.text import Text # type: ignore -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.binpacking2d.packing import ( IDX_BIN, diff --git a/moptipyapps/dynamic_control/controller.py b/moptipyapps/dynamic_control/controller.py index 4c26256d..8690c2d4 100644 --- a/moptipyapps/dynamic_control/controller.py +++ b/moptipyapps/dynamic_control/controller.py @@ -21,7 +21,7 @@ from moptipy.spaces.vectorspace import VectorSpace from moptipy.utils.logger import KeyValueLogSection from moptipy.utils.strings import sanitize_name -from moptipy.utils.types import check_to_int_range, type_error +from pycommons.types import check_to_int_range, type_error class Controller(Component): diff --git a/moptipyapps/dynamic_control/controllers/ann.py b/moptipyapps/dynamic_control/controllers/ann.py index 9cf96634..38901100 100644 --- a/moptipyapps/dynamic_control/controllers/ann.py +++ b/moptipyapps/dynamic_control/controllers/ann.py @@ -21,7 +21,7 @@ from typing import Callable, Final, Iterable, cast import numpy as np -from moptipy.utils.types import check_int_range, type_error +from pycommons.types import check_int_range, type_error from moptipyapps.dynamic_control.controller import Controller from moptipyapps.dynamic_control.controllers.codegen import CodeGenerator diff --git a/moptipyapps/dynamic_control/controllers/codegen.py b/moptipyapps/dynamic_control/controllers/codegen.py index a5b31f01..9a704233 100644 --- a/moptipyapps/dynamic_control/controllers/codegen.py +++ b/moptipyapps/dynamic_control/controllers/codegen.py @@ -5,7 +5,7 @@ import numba # type: ignore import numpy as np -from moptipy.utils.types import type_error +from pycommons.types import type_error class CodeGenerator: diff --git a/moptipyapps/dynamic_control/experiment_raw.py b/moptipyapps/dynamic_control/experiment_raw.py index fac883ca..16294c7b 100644 --- a/moptipyapps/dynamic_control/experiment_raw.py +++ b/moptipyapps/dynamic_control/experiment_raw.py @@ -20,8 +20,7 @@ from moptipy.api.execution import Execution from moptipy.api.experiment import Parallelism, run_experiment from moptipy.api.process import Process -from moptipy.utils.help import argparser -from moptipy.utils.path import Path +from pycommons.io.path import Path, directory_path from moptipyapps.dynamic_control.controllers.ann import anns from moptipyapps.dynamic_control.controllers.cubic import cubic @@ -37,6 +36,7 @@ from moptipyapps.dynamic_control.objective import FigureOfMeritLE from moptipyapps.dynamic_control.systems.lorenz import LORENZ_111 from moptipyapps.dynamic_control.systems.stuart_landau import STUART_LANDAU_111 +from moptipyapps.shared import moptipyapps_argparser def make_instances() -> Iterable[Callable[[], Instance]]: @@ -92,7 +92,7 @@ def on_completion(instance: Any, log_file: Path, process: Process) -> None: :param process: the process """ inst: Final[Instance] = cast(Instance, instance) - dest_dir: Final[Path] = Path.directory(dirname(log_file)) + dest_dir: Final[Path] = directory_path(dirname(log_file)) base_name: str = basename(log_file) base_name = base_name[:base_name.rindex(".")] result: np.ndarray = cast(np.ndarray, process.create()) @@ -108,7 +108,7 @@ def run(base_dir: str, n_runs: int = 5) -> None: :param base_dir: the base directory :param n_runs: the number of runs """ - use_dir: Final[Path] = Path.path(base_dir) + use_dir: Final[Path] = Path(base_dir) use_dir.ensure_dir_exists() instances: Final[Iterable[Callable[[], Instance]]] = make_instances() for maker in instances: @@ -129,10 +129,10 @@ def run(base_dir: str, n_runs: int = 5) -> None: # Run the experiment from the command line if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "Dynamic Control", "Run the dynamic control experiment.") parser.add_argument( "dest", help="the directory to store the experimental results under", - type=Path.path, nargs="?", default="./results/") + type=Path, nargs="?", default="./results/") args: Final[argparse.Namespace] = parser.parse_args() run(args.dest) diff --git a/moptipyapps/dynamic_control/experiment_surrogate.py b/moptipyapps/dynamic_control/experiment_surrogate.py index 550b08b0..5e85d353 100644 --- a/moptipyapps/dynamic_control/experiment_surrogate.py +++ b/moptipyapps/dynamic_control/experiment_surrogate.py @@ -29,8 +29,7 @@ from moptipy.api.experiment import Parallelism, run_experiment from moptipy.api.process import Process from moptipy.spaces.vectorspace import VectorSpace -from moptipy.utils.help import argparser -from moptipy.utils.path import Path +from pycommons.io.path import Path, directory_path from moptipyapps.dynamic_control.controllers.ann import make_ann from moptipyapps.dynamic_control.instance import Instance @@ -48,6 +47,7 @@ from moptipyapps.dynamic_control.systems.three_coupled_oscillators import ( THREE_COUPLED_OSCILLATORS, ) +from moptipyapps.shared import moptipyapps_argparser def make_instances() -> Iterable[Callable[[], SystemModel]]: @@ -145,7 +145,7 @@ def on_completion(instance: Any, log_file: Path, process: Process) -> None: :param process: the process """ inst: Final[SystemModel] = cast(SystemModel, instance) - dest_dir: Final[Path] = Path.directory(dirname(log_file)) + dest_dir: Final[Path] = directory_path(dirname(log_file)) base_name: str = basename(log_file) base_name = base_name[:base_name.rindex(".")] result: np.ndarray = cast(np.ndarray, process.create()) @@ -161,7 +161,7 @@ def run(base_dir: str, n_runs: int = 64) -> None: :param base_dir: the base directory :param n_runs: the number of runs """ - use_dir: Final[Path] = Path.path(base_dir) + use_dir: Final[Path] = Path(base_dir) use_dir.ensure_dir_exists() instances: Final[Iterable[Callable[[], SystemModel]]] = make_instances() keep_instances: Final[list[Callable[[], Instance]]] = [] @@ -223,11 +223,11 @@ def run(base_dir: str, n_runs: int = 64) -> None: # Run the experiment from the command line if __name__ == "__main__": - parser: Final[argparse.ArgumentParser] = argparser( + parser: Final[argparse.ArgumentParser] = moptipyapps_argparser( __file__, "Dynamic Control", "Run the dynamic control surrogate model experiment.") parser.add_argument( "dest", help="the directory to store the experimental results under", - type=Path.path, nargs="?", default="./results/") + type=Path, nargs="?", default="./results/") args: Final[argparse.Namespace] = parser.parse_args() run(args.dest) diff --git a/moptipyapps/dynamic_control/instance.py b/moptipyapps/dynamic_control/instance.py index b29b9d3c..ace6588e 100644 --- a/moptipyapps/dynamic_control/instance.py +++ b/moptipyapps/dynamic_control/instance.py @@ -29,9 +29,9 @@ import numpy as np from moptipy.api.component import Component from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.path import Path from moptipy.utils.strings import sanitize_name -from moptipy.utils.types import type_error +from pycommons.io.path import Path +from pycommons.types import type_error from moptipyapps.dynamic_control.controller import Controller from moptipyapps.dynamic_control.system import System diff --git a/moptipyapps/dynamic_control/objective.py b/moptipyapps/dynamic_control/objective.py index dd84af95..323f9d8c 100644 --- a/moptipyapps/dynamic_control/objective.py +++ b/moptipyapps/dynamic_control/objective.py @@ -24,7 +24,7 @@ from moptipy.api.objective import Objective from moptipy.utils.logger import KeyValueLogSection from moptipy.utils.nputils import array_to_str -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.dynamic_control.instance import Instance from moptipyapps.dynamic_control.ode import diff_from_ode, j_from_ode, run_ode diff --git a/moptipyapps/dynamic_control/results_log.py b/moptipyapps/dynamic_control/results_log.py index 71f4b7f7..8c86249b 100644 --- a/moptipyapps/dynamic_control/results_log.py +++ b/moptipyapps/dynamic_control/results_log.py @@ -5,10 +5,10 @@ from typing import Callable, Final import numpy as np -from moptipy.utils.console import logger from moptipy.utils.logger import CSV_SEPARATOR -from moptipy.utils.path import Path from moptipy.utils.strings import float_to_str +from pycommons.io.console import logger +from pycommons.io.path import Path class ResultsLog(AbstractContextManager): @@ -52,7 +52,7 @@ def __init__(self, state_dim: int, out: TextIOBase | str) -> None: """ super().__init__() if isinstance(out, str): - pp = Path.path(out) + pp = Path(out) logger(f"logging data to file {pp!r}.") out = pp.open_for_write() #: the internal output destination diff --git a/moptipyapps/dynamic_control/results_plot.py b/moptipyapps/dynamic_control/results_plot.py index 15241a55..b47f490f 100644 --- a/moptipyapps/dynamic_control/results_plot.py +++ b/moptipyapps/dynamic_control/results_plot.py @@ -20,11 +20,11 @@ from matplotlib.axes import Axes # type: ignore from matplotlib.figure import Figure # type: ignore from matplotlib.lines import Line2D # type: ignore -from moptipy.utils.console import logger -from moptipy.utils.path import Path from moptipy.utils.strings import float_to_str -from moptipy.utils.types import check_int_range from mpl_toolkits.mplot3d.art3d import Line3D # type: ignore +from pycommons.io.console import logger +from pycommons.io.path import Path +from pycommons.types import check_int_range def _str(f: float) -> str: @@ -222,7 +222,7 @@ def __init__(self, dest_file: str, sup_title: str | None, """ super().__init__() #: the destination file - self.__dest_file: Final[Path] = Path.path(dest_file) + self.__dest_file: Final[Path] = Path(dest_file) logger(f"plotting data to file {self.__dest_file!r}.") #: the state dimensions self.__state_dims: Final[int] = check_int_range( diff --git a/moptipyapps/dynamic_control/starting_points.py b/moptipyapps/dynamic_control/starting_points.py index 453facf5..f8230709 100644 --- a/moptipyapps/dynamic_control/starting_points.py +++ b/moptipyapps/dynamic_control/starting_points.py @@ -30,7 +30,7 @@ from moptipy.api.execution import Execution from moptipy.api.objective import Objective from moptipy.spaces.vectorspace import VectorSpace -from moptipy.utils.console import logger +from pycommons.io.console import logger @numba.njit(cache=True, inline="always", fastmath=True, boundscheck=False) diff --git a/moptipyapps/dynamic_control/surrogate_optimizer.py b/moptipyapps/dynamic_control/surrogate_optimizer.py index 938d5ae8..8d2d8827 100644 --- a/moptipyapps/dynamic_control/surrogate_optimizer.py +++ b/moptipyapps/dynamic_control/surrogate_optimizer.py @@ -99,9 +99,9 @@ from moptipy.spaces.vectorspace import VectorSpace from moptipy.utils.logger import KeyValueLogSection from moptipy.utils.nputils import rand_seed_generate -from moptipy.utils.path import Path -from moptipy.utils.types import check_int_range, type_error from numpy.random import Generator +from pycommons.io.path import Path +from pycommons.types import check_int_range, type_error from moptipyapps.dynamic_control.model_objective import ModelObjective from moptipyapps.dynamic_control.objective import FigureOfMerit @@ -335,7 +335,7 @@ def solve(self, process: Process) -> None: controllers_dir: Path | None = None controllers_name: str | None = None if log_dir_name is not None: - log_dir: Final[Path] = Path.path(log_dir_name) + log_dir: Final[Path] = Path(log_dir_name) log_dir.ensure_dir_exists() prefix: str = "modelTraining" model_training_dir = log_dir.resolve_inside(prefix) diff --git a/moptipyapps/dynamic_control/system.py b/moptipyapps/dynamic_control/system.py index 8d85870b..8e0104e3 100644 --- a/moptipyapps/dynamic_control/system.py +++ b/moptipyapps/dynamic_control/system.py @@ -23,8 +23,8 @@ from moptipy.api.component import Component from moptipy.utils.logger import KeyValueLogSection from moptipy.utils.nputils import array_to_str -from moptipy.utils.path import Path -from moptipy.utils.types import check_int_range, type_error +from pycommons.io.path import Path +from pycommons.types import check_int_range, type_error from moptipyapps.dynamic_control.ode import multi_run_ode from moptipyapps.dynamic_control.results_log import ResultsLog @@ -210,7 +210,7 @@ def plot_points(self, dest_dir: str, skip_if_exists: bool = True) -> Path: :return: the plotted file """ name: Final[str] = f"{self.name}_points" - dest_folder: Final[Path] = Path.path(dest_dir) + dest_folder: Final[Path] = Path(dest_dir) dest_folder.ensure_dir_exists() dest_file: Final[Path] = dest_folder.resolve_inside(f"{name}.pdf") if dest_file.ensure_file_exists() and skip_if_exists: @@ -314,7 +314,7 @@ def describe_system( :param skip_if_exists: if the file already exists :returns: the paths of the generated files """ - dest: Final[Path] = Path.path(dest_dir) + dest: Final[Path] = Path(dest_dir) dest.ensure_dir_exists() the_title = f"{self.name} system" diff --git a/moptipyapps/dynamic_control/system_model.py b/moptipyapps/dynamic_control/system_model.py index 15301571..fbc76ab2 100644 --- a/moptipyapps/dynamic_control/system_model.py +++ b/moptipyapps/dynamic_control/system_model.py @@ -60,7 +60,7 @@ from typing import Final from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.dynamic_control.controller import Controller from moptipyapps.dynamic_control.instance import Instance diff --git a/moptipyapps/dynamic_control/systems/lorenz.py b/moptipyapps/dynamic_control/systems/lorenz.py index 7048675b..672d341b 100644 --- a/moptipyapps/dynamic_control/systems/lorenz.py +++ b/moptipyapps/dynamic_control/systems/lorenz.py @@ -25,7 +25,7 @@ import numba # type: ignore import numpy as np -from moptipy.utils.types import check_int_range +from pycommons.types import check_int_range from moptipyapps.dynamic_control.starting_points import ( make_interesting_starting_points, diff --git a/moptipyapps/dynamic_control/systems/stuart_landau.py b/moptipyapps/dynamic_control/systems/stuart_landau.py index 2ebf2860..3165cfee 100644 --- a/moptipyapps/dynamic_control/systems/stuart_landau.py +++ b/moptipyapps/dynamic_control/systems/stuart_landau.py @@ -26,7 +26,7 @@ import numba # type: ignore import numpy as np -from moptipy.utils.types import check_int_range +from pycommons.types import check_int_range from moptipyapps.dynamic_control.starting_points import ( make_interesting_starting_points, diff --git a/moptipyapps/dynamic_control/systems/three_coupled_oscillators.py b/moptipyapps/dynamic_control/systems/three_coupled_oscillators.py index ec10a511..3d42cd9f 100644 --- a/moptipyapps/dynamic_control/systems/three_coupled_oscillators.py +++ b/moptipyapps/dynamic_control/systems/three_coupled_oscillators.py @@ -22,8 +22,8 @@ import numba # type: ignore import numpy as np -from moptipy.utils.path import Path -from moptipy.utils.types import check_int_range +from pycommons.io.path import Path +from pycommons.types import check_int_range from moptipyapps.dynamic_control.starting_points import ( make_interesting_starting_points, diff --git a/moptipyapps/order1d/instance.py b/moptipyapps/order1d/instance.py index 3d7e9a77..f5b23a35 100644 --- a/moptipyapps/order1d/instance.py +++ b/moptipyapps/order1d/instance.py @@ -11,7 +11,7 @@ num_to_str_for_name, sanitize_name, ) -from moptipy.utils.types import check_int_range, type_error +from pycommons.types import check_int_range, type_error from scipy.stats import rankdata # type: ignore from moptipyapps.qap.instance import Instance as QAPInstance diff --git a/moptipyapps/order1d/space.py b/moptipyapps/order1d/space.py index 02bc5e30..1787987a 100644 --- a/moptipyapps/order1d/space.py +++ b/moptipyapps/order1d/space.py @@ -25,7 +25,7 @@ from moptipy.spaces.permutations import Permutations from moptipy.utils.logger import CSV_SEPARATOR from moptipy.utils.strings import float_to_str -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.order1d.instance import ( _SUGGESTED_X_IN_0_1, diff --git a/moptipyapps/qap/instance.py b/moptipyapps/qap/instance.py index 4877cca0..e317e116 100644 --- a/moptipyapps/qap/instance.py +++ b/moptipyapps/qap/instance.py @@ -48,7 +48,7 @@ is_np_int, ) from moptipy.utils.strings import sanitize_name -from moptipy.utils.types import check_int_range, check_to_int_range, type_error +from pycommons.types import check_int_range, check_to_int_range, type_error from moptipyapps.qap.qaplib import open_resource_stream diff --git a/moptipyapps/qap/objective.py b/moptipyapps/qap/objective.py index 1f61a006..7d090b72 100644 --- a/moptipyapps/qap/objective.py +++ b/moptipyapps/qap/objective.py @@ -46,7 +46,7 @@ import numpy as np from moptipy.api.objective import Objective from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.qap.instance import Instance from moptipyapps.shared import SCOPE_INSTANCE diff --git a/moptipyapps/shared.py b/moptipyapps/shared.py index 2ba7f66c..d8bbe636 100644 --- a/moptipyapps/shared.py +++ b/moptipyapps/shared.py @@ -1,8 +1,37 @@ """Some shared variables and constants.""" +import argparse from typing import Final import moptipy.examples.jssp.instance as ins +from pycommons.io.arguments import make_argparser, make_epilog + +from moptipyapps.version import __version__ #: the instance scope SCOPE_INSTANCE: Final[str] = ins.SCOPE_INSTANCE + + +def moptipyapps_argparser(file: str, description: str, + epilog: str) -> argparse.ArgumentParser: + """ + Create an argument parser with default settings. + + :param file: the `__file__` special variable of the calling script + :param description: the description string + :param epilog: the epilogue string + :returns: the argument parser + + >>> ap = moptipyapps_argparser( + ... __file__, "This is a test program.", "This is a test.") + >>> isinstance(ap, argparse.ArgumentParser) + True + >>> "Copyright" in ap.epilog + True + """ + return make_argparser( + file, description, + make_epilog(epilog, 2023, 2024, "Thomas Weise", + url="https://thomasweise.github.io/moptipyapps", + email="tweise@hfuu.edu.cn, tweise@ustc.edu.cn"), + __version__) diff --git a/moptipyapps/tests/on_binpacking2d.py b/moptipyapps/tests/on_binpacking2d.py index 8314fdc2..3d4e0b53 100644 --- a/moptipyapps/tests/on_binpacking2d.py +++ b/moptipyapps/tests/on_binpacking2d.py @@ -15,8 +15,8 @@ from moptipy.tests.encoding import validate_encoding from moptipy.tests.objective import validate_objective from moptipy.tests.space import validate_space -from moptipy.utils.types import type_error from numpy.random import Generator, default_rng +from pycommons.types import type_error from moptipyapps.binpacking2d.encodings.ibl_encoding_1 import ( ImprovedBottomLeftEncoding1, diff --git a/moptipyapps/tests/on_tsp.py b/moptipyapps/tests/on_tsp.py index 8da357a4..2ef69a81 100644 --- a/moptipyapps/tests/on_tsp.py +++ b/moptipyapps/tests/on_tsp.py @@ -8,8 +8,8 @@ from moptipy.spaces.permutations import Permutations from moptipy.tests.algorithm import validate_algorithm from moptipy.tests.objective import validate_objective -from moptipy.utils.types import type_error from numpy.random import Generator, default_rng +from pycommons.types import type_error from moptipyapps.tsp.instance import Instance from moptipyapps.tsp.tour_length import TourLength diff --git a/moptipyapps/tsp/ea1p1_revn.py b/moptipyapps/tsp/ea1p1_revn.py index 71d0dfe9..665be7d0 100644 --- a/moptipyapps/tsp/ea1p1_revn.py +++ b/moptipyapps/tsp/ea1p1_revn.py @@ -44,8 +44,8 @@ from moptipy.api.algorithm import Algorithm from moptipy.api.process import Process from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.types import type_error from numpy.random import Generator +from pycommons.types import type_error from moptipyapps.shared import SCOPE_INSTANCE from moptipyapps.tsp.instance import Instance diff --git a/moptipyapps/tsp/fea1p1_revn.py b/moptipyapps/tsp/fea1p1_revn.py index 6a290a01..aabe3e8a 100644 --- a/moptipyapps/tsp/fea1p1_revn.py +++ b/moptipyapps/tsp/fea1p1_revn.py @@ -38,8 +38,8 @@ from moptipy.api.process import Process from moptipy.utils.logger import KeyValueLogSection from moptipy.utils.nputils import DEFAULT_INT -from moptipy.utils.types import type_error from numpy.random import Generator +from pycommons.types import type_error from moptipyapps.shared import SCOPE_INSTANCE from moptipyapps.tsp.instance import Instance diff --git a/moptipyapps/tsp/instance.py b/moptipyapps/tsp/instance.py index f78f1b6b..d15343db 100644 --- a/moptipyapps/tsp/instance.py +++ b/moptipyapps/tsp/instance.py @@ -143,9 +143,9 @@ from moptipy.api.component import Component from moptipy.utils.logger import KeyValueLogSection from moptipy.utils.nputils import int_range_to_dtype -from moptipy.utils.path import Path from moptipy.utils.strings import sanitize_name -from moptipy.utils.types import check_int_range, check_to_int_range, type_error +from pycommons.io.path import Path, file_path +from pycommons.types import check_int_range, check_to_int_range, type_error from moptipyapps.tsp.tsplib import open_resource_stream @@ -919,7 +919,7 @@ def from_file( >>> inst.name 'br17' """ - file: Final[Path] = Path.file(path) + file: Final[Path] = file_path(path) with file.open_for_read() as stream: try: return _from_stream( diff --git a/moptipyapps/tsp/known_optima.py b/moptipyapps/tsp/known_optima.py index 1af2c2f9..85c58670 100644 --- a/moptipyapps/tsp/known_optima.py +++ b/moptipyapps/tsp/known_optima.py @@ -28,8 +28,8 @@ import moptipy.utils.nputils as npu import numpy as np -from moptipy.utils.path import Path -from moptipy.utils.types import check_to_int_range, type_error +from pycommons.io.path import Path, file_path +from pycommons.types import check_to_int_range, type_error from moptipyapps.tsp.tsplib import open_resource_stream @@ -88,7 +88,7 @@ def opt_tour_from_file(path: str) -> np.ndarray: :param path: the path to the file :return: the tour """ - file: Final[Path] = Path.file(path) + file: Final[Path] = file_path(path) with file.open_for_read() as stream: try: return _from_stream(cast(TextIO, stream)) diff --git a/moptipyapps/tsp/tour_length.py b/moptipyapps/tsp/tour_length.py index 7537c199..30c50511 100644 --- a/moptipyapps/tsp/tour_length.py +++ b/moptipyapps/tsp/tour_length.py @@ -74,7 +74,7 @@ import numpy as np from moptipy.api.objective import Objective from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.shared import SCOPE_INSTANCE from moptipyapps.tsp.instance import Instance diff --git a/moptipyapps/ttp/errors.py b/moptipyapps/ttp/errors.py index a4580587..204b103c 100644 --- a/moptipyapps/ttp/errors.py +++ b/moptipyapps/ttp/errors.py @@ -22,7 +22,7 @@ import numpy as np from moptipy.api.objective import Objective from moptipy.utils.nputils import int_range_to_dtype -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.ttp.game_plan import GamePlan from moptipyapps.ttp.instance import Instance diff --git a/moptipyapps/ttp/game_encoding.py b/moptipyapps/ttp/game_encoding.py index 7e6a7308..b79f017a 100644 --- a/moptipyapps/ttp/game_encoding.py +++ b/moptipyapps/ttp/game_encoding.py @@ -38,7 +38,7 @@ import numpy as np from moptipy.api.encoding import Encoding from moptipy.spaces.permutations import Permutations -from moptipy.utils.types import check_int_range +from pycommons.types import check_int_range from moptipyapps.ttp.instance import Instance diff --git a/moptipyapps/ttp/game_plan.py b/moptipyapps/ttp/game_plan.py index 9cf9d3ed..e22e75de 100644 --- a/moptipyapps/ttp/game_plan.py +++ b/moptipyapps/ttp/game_plan.py @@ -38,7 +38,7 @@ import numpy as np from moptipy.api.component import Component from moptipy.utils.logger import CSV_SEPARATOR -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.ttp.instance import Instance diff --git a/moptipyapps/ttp/game_plan_space.py b/moptipyapps/ttp/game_plan_space.py index 4e0868f1..a87d4039 100644 --- a/moptipyapps/ttp/game_plan_space.py +++ b/moptipyapps/ttp/game_plan_space.py @@ -11,7 +11,7 @@ import numpy as np from moptipy.api.space import Space from moptipy.utils.logger import CSV_SEPARATOR, KeyValueLogSection -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.shared import SCOPE_INSTANCE from moptipyapps.ttp.game_plan import GamePlan diff --git a/moptipyapps/ttp/instance.py b/moptipyapps/ttp/instance.py index e6103554..d6dd7b2b 100644 --- a/moptipyapps/ttp/instance.py +++ b/moptipyapps/ttp/instance.py @@ -41,9 +41,9 @@ from defusedxml import ElementTree # type: ignore from moptipy.utils.logger import KeyValueLogSection from moptipy.utils.nputils import DEFAULT_INT, int_range_to_dtype -from moptipy.utils.path import Path from moptipy.utils.strings import sanitize_name -from moptipy.utils.types import check_int_range, check_to_int_range, type_error +from pycommons.io.path import Path, file_path +from pycommons.types import check_int_range, check_to_int_range, type_error from moptipyapps.tsp.instance import Instance as TSPInstance from moptipyapps.ttp.robinx import open_resource_stream @@ -395,7 +395,7 @@ def from_file(path: str, lower_bound_getter: Callable[[ >>> inst.name 'con20' """ - file: Final[Path] = Path.file(path) + file: Final[Path] = file_path(path) with file.open_for_read() as stream: try: return _from_stream(cast(TextIO, stream)) diff --git a/moptipyapps/ttp/plan_length.py b/moptipyapps/ttp/plan_length.py index 4a2daa70..7eea1518 100644 --- a/moptipyapps/ttp/plan_length.py +++ b/moptipyapps/ttp/plan_length.py @@ -56,7 +56,7 @@ import numpy as np from moptipy.api.objective import Objective from moptipy.utils.logger import KeyValueLogSection -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.ttp.game_plan import GamePlan from moptipyapps.ttp.instance import Instance diff --git a/moptipyapps/version.py b/moptipyapps/version.py index d4250b13..ead9f2d0 100644 --- a/moptipyapps/version.py +++ b/moptipyapps/version.py @@ -1,4 +1,4 @@ """An internal file with the version of the `moptipyapps` package.""" from typing import Final -__version__: Final[str] = "0.8.43" +__version__: Final[str] = "0.8.44" diff --git a/requirements-dev.txt b/requirements-dev.txt index 823b5502..f732aa83 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -31,6 +31,8 @@ # Here we only list the top-level dependencies that are actually used in the # `moptipyapps` build process. # +# the common tools package +pycommons[dev] == 0.8.11 # We need pytest to run the unit tests. # Unit tests test components of our package, e.g., functions or objects, and diff --git a/requirements.txt b/requirements.txt index c39ac971..9a6315c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,10 @@ # `moptipy` provides the basic optimization infrastructure and the spaces and # tools that we use for optimization. -moptipy == 0.9.103 +moptipy == 0.9.104 + +# the common tools package +pycommons == 0.8.11 # `numpy` is needed for its efficient data structures. numpy == 1.26.1 diff --git a/setup.cfg b/setup.cfg index 276bafea..af624ff1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,10 +41,11 @@ include_package_data = True install_requires = certifi >= 2023.7.22 defusedxml >= 0.7.1 - moptipy >= 0.9.103 + moptipy >= 0.9.104 numpy >= 1.26.1 numba >= 0.58.1 - matplotlib >= 3.8.0 + matplotlib >= 3.8 + pycommons >= 0.8.11 scipy >= 1.11.3 urllib3 >= 1.26.18 packages = find: diff --git a/setup.py b/setup.py index e2f402cb..e4984339 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ """The setup and installation script.""" -import re +from re import compile as re_compile +from re import sub as re_sub +from typing import Final, Pattern from setuptools import setup @@ -13,28 +15,33 @@ # that we need to fix all references following the pattern `[xxx](#12-hello)` # to `[xxx]({docu_url#hello)`, where `docu_url` is the url of our # documentation. We do this with a regular expression `regex_search`. -new_lines = [] +new_lines: Final[list[str]] = [] in_code: bool = False # we only process non-code lines + +# the base url where the documentation will land +doc_url: Final[str] = "https://thomasweise.github.io/moptipyapps" +# the url of our repository +repo_url: Final[str] = "https://github.com/thomasWeise/moptipyapps" + # detects strings of the form [xyz](#123-bla) and gives \1=xyz and \2=bla -regex_search = re.compile("(\\[.+?])\\(#\\d+-(.+?)\\)") -regex_repl: str = \ - "\\1(https://thomasweise.github.io/moptipyapps/index.html#\\2)" -license_old: str = \ - "https://github.com/thomasWeise/moptipyapps/blob/main/LICENSE" -license_new: str = "https://thomasweise.github.io/moptipyapps/LICENSE.html" - -for line in old_lines: - line = line.rstrip() +regex_search: Final[Pattern] = re_compile("(\\[.+?])\\(#\\d+-(.+?)\\)") +regex_repl: Final[str] = f"\\1({doc_url}#\\2)" + +# other replacements +license_old: Final[str] = f"{repo_url}/blob/main/LICENSE" +license_new: Final[str] = f"{doc_url}/LICENSE.html" + +for full_line in old_lines: + line: str = str.rstrip(full_line) if in_code: if line.startswith("```"): in_code = False # toggle to non-code - else: - if line.startswith("```"): - in_code = True # toggle to code - else: # fix all internal urls - # replace links of the form "#12-bla" to "#bla" - line = re.sub(regex_search, regex_repl, line) - line = line.replace(license_old, license_new) + elif line.startswith("```"): + in_code = True # toggle to code + else: # fix all internal urls + # replace links of the form "#12-bla" to "#bla" + line = re_sub(regex_search, regex_repl, line) + line = str.replace(line, license_old, license_new) new_lines.append(line) # Now we can use the code in the setup. diff --git a/tests/binpacking2d/test_binpacking2d_experiment.py b/tests/binpacking2d/test_binpacking2d_experiment.py index c65eb4cb..3c8b90ff 100644 --- a/tests/binpacking2d/test_binpacking2d_experiment.py +++ b/tests/binpacking2d/test_binpacking2d_experiment.py @@ -16,8 +16,8 @@ Op1Swap2OrFlip, ) from moptipy.spaces.signed_permutations import SignedPermutations -from moptipy.utils.path import Path -from moptipy.utils.temp import TempDir +from pycommons.io.path import Path +from pycommons.io.temp import temp_dir from moptipyapps.binpacking2d.encodings.ibl_encoding_1 import ( ImprovedBottomLeftEncoding1, @@ -188,7 +188,7 @@ def __evaluate(results: Path, evaluation: Path) -> None: def test_experiment_empty_1() -> None: """Test the bin packing with BinCountAndLastEmpty and encoding 1.""" - with TempDir.create() as td: + with temp_dir() as td: results = td.resolve_inside("results") results.ensure_dir_exists() evaluation = td.resolve_inside("evaluation") @@ -205,7 +205,7 @@ def test_experiment_empty_1() -> None: def test_experiment_empty_2() -> None: """Test the bin packing with BinCountAndLastEmpty and encoding 2.""" - with TempDir.create() as td: + with temp_dir() as td: results = td.resolve_inside("results") results.ensure_dir_exists() evaluation = td.resolve_inside("evaluation") @@ -222,7 +222,7 @@ def test_experiment_empty_2() -> None: def test_experiment_small_1() -> None: """Test the bin packing with BinCountAndLastSmall and encoding 1.""" - with TempDir.create() as td: + with temp_dir() as td: results = td.resolve_inside("results") results.ensure_dir_exists() evaluation = td.resolve_inside("evaluation") @@ -239,7 +239,7 @@ def test_experiment_small_1() -> None: def test_experiment_small_2() -> None: """Test the bin packing with BinCountAndLastSmall and encoding 2.""" - with TempDir.create() as td: + with temp_dir() as td: results = td.resolve_inside("results") results.ensure_dir_exists() evaluation = td.resolve_inside("evaluation") diff --git a/tests/binpacking2d/test_binpacking2d_instance.py b/tests/binpacking2d/test_binpacking2d_instance.py index e7c4293d..57bcce3e 100644 --- a/tests/binpacking2d/test_binpacking2d_instance.py +++ b/tests/binpacking2d/test_binpacking2d_instance.py @@ -1,7 +1,7 @@ """Test loading and validity of bin packing 2D instances.""" -from moptipy.utils.types import type_error +from pycommons.types import type_error from moptipyapps.binpacking2d.instance import Instance diff --git a/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py b/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py index b4c74433..d919aaff 100644 --- a/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py +++ b/tests/binpacking2d/test_binpacking2d_packing_result_and_statistics.py @@ -14,8 +14,8 @@ Op1Swap2OrFlip, ) from moptipy.spaces.signed_permutations import SignedPermutations -from moptipy.utils.temp import TempDir, TempFile from numpy.random import Generator, default_rng +from pycommons.io.temp import temp_dir, temp_file from moptipyapps.binpacking2d.encodings.ibl_encoding_2 import ( ImprovedBottomLeftEncoding2, @@ -89,7 +89,7 @@ def test_packing_results_experiment() -> None: n_runs: Final[int] = int(random.integers(2, 7)) - with TempDir.create() as td: + with temp_dir() as td: run_experiment(base_dir=td, instances=instance_factories, setups=algorithms, @@ -109,7 +109,7 @@ def test_packing_results_experiment() -> None: all_algorithms.add(res.end_result.algorithm) assert len(all_algorithms) == len(algorithms) assert len(all_objectives) == 2 - with TempFile.create() as tf: + with temp_file() as tf: PackingResult.to_csv(results_1, tf) PackingResult.from_csv(tf, results_2.append) assert len(results_2) == len(results_1) @@ -122,5 +122,5 @@ def test_packing_results_experiment() -> None: end_stats_2: Final[list[PackingStatistics]] = [] PackingStatistics.from_packing_results(results_2, end_stats_2.append) assert len(end_stats_1) == len(end_stats_2) - with TempFile.create() as tf2: + with temp_file() as tf2: PackingStatistics.to_csv(end_stats_1, tf2) diff --git a/tests/binpacking2d/test_make_instances.py b/tests/binpacking2d/test_make_instances.py index cd416275..bd1b0ca7 100644 --- a/tests/binpacking2d/test_make_instances.py +++ b/tests/binpacking2d/test_make_instances.py @@ -1,6 +1,6 @@ """Test that downloading and making the instances is consistent.""" -from moptipy.utils.temp import TempFile +from pycommons.io.temp import temp_file from moptipyapps.binpacking2d.instance import Instance from moptipyapps.binpacking2d.make_instances import ( @@ -12,10 +12,10 @@ def test_make_instances() -> None: """Test that the instance resource is the same as the current download.""" insts = list(Instance.list_resources()) - with TempFile.create() as tf: + with temp_file() as tf: assert len(list(make_2dpacklib_resource(dest_file=tf)[1])) \ == len(insts) - for i, row in enumerate(tf.read_all_list()): + for i, row in enumerate(str.splitlines(tf.read_all_str())): i1 = Instance.from_compact_str(row) i2 = Instance.from_resource(insts[i]) assert i1.name == i2.name diff --git a/tests/dynamic_control/test_experiment_raw.py b/tests/dynamic_control/test_experiment_raw.py index dc67f86e..83eda7bf 100644 --- a/tests/dynamic_control/test_experiment_raw.py +++ b/tests/dynamic_control/test_experiment_raw.py @@ -6,8 +6,8 @@ from moptipy.api.experiment import run_experiment from moptipy.evaluation.end_results import EndResult from moptipy.utils.nputils import rand_seeds_from_str -from moptipy.utils.temp import TempDir from numpy.random import Generator, default_rng +from pycommons.io.temp import temp_dir from moptipyapps.dynamic_control.experiment_raw import cmaes, make_instances from moptipyapps.dynamic_control.instance import Instance @@ -63,7 +63,7 @@ def test_experiment_raw(random: Generator = default_rng()) -> None: insts: list[Callable[[], Instance]] = __make_instances() insts = [insts[random.integers(len(insts))]] - with TempDir.create() as use_dir: + with temp_dir() as use_dir: er.clear() run_experiment(base_dir=use_dir, instances=insts, diff --git a/tests/dynamic_control/test_experiment_surrogate.py b/tests/dynamic_control/test_experiment_surrogate.py index fd9a8feb..42738ba5 100644 --- a/tests/dynamic_control/test_experiment_surrogate.py +++ b/tests/dynamic_control/test_experiment_surrogate.py @@ -5,8 +5,8 @@ from moptipy.api.execution import Execution from moptipy.api.experiment import run_experiment from moptipy.evaluation.end_results import EndResult -from moptipy.utils.temp import TempDir from numpy.random import Generator, default_rng +from pycommons.io.temp import temp_dir from moptipyapps.dynamic_control.experiment_surrogate import ( base_setup, @@ -76,7 +76,7 @@ def test_experiment_surrogate(random: Generator = default_rng()) -> None: insts: list[Callable[[], SystemModel]] = list(__make_instances(random)) insts = [insts[random.integers(len(insts))]] - with TempDir.create() as use_dir: + with temp_dir() as use_dir: er.clear() run_experiment(base_dir=use_dir, instances=insts, diff --git a/tests/test_examples_in_examples_directory.py b/tests/test_examples_in_examples_directory.py index f7a24f2e..13f2d916 100644 --- a/tests/test_examples_in_examples_directory.py +++ b/tests/test_examples_in_examples_directory.py @@ -1,36 +1,12 @@ """Test all the example code in the project's examples directory.""" -import os.path -from typing import Final -from moptipy.utils.console import logger -from moptipy.utils.path import Path -from moptipy.utils.temp import TempDir -from numpy.random import default_rng + +from pycommons.dev.tests.examples_in_dir import check_examples_in_dir +from pycommons.io.path import file_path def test_examples_in_examples_directory() -> None: """Test all the examples in the examples directory.""" # First, we resolve the directories - base_dir = Path.directory(os.path.join(os.path.dirname(__file__), "../")) - examples_dir = Path.directory(base_dir.resolve_inside("examples")) - logger( - f"executing all examples from examples directory {examples_dir!r}.") - - wd: Final[str] = os.getcwd() # get current working directory - with TempDir.create() as td: # create temporary working directory - logger(f"using temp directory {td!r}.") - os.chdir(td) # set it as working directory - files: list[str] = os.listdir(examples_dir) - logger(f"got {len(files)} potential files") - default_rng().shuffle(files) # shuffle the order for randomness - for name in files: # find all files in examples - if name.endswith(".py"): # if it is a python file - file: Path = Path.file(examples_dir.resolve_inside(name)) - logger(f"now compiling file {file!r}.") - code = compile( # noqa # nosec - file.read_all_str(), file, mode="exec") # noqa # nosec - logger(f"now executing file {file!r}.") - exec(code, {}) # execute file # noqa # nosec - logger(f"successfully executed example {file!r}.") - os.chdir(wd) # go back to current original directory - logger("finished executing all examples from README.md.") + check_examples_in_dir(file_path(__file__).up(2).resolve_inside( + "examples")) diff --git a/tests/test_links_in_documentation.py b/tests/test_links_in_documentation.py index 3e5c589a..b8b3f284 100644 --- a/tests/test_links_in_documentation.py +++ b/tests/test_links_in_documentation.py @@ -1,445 +1,25 @@ """Test all the links in the project's *.md files.""" -import os.path -from os import environ -from random import choice -from time import sleep -from typing import Final - -# noinspection PyPackageRequirements -import certifi - # noinspection PyPackageRequirements -import urllib3 -from moptipy.utils.console import logger -from moptipy.utils.path import Path -from moptipy.utils.strings import replace_all - -# noinspection PyPackageRequirements -from urllib3.util.url import Url, parse_url - -#: The hosts that somtimes are unreachable from my local machine. -#: When the test is executed in a GitHub workflow, all hosts should be -#: reachable. -SOMETIMES_UNREACHABLE_HOSTS: Final[set[str]] = \ - () if "GITHUB_JOB" in environ else \ - {"github.com", "img.shields.io", "pypi.org", "docs.python.org"} - - -def __ve(msg: str, text: str, idx: int) -> ValueError: - """ - Raise a value error for the given text piece. - - :param msg: the message - :param text: the string - :param idx: the index - :returns: a value error ready to be raised - """ - piece = text[max(0, idx - 32):min(len(text), idx + 64)].strip() - return ValueError(f"{msg}: '...{piece}...'") - - -#: The headers to use for the HTTP requests. -#: It seems that some websites may throttle requests. -#: Maybe by using different headers, we can escape this. -__HEADERS: Final[tuple[dict[str, str], ...]] = tuple([ - {"User-Agent": ua} for ua in [ - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101" - " Firefox/106.0", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like " - "Gecko) Chrome/109.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, " - "like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.55", - "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 " - "Version/12.16.2", - "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) " - "like Gecko", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14" - " (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", - "Mozilla/5.0 (PLAYSTATION 3; 3.55)", - ]]) - - -def __needs_body(base_url: str) -> bool: - """ - Check whether we need the body of the given url. - - If the complete body of the document needs to be downloaded, this function - returns `True`. This is the case, for example, if we are talking about - html documents. In this case, we need to (later) scan for internal - references, i.e., for stuff like `id="..."` attributes. However, if the - url does not point to an HTML document, maybe a PDF, then we do not need - the whole body and return `False`. In the latter case, it is sufficient to - do a `HEAD` HTTP request, in the former case we need a full `GET`. - - :param base_url: the url string - :returns: `True` if the body is needed, `False` otherwise - """ - return base_url.endswith((".html", ".htm", "/")) - - -def __check(url: str, valid_urls: dict[str, str | None], - http: urllib3.PoolManager = urllib3.PoolManager( - cert_reqs="CERT_REQUIRED", ca_certs=certifi.where())) -> None: - """ - Check if an url can be reached. - - :param url: str - :param valid_urls: the set of valid urls - :param http: the pool manager - """ - if (url != url.strip()) or (len(url) < 3): - raise ValueError(f"invalid url {url!r}") - if url in valid_urls: - return - if url.startswith("mailto:"): - return - if not url.startswith("http"): - raise ValueError(f"invalid url {url!r}") - - base_url: str = url - selector: str | None = None - needs_body: bool - i = url.find("#") - if i >= 0: - base_url = url[:i] - needs_body = __needs_body(base_url) - if not needs_body: - raise ValueError(f"invalid url: {url!r}") - - selector = url[i + 1:] - if (len(selector) <= 0) or (len(base_url) <= 0) \ - or len(selector.strip()) != len(selector) \ - or (len(base_url.strip()) != len(base_url)): - raise ValueError(f"invalid url: {url!r}") - - if base_url in valid_urls: - body = valid_urls[base_url] - if body is None: - raise ValueError( - f"no body for {url!r} with base {base_url!r}??") - for qt in ("", "'", '"'): - if f"id={qt}{selector}{qt}" in body: - return - raise ValueError( - f"did not find id={selector!r} of {url!r} in body " - f"of {base_url!r}: {body!r}") - else: - needs_body = __needs_body(base_url) - - code: int - body: str | None - method = "GET" if needs_body or ( - "coral.ise.lehigh.edu" in base_url) else "HEAD" - error: Exception | None = None - response = None -# Sometimes, access to the URLs on GitHub fails. -# I think they probably throttle access from here. -# Therefore, we first do a request with 5s timeout and 0 retries. -# If that fails, we wait 2 seconds and try with timeout 8 and 3 retries. -# If that fails, we wait for 5s, then try with timeout 30 and 3 retries. -# If that fails too, we assume that the URL is really incorrect, which rarely -# should not be the case (justifying the many retries). - toggle: bool = True - try: - for sltrt in [(0, 0, 5), (2, 3, 8), (5, 3, 30)]: - sleep_time, retries, timeout = sltrt - if sleep_time > 0: - sleep(sleep_time) - header: dict[str, str] = choice(__HEADERS) - try: - response = http.request( - method, base_url, timeout=timeout, redirect=True, - retries=retries, headers=header) - error = None - break - except Exception as be: - method = "GET" if needs_body or toggle else "HEAD" - toggle = not toggle - logger(f"sleep={sleep_time}, retries={retries}, " - f"timeout={timeout}, error={str(be)!r}, and " - f"header={header!r} for {base_url!r}.") - if error is not None: - bz = be - while True: - if bz.__cause__ is None: - bz.__cause__ = error - break - bz = bz.__cause__ - error = be - if error is not None: - raise error # noqa - if response is None: - raise ValueError(f"no response from url={base_url!r}?") # noqa - code = response.status - body = response.data.decode("utf-8") if needs_body else None - except Exception as be: - # sometimes, I cannot reach github from here... - parsed: Final[Url] = parse_url(url) - host: Final[str | None] = parsed.hostname - if host is None: - raise ValueError(f"url {url!r} has None as host??") from be - if host in SOMETIMES_UNREACHABLE_HOSTS: # sometimes not reachable! - return - raise ValueError(f"invalid url {url!r}.") from be - - logger(f"checked url {url!r} got code {code} for method {method!r} and " - f"{0 if body is None else len(body)} chars.") - if code != 200: - raise ValueError(f"url {url!r} returns code {code}.") - - if selector is not None: - for qt in ("", "'", '"'): - if f"id={qt}{selector}{qt}" in body: - return - raise ValueError( - f"did not find id={selector!r} of {url!r} in body " - f"of {base_url!r}: {body!r}") - - if needs_body and (body is None): - raise ValueError(f"huh? body for {url!r} / {base_url!r} is None?") - - valid_urls[base_url] = body - if url != base_url: - valid_urls[url] = None - - -def check_links_in_file(file: str) -> None: - """ - Test all the links in the README.md file. - - :param file: the file to check - """ - # First, we load the file as a single string - base_dir = Path.directory(os.path.join(os.path.dirname(__file__), "../")) - readme = Path.file(base_dir.resolve_inside(file)) - logger(f"testing all links from the {file!r} file {readme!r}.") - text = readme.read_all_str() - logger(f"got {len(text)} characters.") - if len(text) <= 0: - raise ValueError(f"{file!r} file at {readme!r} is empty?") - del readme - - # remove all code blocks - start: int = -1 - lines: Final[list[str]] = [] - while True: - start += 1 - i = text.find("\n```", start) - if i < start: - lines.append(text[start:].strip()) - break - j = text.find("\n```", i + 1) - if j < i: - raise __ve("Multi-line code start without end", text, i) - k = text.find("\n", j + 1) - if k < j: - raise __ve("Code end without newline", text, i) - lines.append(text[start:i].strip()) - start = k - - text = "\n".join(lines).strip() - lines.clear() - - # these are all urls that have been verified - valid_urls: Final[dict[str, str | None]] = {} - - # build the map of local reference marks - start = -1 - while True: - start += 1 - i = text.find("\n#", start) - if i < start: - break - j = text.find(" ", i + 1) - if j < i: - raise __ve("Headline without space after #", text, i) - k = text.find("\n", j + 1) - if k < j: - raise __ve("Headline without end", text, i) - rid: str = text[j:k].strip().replace(" ", "-") - for ch in ".:,()`/": - rid = rid.replace(ch, "") - rid = replace_all("--", "-", rid).lower() - if (len(rid) <= 2) or ((rid[0] not in "123456789") - and (start > 0)) or ("-" not in rid): - raise __ve(f"invalid id {rid!r}", text, i) - valid_urls[f"#{rid}"] = None - start = k - - # remove all inline code - start = -1 - while True: - start += 1 - i = text.find("`", start) - if i < start: - lines.append(text[start:].strip()) - break - j = text.find("`", i + 1) - if j < i: - raise __ve("Multi-line code start without end", text, i) - lines.append(text[start:i].strip()) - start = j - text = "\n".join(lines).strip() - lines.clear() - - logger("now checking '![...]()' style urls") - - # now gather the links to images and remove them - start = -1 - lines.clear() - while True: - start += 1 - i = text.find("![", start) - if i < start: - lines.append(text[start:]) - break - j = text.find("]", i + 1) - if j <= i: - break - if "\n" in text[i:j]: - start = i - j += 1 - if text[j] != "(": - raise __ve("invalid image sequence", text, i) - k = text.find(")", j + 1) - if k <= j: - raise __ve("no closing gap for image sequence", text, i) - - __check(text[j + 1:k], valid_urls) - - lines.append(text[start:i]) - start = k - - text = "\n".join(lines) - lines.clear() - - logger("now checking '[...]()' style urls") - - # now gather the links and remove them - start = -1 - lines.clear() - while True: - start += 1 - i = text.find("[", start) - if i < start: - lines.append(text[start:]) - break - j = text.find("]", i + 1) - if j <= i: - break - if "\n" in text[i:j]: - lines.append(text[start:i]) - start = i - continue - j += 1 - if text[j] != "(": - raise __ve("invalid link", text, i) - k = text.find(")", j + 1) - if k <= j: - raise __ve("no closing gap for link", text, i) - - __check(text[j + 1:k], valid_urls) - - lines.append(text[start:i]) - start = k - - text = "\n".join(lines) - lines.clear() - - logger("now checking ' href=' style urls") - - # now gather the href links and remove them - for quot in "'\"": - start = -1 - lines.clear() - while True: - start += 1 - start_str = f" href={quot}" - i = text.find(start_str, start) - if i < start: - lines.append(text[start:]) - break - j = text.find(quot, i + len(start_str)) - if j <= i: - break - if "\n" in text[i:j]: - lines.append(text[start:i]) - start = i - continue - __check(text[i + len(start_str):j], valid_urls) - - lines.append(text[start:i]) - start = j - - text = "\n".join(lines) - lines.clear() - - logger("now checking ' src=' style urls") - # now gather the image links and remove them - for quot in "'\"": - start = -1 - lines.clear() - while True: - start += 1 - start_str = f" src={quot}" - i = text.find(start_str, start) - if i < start: - lines.append(text[start:]) - break - j = text.find(quot, i + len(start_str)) - if j <= i: - break - if "\n" in text[i:j]: - lines.append(text[start:i]) - start = i - continue - __check(text[i + len(start_str):j], valid_urls) - - lines.append(text[start:i]) - start = j - - text = "\n".join(lines) - lines.clear() - - # now checking <...>-style URLs - start = -1 - lines.clear() - while True: - start += 1 - i = text.find("", i + 1) - if j <= i: - break - if "\n" in text[i:j]: - lines.append(text[start:i]) - start = i - continue - __check(text[i + 1:j], valid_urls) - - lines.append(text[start:i]) - start = j - - logger(f"finished testing all links from {file!r}.") +from pycommons.dev.tests.links_in_md import check_links_in_md +from pycommons.io.path import file_path def test_all_links_in_readme_md() -> None: """Test all the links in the README.md file.""" - check_links_in_file("README.md") + check_links_in_md(file_path(__file__).up(2).resolve_inside("README.md")) def test_all_links_in_contributing_md() -> None: """Test all the links in the CONTRIBUTING.md file.""" - check_links_in_file("CONTRIBUTING.md") + check_links_in_md(file_path(__file__).up(2).resolve_inside( + "CONTRIBUTING.md")) def test_all_links_in_security_md() -> None: """Test all the links in the SECURITY.md file.""" - check_links_in_file("SECURITY.md") + check_links_in_md(file_path(__file__).up(2).resolve_inside("SECURITY.md")) def test_all_links_in_license() -> None: """Test all the links in the LICENSE file.""" - check_links_in_file("LICENSE") + check_links_in_md(file_path(__file__).up(2).resolve_inside("LICENSE")) diff --git a/tests/ttp/test_game_encoding.py b/tests/ttp/test_game_encoding.py index 536f69f7..db209e38 100644 --- a/tests/ttp/test_game_encoding.py +++ b/tests/ttp/test_game_encoding.py @@ -6,8 +6,8 @@ import numpy as np from moptipy.spaces.permutations import Permutations from moptipy.utils.nputils import int_range_to_dtype -from moptipy.utils.types import type_error from numpy.random import Generator, default_rng +from pycommons.types import type_error from moptipyapps.ttp.game_encoding import ( map_games,