From d8cd003d55ed116c803020bd2d089185a853a0b2 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Sat, 11 Jan 2025 10:28:15 +0100 Subject: [PATCH 1/7] Use PEP-723 inline dependency spec for scripts in utils/ --- .github/workflows/test-install.yml | 5 +--- .pre-commit-config.yaml | 2 +- utils/__init__.py | 9 ------- utils/dependency_management.py | 41 ++++++++++++++++++++++++------ utils/requirements.txt | 5 ---- utils/validate_consistency.py | 6 +++++ 6 files changed, 41 insertions(+), 27 deletions(-) delete mode 100644 utils/__init__.py delete mode 100644 utils/requirements.txt diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index f9e050dbe2..5ed7bf0339 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -42,11 +42,8 @@ jobs: with: version: latest - - name: Install utils/ dependencies - run: uv pip install --system -r utils/requirements.txt - - name: Validate conda environment file - run: python ./utils/dependency_management.py validate-environment-yml + run: uv run --script ./utils/dependency_management.py validate-environment-yml create-conda-environment: # Verify that we can create a valid conda environment from the environment.yml file. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1dc6e7d48..27a5ccf790 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -217,7 +217,7 @@ repos: )$ - id: verdi-autodocs - name: Automatically generating verdi docs + name: Generate verdi docs entry: python ./utils/validate_consistency.py verdi-autodocs language: system pass_filenames: false diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index f749d294e7..0000000000 --- a/utils/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -"""Utilities for installation and pre-commit checks & more.""" diff --git a/utils/dependency_management.py b/utils/dependency_management.py index aa15a6e3db..fe1218afa9 100755 --- a/utils/dependency_management.py +++ b/utils/dependency_management.py @@ -9,6 +9,21 @@ ########################################################################### """Utility CLI to manage dependencies for aiida-core.""" +# To run this script, we recommend using uv that will +# automatically install required dependencies based on specifications below. E.g. +# `uv run dependency-management.py --help` +# +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "click==7.1", +# "packaging==23.1", +# "pyyaml==6.0.1", +# "requests==2.25.1", +# "tomli==2.0.0", +# ] +# /// + import os import re import subprocess @@ -17,15 +32,11 @@ from pathlib import Path import click -import requests -import tomli -import yaml from packaging.requirements import Requirement -from packaging.version import parse ROOT = Path(__file__).resolve().parent.parent # repository root -SETUPTOOLS_CONDA_MAPPINGS = { +PYPI_CONDA_MAPPINGS = { 'graphviz': 'python-graphviz', 'docstring-parser': 'docstring_parser', } @@ -41,10 +52,17 @@ class DependencySpecificationError(click.ClickException): def _load_pyproject(): """Load the setup configuration from the 'pyproject.toml' file.""" + # TODO: Require tomli only for Python <3.11 + # if sys.version < 3.11: + # import tomli as tomllib + # else + # import tomllib + import tomli as tomllib + try: with open(ROOT / 'pyproject.toml', 'rb') as handle: - return tomli.load(handle) - except tomli.TOMLDecodeError as error: + return tomllib.load(handle) + except tomllib.TOMLDecodeError as error: raise DependencySpecificationError(f"Error while parsing 'pyproject.toml' file: {error}") except FileNotFoundError: raise DependencySpecificationError("The 'pyproject.toml' file is missing!") @@ -52,6 +70,8 @@ def _load_pyproject(): def _load_environment_yml(): """Load the conda environment specification from the 'environment.yml' file.""" + import yaml + try: with open(ROOT / 'environment.yml', encoding='utf8') as file: return yaml.load(file, Loader=yaml.SafeLoader) @@ -67,7 +87,7 @@ def _setuptools_to_conda(req): In case that the same underlying dependency is listed under different names on PyPI and conda-forge. """ - for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items(): + for pattern, replacement in PYPI_CONDA_MAPPINGS.items(): if re.match(pattern, str(req)): req = Requirement(re.sub(pattern, replacement, str(req))) break @@ -117,6 +137,8 @@ def cli(): @cli.command('generate-environment-yml') def generate_environment_yml(): """Generate 'environment.yml' file.""" + import yaml + # needed for ordered dict, see https://stackoverflow.com/a/52621703 yaml.add_representer( OrderedDict, @@ -295,6 +317,9 @@ def identify_outdated(extras, pre_releases): This function can thus be used to identify dependencies where the specification must be loosened. """ + import requests + from packaging.version import parse + # Read the requirements from 'pyproject.toml'' pyproject = _load_pyproject() diff --git a/utils/requirements.txt b/utils/requirements.txt deleted file mode 100644 index 7fb8a640c2..0000000000 --- a/utils/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -click==7.1.2 -packaging==23.1 -pyyaml==6.0.1 -requests==2.25.1 -tomli==2.0.0 diff --git a/utils/validate_consistency.py b/utils/validate_consistency.py index 98796d3d1a..0af2d50fcd 100644 --- a/utils/validate_consistency.py +++ b/utils/validate_consistency.py @@ -8,6 +8,12 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Autogenerate verdi CLI documentation from click.""" +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "click", +# ] +# /// import os From 3681471315956c9891f4dd9eae6d07b2a217e323 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 24 Jan 2025 17:54:45 +0000 Subject: [PATCH 2/7] Remove scripp deps from validate_consistency --- utils/validate_consistency.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/utils/validate_consistency.py b/utils/validate_consistency.py index 0af2d50fcd..98796d3d1a 100644 --- a/utils/validate_consistency.py +++ b/utils/validate_consistency.py @@ -8,12 +8,6 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Autogenerate verdi CLI documentation from click.""" -# /// script -# requires-python = ">=3.9" -# dependencies = [ -# "click", -# ] -# /// import os From ad291f3d0c60e05c0eb85860aa1c7fbcdd88b17e Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 31 Jan 2025 11:11:06 +0000 Subject: [PATCH 3/7] Try uv-run hook on forked uv-pre-commit --- .pre-commit-config.yaml | 137 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27a5ccf790..992292b9ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,11 +56,144 @@ repos: environment.yml| )$ -- repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.5.22 + #- repo: file:///home/hollas/atmospec/uv-pre-commit +- repo: https://github.com/danielhollas/uv-pre-commit + rev: uv-run hooks: # Check and update the uv lockfile - id: uv-lock + - id: uv-run + name: Update conda environment file + args: [./utils/dependency_management.py, generate-environment-yml] + files: >- + (?x)^( + pyproject.toml| + utils/dependency_management.py + )$ + - id: uv-run + name: Validate environment.yml + args: [./utils/dependency_management.py, validate-environment-yml] + files: >- + (?x)^( + pyproject.toml| + utils/dependency_management.py| + environment.yml + )$ + - id: uv-run + name: mypy type check + args: [mypy, --config-file=pyproject.toml] + pass_filenames: true + types: [python] + exclude: >- + (?x)^( + .github/.*| + .molecule/.*| + .docker/.*| + docs/.*| + utils/.*| + tests/.*| + + src/aiida/calculations/arithmetic/add.py| + src/aiida/calculations/diff_tutorial/calculations.py| + src/aiida/calculations/templatereplacer.py| + src/aiida/calculations/transfer.py| + src/aiida/cmdline/commands/cmd_archive.py| + src/aiida/cmdline/commands/cmd_calcjob.py| + src/aiida/cmdline/commands/cmd_code.py| + src/aiida/cmdline/commands/cmd_computer.py| + src/aiida/cmdline/commands/cmd_data/cmd_list.py| + src/aiida/cmdline/commands/cmd_data/cmd_upf.py| + src/aiida/cmdline/commands/cmd_devel.py| + src/aiida/cmdline/commands/cmd_group.py| + src/aiida/cmdline/commands/cmd_node.py| + src/aiida/cmdline/commands/cmd_shell.py| + src/aiida/cmdline/commands/cmd_storage.py| + src/aiida/cmdline/params/options/commands/setup.py| + src/aiida/cmdline/params/options/interactive.py| + src/aiida/cmdline/params/options/main.py| + src/aiida/cmdline/params/options/multivalue.py| + src/aiida/cmdline/params/types/group.py| + src/aiida/cmdline/utils/ascii_vis.py| + src/aiida/cmdline/utils/common.py| + src/aiida/cmdline/utils/echo.py| + src/aiida/common/extendeddicts.py| + src/aiida/common/utils.py| + src/aiida/engine/daemon/execmanager.py| + src/aiida/engine/processes/calcjobs/manager.py| + src/aiida/engine/processes/calcjobs/monitors.py| + src/aiida/engine/processes/calcjobs/tasks.py| + src/aiida/engine/processes/control.py| + src/aiida/engine/processes/ports.py| + src/aiida/manage/configuration/__init__.py| + src/aiida/manage/configuration/config.py| + src/aiida/manage/external/rmq/launcher.py| + src/aiida/manage/tests/main.py| + src/aiida/manage/tests/pytest_fixtures.py| + src/aiida/orm/comments.py| + src/aiida/orm/computers.py| + src/aiida/orm/implementation/storage_backend.py| + src/aiida/orm/nodes/comments.py| + src/aiida/orm/nodes/data/array/bands.py| + src/aiida/orm/nodes/data/array/trajectory.py| + src/aiida/orm/nodes/data/cif.py| + src/aiida/orm/nodes/data/remote/base.py| + src/aiida/orm/nodes/data/structure.py| + src/aiida/orm/nodes/data/upf.py| + src/aiida/orm/nodes/process/calculation/calcjob.py| + src/aiida/orm/nodes/process/process.py| + src/aiida/orm/utils/builders/code.py| + src/aiida/orm/utils/builders/computer.py| + src/aiida/orm/utils/calcjob.py| + src/aiida/orm/utils/node.py| + src/aiida/repository/backend/disk_object_store.py| + src/aiida/repository/backend/sandbox.py| + src/aiida/restapi/common/utils.py| + src/aiida/restapi/resources.py| + src/aiida/restapi/run_api.py| + src/aiida/restapi/translator/base.py| + src/aiida/restapi/translator/computer.py| + src/aiida/restapi/translator/group.py| + src/aiida/restapi/translator/nodes/.*| + src/aiida/restapi/translator/user.py| + src/aiida/schedulers/plugins/direct.py| + src/aiida/schedulers/plugins/lsf.py| + src/aiida/schedulers/plugins/pbsbaseclasses.py| + src/aiida/schedulers/plugins/sge.py| + src/aiida/schedulers/plugins/slurm.py| + src/aiida/storage/psql_dos/migrations/utils/integrity.py| + src/aiida/storage/psql_dos/migrations/utils/legacy_workflows.py| + src/aiida/storage/psql_dos/migrations/utils/migrate_repository.py| + src/aiida/storage/psql_dos/migrations/utils/parity.py| + src/aiida/storage/psql_dos/migrations/utils/reflect.py| + src/aiida/storage/psql_dos/migrations/utils/utils.py| + src/aiida/storage/psql_dos/migrations/versions/1de112340b16_django_parity_1.py| + src/aiida/storage/psql_dos/migrator.py| + src/aiida/storage/psql_dos/models/.*| + src/aiida/storage/psql_dos/orm/.*| + src/aiida/storage/sqlite_temp/backend.py| + src/aiida/storage/sqlite_zip/backend.py| + src/aiida/storage/sqlite_zip/migrations/legacy_to_main.py| + src/aiida/storage/sqlite_zip/migrator.py| + src/aiida/storage/sqlite_zip/models.py| + src/aiida/storage/sqlite_zip/orm.py| + src/aiida/tools/data/array/kpoints/legacy.py| + src/aiida/tools/data/array/kpoints/seekpath.py| + src/aiida/tools/data/orbital/orbital.py| + src/aiida/tools/data/orbital/realhydrogen.py| + src/aiida/tools/dbimporters/plugins/.*| + src/aiida/tools/graph/age_entities.py| + src/aiida/tools/graph/age_rules.py| + src/aiida/tools/graph/deletions.py| + src/aiida/tools/graph/graph_traversers.py| + src/aiida/tools/groups/paths.py| + src/aiida/tools/query/calculation.py| + src/aiida/tools/query/mapping.py| + src/aiida/transports/cli.py| + src/aiida/transports/plugins/local.py| + src/aiida/transports/plugins/ssh.py| + src/aiida/workflows/arithmetic/multiply_add.py| + )$ + - repo: local From 6a8f76b4c3cb7790fa45ab98ceda156488b980d1 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Fri, 31 Jan 2025 22:38:55 +0000 Subject: [PATCH 4/7] More testing --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 992292b9ae..de34ce6141 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,9 +56,9 @@ repos: environment.yml| )$ - #- repo: file:///home/hollas/atmospec/uv-pre-commit -- repo: https://github.com/danielhollas/uv-pre-commit - rev: uv-run +- repo: file:///home/hollas/atmospec/uv-pre-commit + #- repo: https://github.com/danielhollas/uv-pre-commit + rev: c2c5e6a1c7d17dd3c237aae6557d15a142b89b8c hooks: # Check and update the uv lockfile - id: uv-lock From c975b8045853cc35f4b2bb6afb2e695bfb3b88de Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 5 Feb 2025 10:32:26 +0000 Subject: [PATCH 5/7] RTD conf 'simplification' --- .readthedocs.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 8f71246af5..6373e26569 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -17,15 +17,17 @@ build: # https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-uv pre_create_environment: - asdf plugin add uv - - asdf install uv 0.5.22 - - asdf global uv 0.5.22 + - asdf install uv 0.5.28 + - asdf global uv 0.5.28 create_environment: - - uv venv + - uv venv $READTHEDOCS_VIRTUALENV_PATH install: - - uv sync --extra docs --extra tests --extra rest --extra atomic_tools - build: - html: - - uv run sphinx-build -T -W --keep-going -b html -d _build/doctrees -D language=en docs/source $READTHEDOCS_OUTPUT/html + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --extra docs --extra tests --extra rest --extra atomic_tools + +sphinx: + builder: html + fail_on_warning: true + configuration: docs/source/conf.py search: ranking: From d9db4e27f4ae6ec52266fd5d951e493a9d9c0a8b Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 5 Feb 2025 10:54:06 +0000 Subject: [PATCH 6/7] Add typing to src/aiida/workflows/arithmetic/multiply_add.py --- .pre-commit-config.yaml | 2 -- .../workflows/arithmetic/multiply_add.py | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de34ce6141..ee155d9a0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -191,7 +191,6 @@ repos: src/aiida/transports/cli.py| src/aiida/transports/plugins/local.py| src/aiida/transports/plugins/ssh.py| - src/aiida/workflows/arithmetic/multiply_add.py| )$ @@ -323,7 +322,6 @@ repos: src/aiida/transports/cli.py| src/aiida/transports/plugins/local.py| src/aiida/transports/plugins/ssh.py| - src/aiida/workflows/arithmetic/multiply_add.py| )$ - id: generate-conda-environment diff --git a/src/aiida/workflows/arithmetic/multiply_add.py b/src/aiida/workflows/arithmetic/multiply_add.py index b1d29b9aae..5adb84d71a 100644 --- a/src/aiida/workflows/arithmetic/multiply_add.py +++ b/src/aiida/workflows/arithmetic/multiply_add.py @@ -9,12 +9,18 @@ # start-marker for docs """Implementation of the MultiplyAddWorkChain for testing and demonstration purposes.""" +from __future__ import annotations + +from typing import TypeVar + from aiida.engine import ToContext, WorkChain, calcfunction -from aiida.orm import AbstractCode, Int +from aiida.orm import AbstractCode, Int, ProcessNode from aiida.plugins.factories import CalculationFactory ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') +T = TypeVar('T') + @calcfunction def multiply(x, y): @@ -25,7 +31,7 @@ class MultiplyAddWorkChain(WorkChain): """WorkChain to multiply two numbers and add a third, for testing and demonstration purposes.""" @classmethod - def define(cls, spec): + def define(cls, spec) -> None: """Specify inputs and outputs.""" super().define(spec) spec.input('x', valid_type=Int) @@ -41,24 +47,25 @@ def define(cls, spec): spec.output('result', valid_type=Int) spec.exit_code(400, 'ERROR_NEGATIVE_NUMBER', message='The result is a negative number.') - def multiply(self): + def multiply(self) -> None: """Multiply two integers.""" self.ctx.product = multiply(self.inputs.x, self.inputs.y) - def add(self): + def add(self) -> dict[str, ProcessNode]: """Add two numbers using the `ArithmeticAddCalculation` calculation job plugin.""" inputs = {'x': self.ctx.product, 'y': self.inputs.z, 'code': self.inputs.code} - future = self.submit(ArithmeticAddCalculation, **inputs) + future = self.submit(ArithmeticAddCalculation, **inputs) # type: ignore[arg-type] self.report(f'Submitted the `ArithmeticAddCalculation`: {future}') return ToContext(addition=future) - def validate_result(self): + def validate_result(self) -> int | None: """Make sure the result is not negative.""" result = self.ctx.addition.outputs.sum if result.value < 0: return self.exit_codes.ERROR_NEGATIVE_NUMBER + return None - def result(self): + def result(self) -> None: """Add the result to the outputs.""" self.out('result', self.ctx.addition.outputs.sum) From a6e8fa410bc6d4aac14a8f504bdc3192c9d3dd92 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 5 Feb 2025 14:51:35 +0000 Subject: [PATCH 7/7] RTD: Remove env creation step --- .readthedocs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 6373e26569..1345e92882 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,8 +19,6 @@ build: - asdf plugin add uv - asdf install uv 0.5.28 - asdf global uv 0.5.28 - create_environment: - - uv venv $READTHEDOCS_VIRTUALENV_PATH install: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --extra docs --extra tests --extra rest --extra atomic_tools