WIP: Use inline script dependencies (PEP-723) for utils/*py scripts

.github/workflows/test-install.yml

@@ -42,11 +42,8 @@ jobs:
       with:
         version: latest
 
-    - name: Install utils/ dependencies
-      run: uv pip install --system -r utils/requirements.txt
-
     - name: Validate conda environment file
-      run: python ./utils/dependency_management.py validate-environment-yml
+      run: uv run --script ./utils/dependency_management.py validate-environment-yml
 
   create-conda-environment:
     # Verify that we can create a valid conda environment from the environment.yml file.

.pre-commit-config.yaml

@@ -56,11 +56,143 @@ repos:
         environment.yml|
       )$
 
-- repo: https://github.com/astral-sh/uv-pre-commit
-  rev: 0.5.22
+- repo: file:///home/hollas/atmospec/uv-pre-commit
+  #- repo: https://github.com/danielhollas/uv-pre-commit
+  rev: c2c5e6a1c7d17dd3c237aae6557d15a142b89b8c
   hooks:
     # Check and update the uv lockfile
   - id: uv-lock
+  - id: uv-run
+    name: Update conda environment file
+    args: [./utils/dependency_management.py, generate-environment-yml]
+    files: >-
+      (?x)^(
+        pyproject.toml|
+        utils/dependency_management.py
+      )$
+  - id: uv-run
+    name: Validate environment.yml
+    args: [./utils/dependency_management.py, validate-environment-yml]
+    files: >-
+      (?x)^(
+        pyproject.toml|
+        utils/dependency_management.py|
+        environment.yml
+      )$
+  - id: uv-run
+    name: mypy type check
+    args: [mypy, --config-file=pyproject.toml]
+    pass_filenames: true
+    types: [python]
+    exclude: >-
+      (?x)^(
+        .github/.*|
+        .molecule/.*|
+        .docker/.*|
+        docs/.*|
+        utils/.*|
+        tests/.*|
+
+        src/aiida/calculations/arithmetic/add.py|
+        src/aiida/calculations/diff_tutorial/calculations.py|
+        src/aiida/calculations/templatereplacer.py|
+        src/aiida/calculations/transfer.py|
+        src/aiida/cmdline/commands/cmd_archive.py|
+        src/aiida/cmdline/commands/cmd_calcjob.py|
+        src/aiida/cmdline/commands/cmd_code.py|
+        src/aiida/cmdline/commands/cmd_computer.py|
+        src/aiida/cmdline/commands/cmd_data/cmd_list.py|
+        src/aiida/cmdline/commands/cmd_data/cmd_upf.py|
+        src/aiida/cmdline/commands/cmd_devel.py|
+        src/aiida/cmdline/commands/cmd_group.py|
+        src/aiida/cmdline/commands/cmd_node.py|
+        src/aiida/cmdline/commands/cmd_shell.py|
+        src/aiida/cmdline/commands/cmd_storage.py|
+        src/aiida/cmdline/params/options/commands/setup.py|
+        src/aiida/cmdline/params/options/interactive.py|
+        src/aiida/cmdline/params/options/main.py|
+        src/aiida/cmdline/params/options/multivalue.py|
+        src/aiida/cmdline/params/types/group.py|
+        src/aiida/cmdline/utils/ascii_vis.py|
+        src/aiida/cmdline/utils/common.py|
+        src/aiida/cmdline/utils/echo.py|
+        src/aiida/common/extendeddicts.py|
+        src/aiida/common/utils.py|
+        src/aiida/engine/daemon/execmanager.py|
+        src/aiida/engine/processes/calcjobs/manager.py|
+        src/aiida/engine/processes/calcjobs/monitors.py|
+        src/aiida/engine/processes/calcjobs/tasks.py|
+        src/aiida/engine/processes/control.py|
+        src/aiida/engine/processes/ports.py|
+        src/aiida/manage/configuration/__init__.py|
+        src/aiida/manage/configuration/config.py|
+        src/aiida/manage/external/rmq/launcher.py|
+        src/aiida/manage/tests/main.py|
+        src/aiida/manage/tests/pytest_fixtures.py|
+        src/aiida/orm/comments.py|
+        src/aiida/orm/computers.py|
+        src/aiida/orm/implementation/storage_backend.py|
+        src/aiida/orm/nodes/comments.py|
+        src/aiida/orm/nodes/data/array/bands.py|
+        src/aiida/orm/nodes/data/array/trajectory.py|
+        src/aiida/orm/nodes/data/cif.py|
+        src/aiida/orm/nodes/data/remote/base.py|
+        src/aiida/orm/nodes/data/structure.py|
+        src/aiida/orm/nodes/data/upf.py|
+        src/aiida/orm/nodes/process/calculation/calcjob.py|
+        src/aiida/orm/nodes/process/process.py|
+        src/aiida/orm/utils/builders/code.py|
+        src/aiida/orm/utils/builders/computer.py|
+        src/aiida/orm/utils/calcjob.py|
+        src/aiida/orm/utils/node.py|
+        src/aiida/repository/backend/disk_object_store.py|
+        src/aiida/repository/backend/sandbox.py|
+        src/aiida/restapi/common/utils.py|
+        src/aiida/restapi/resources.py|
+        src/aiida/restapi/run_api.py|
+        src/aiida/restapi/translator/base.py|
+        src/aiida/restapi/translator/computer.py|
+        src/aiida/restapi/translator/group.py|
+        src/aiida/restapi/translator/nodes/.*|
+        src/aiida/restapi/translator/user.py|
+        src/aiida/schedulers/plugins/direct.py|
+        src/aiida/schedulers/plugins/lsf.py|
+        src/aiida/schedulers/plugins/pbsbaseclasses.py|
+        src/aiida/schedulers/plugins/sge.py|
+        src/aiida/schedulers/plugins/slurm.py|
+        src/aiida/storage/psql_dos/migrations/utils/integrity.py|
+        src/aiida/storage/psql_dos/migrations/utils/legacy_workflows.py|
+        src/aiida/storage/psql_dos/migrations/utils/migrate_repository.py|
+        src/aiida/storage/psql_dos/migrations/utils/parity.py|
+        src/aiida/storage/psql_dos/migrations/utils/reflect.py|
+        src/aiida/storage/psql_dos/migrations/utils/utils.py|
+        src/aiida/storage/psql_dos/migrations/versions/1de112340b16_django_parity_1.py|
+        src/aiida/storage/psql_dos/migrator.py|
+        src/aiida/storage/psql_dos/models/.*|
+        src/aiida/storage/psql_dos/orm/.*|
+        src/aiida/storage/sqlite_temp/backend.py|
+        src/aiida/storage/sqlite_zip/backend.py|
+        src/aiida/storage/sqlite_zip/migrations/legacy_to_main.py|
+        src/aiida/storage/sqlite_zip/migrator.py|
+        src/aiida/storage/sqlite_zip/models.py|
+        src/aiida/storage/sqlite_zip/orm.py|
+        src/aiida/tools/data/array/kpoints/legacy.py|
+        src/aiida/tools/data/array/kpoints/seekpath.py|
+        src/aiida/tools/data/orbital/orbital.py|
+        src/aiida/tools/data/orbital/realhydrogen.py|
+        src/aiida/tools/dbimporters/plugins/.*|
+        src/aiida/tools/graph/age_entities.py|
+        src/aiida/tools/graph/age_rules.py|
+        src/aiida/tools/graph/deletions.py|
+        src/aiida/tools/graph/graph_traversers.py|
+        src/aiida/tools/groups/paths.py|
+        src/aiida/tools/query/calculation.py|
+        src/aiida/tools/query/mapping.py|
+        src/aiida/transports/cli.py|
+        src/aiida/transports/plugins/local.py|
+        src/aiida/transports/plugins/ssh.py|
+      )$
+
 
 - repo: local
 
@@ -190,7 +322,6 @@ repos:
         src/aiida/transports/cli.py|
         src/aiida/transports/plugins/local.py|
         src/aiida/transports/plugins/ssh.py|
-        src/aiida/workflows/arithmetic/multiply_add.py|
       )$
 
   - id: generate-conda-environment
@@ -217,7 +348,7 @@ repos:
       )$
 
   - id: verdi-autodocs
-    name: Automatically generating verdi docs
+    name: Generate verdi docs
     entry: python ./utils/validate_consistency.py verdi-autodocs
     language: system
     pass_filenames: false

.readthedocs.yml

@@ -17,15 +17,15 @@ build:
     # https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-uv
     pre_create_environment:
     - asdf plugin add uv
-    - asdf install uv 0.5.22
-    - asdf global uv 0.5.22
-    create_environment:
-    - uv venv
+    - asdf install uv 0.5.28
+    - asdf global uv 0.5.28
     install:
-    - uv sync --extra docs --extra tests --extra rest --extra atomic_tools
-    build:
-      html:
-      - uv run sphinx-build -T -W --keep-going -b html -d _build/doctrees -D language=en docs/source $READTHEDOCS_OUTPUT/html
+    - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --extra docs --extra tests --extra rest --extra atomic_tools
+
+sphinx:
+  builder: html
+  fail_on_warning: true
+  configuration: docs/source/conf.py
 
 search:
   ranking:

src/aiida/workflows/arithmetic/multiply_add.py

@@ -9,12 +9,18 @@
 # start-marker for docs
 """Implementation of the MultiplyAddWorkChain for testing and demonstration purposes."""
 
+from __future__ import annotations
+
+from typing import TypeVar
+
 from aiida.engine import ToContext, WorkChain, calcfunction
-from aiida.orm import AbstractCode, Int
+from aiida.orm import AbstractCode, Int, ProcessNode
 from aiida.plugins.factories import CalculationFactory
 
 ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add')
 
+T = TypeVar('T')
+
 
 @calcfunction
 def multiply(x, y):
@@ -25,7 +31,7 @@ class MultiplyAddWorkChain(WorkChain):
     """WorkChain to multiply two numbers and add a third, for testing and demonstration purposes."""
 
     @classmethod
-    def define(cls, spec):
+    def define(cls, spec) -> None:
         """Specify inputs and outputs."""
         super().define(spec)
         spec.input('x', valid_type=Int)
@@ -41,24 +47,25 @@ def define(cls, spec):
         spec.output('result', valid_type=Int)
         spec.exit_code(400, 'ERROR_NEGATIVE_NUMBER', message='The result is a negative number.')
 
-    def multiply(self):
+    def multiply(self) -> None:
         """Multiply two integers."""
         self.ctx.product = multiply(self.inputs.x, self.inputs.y)
 
-    def add(self):
+    def add(self) -> dict[str, ProcessNode]:
         """Add two numbers using the `ArithmeticAddCalculation` calculation job plugin."""
         inputs = {'x': self.ctx.product, 'y': self.inputs.z, 'code': self.inputs.code}
-        future = self.submit(ArithmeticAddCalculation, **inputs)
+        future = self.submit(ArithmeticAddCalculation, **inputs)  # type: ignore[arg-type]
         self.report(f'Submitted the `ArithmeticAddCalculation`: {future}')
         return ToContext(addition=future)
 
-    def validate_result(self):
+    def validate_result(self) -> int | None:
         """Make sure the result is not negative."""
         result = self.ctx.addition.outputs.sum
 
         if result.value < 0:
             return self.exit_codes.ERROR_NEGATIVE_NUMBER
+        return None
 
-    def result(self):
+    def result(self) -> None:
         """Add the result to the outputs."""
         self.out('result', self.ctx.addition.outputs.sum)

utils/dependency_management.py

@@ -9,6 +9,21 @@
 ###########################################################################
 """Utility CLI to manage dependencies for aiida-core."""
 
+# To run this script, we recommend using uv that will
+# automatically install required dependencies based on specifications below. E.g.
+# `uv run dependency-management.py --help`
+#
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "click==7.1",
+#     "packaging==23.1",
+#     "pyyaml==6.0.1",
+#     "requests==2.25.1",
+#     "tomli==2.0.0",
+# ]
+# ///
+
 import os
 import re
 import subprocess
@@ -17,15 +32,11 @@
 from pathlib import Path
 
 import click
-import requests
-import tomli
-import yaml
 from packaging.requirements import Requirement
-from packaging.version import parse
 
 ROOT = Path(__file__).resolve().parent.parent  # repository root
 
-SETUPTOOLS_CONDA_MAPPINGS = {
+PYPI_CONDA_MAPPINGS = {
     'graphviz': 'python-graphviz',
     'docstring-parser': 'docstring_parser',
 }
@@ -41,17 +52,26 @@ class DependencySpecificationError(click.ClickException):
 
 def _load_pyproject():
     """Load the setup configuration from the 'pyproject.toml' file."""
+    # TODO: Require tomli only for Python <3.11
+    # if sys.version < 3.11:
+    #    import tomli as tomllib
+    # else
+    #    import tomllib
+    import tomli as tomllib
+
     try:
         with open(ROOT / 'pyproject.toml', 'rb') as handle:
-            return tomli.load(handle)
-    except tomli.TOMLDecodeError as error:
+            return tomllib.load(handle)
+    except tomllib.TOMLDecodeError as error:
         raise DependencySpecificationError(f"Error while parsing 'pyproject.toml' file: {error}")
     except FileNotFoundError:
         raise DependencySpecificationError("The 'pyproject.toml' file is missing!")
 
 
 def _load_environment_yml():
     """Load the conda environment specification from the 'environment.yml' file."""
+    import yaml
+
     try:
         with open(ROOT / 'environment.yml', encoding='utf8') as file:
             return yaml.load(file, Loader=yaml.SafeLoader)
@@ -67,7 +87,7 @@ def _setuptools_to_conda(req):
     In case that the same underlying dependency is listed under different names
     on PyPI and conda-forge.
     """
-    for pattern, replacement in SETUPTOOLS_CONDA_MAPPINGS.items():
+    for pattern, replacement in PYPI_CONDA_MAPPINGS.items():
         if re.match(pattern, str(req)):
             req = Requirement(re.sub(pattern, replacement, str(req)))
             break
@@ -117,6 +137,8 @@ def cli():
 @cli.command('generate-environment-yml')
 def generate_environment_yml():
     """Generate 'environment.yml' file."""
+    import yaml
+
     # needed for ordered dict, see https://stackoverflow.com/a/52621703
     yaml.add_representer(
         OrderedDict,
@@ -295,6 +317,9 @@ def identify_outdated(extras, pre_releases):
     This function can thus be used to identify dependencies where the
     specification must be loosened.
     """
+    import requests
+    from packaging.version import parse
+
     # Read the requirements from 'pyproject.toml''
     pyproject = _load_pyproject()