From 54475ed74996151f7c6b956387e884b4d1521256 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Thu, 6 Feb 2025 14:16:40 -0500 Subject: [PATCH 01/12] Upgrade mypy to 1.15 Mypy 1.15 includes fix for , allowing several "type: ignore" comments to be removed. --- .github/workflows/ci-additional.yaml | 4 +-- .pre-commit-config.yaml | 38 +++++++++++++++++++--------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index fef104a1ddc..0619fa1a552 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -125,7 +125,7 @@ jobs: - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: - file: mypy_report/cobertura.xml + files: mypy_report/cobertura.xml flags: mypy env_vars: PYTHON_VERSION name: codecov-umbrella @@ -176,7 +176,7 @@ jobs: - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: - file: mypy_report/cobertura.xml + files: mypy_report/cobertura.xml flags: mypy-min env_vars: PYTHON_VERSION name: codecov-umbrella diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f02dbf9dc69..3ee39291c3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,23 +42,37 @@ repos: - id: prettier args: [--cache-location=.prettier_cache/cache] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.1 + rev: v1.15.0 hooks: - id: mypy - # Copied from setup.cfg - exclude: "properties|asv_bench" + files: "^xarray" + exclude: "^xarray/util/generate_.*\\.py$" # This is slow and so we take it out of the fast-path; requires passing # `--hook-stage manual` to pre-commit stages: [manual] - additional_dependencies: [ - # Type stubs - types-python-dateutil, - types-setuptools, - types-PyYAML, - types-pytz, - typing-extensions>=4.1.0, - numpy, - ] + additional_dependencies: + # Type stubs plus additional dependencies from ci/requirements/environment.yml + # required in order to satisfy most (ideally all) type checks. This is rather + # brittle, so it is difficult (if not impossible) to get mypy to succeed in + # this context, even when it succeeds in CI. + - dask + - distributed + - hypothesis + - matplotlib + - numpy==2.1.3 + - pandas-stubs + - pytest + - types-colorama + - types-defusedxml + - types-docutils + - types-pexpect + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools + - typing-extensions>=4.1.0 - repo: https://github.com/citation-file-format/cff-converter-python rev: ebf0b5e44d67f8beaa1cd13a0d0393ea04c6058d hooks: From 369bb327cae26d6ee9b94a6fa1e0709ac86b13cc Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 7 Feb 2025 15:45:14 -0500 Subject: [PATCH 02/12] Add type annotations to DataTree.pipe tests --- xarray/tests/test_datatree.py | 40 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/xarray/tests/test_datatree.py b/xarray/tests/test_datatree.py index 715d80e084a..c87a1e1329e 100644 --- a/xarray/tests/test_datatree.py +++ b/xarray/tests/test_datatree.py @@ -1,7 +1,7 @@ import re import sys import typing -from collections.abc import Mapping +from collections.abc import Callable, Mapping from copy import copy, deepcopy from textwrap import dedent @@ -1589,27 +1589,53 @@ def test_assign(self) -> None: class TestPipe: - def test_noop(self, create_test_datatree) -> None: + def test_noop(self, create_test_datatree: Callable[[], DataTree]) -> None: dt = create_test_datatree() actual = dt.pipe(lambda tree: tree) assert actual.identical(dt) - def test_params(self, create_test_datatree) -> None: + def test_args(self, create_test_datatree: Callable[[], DataTree]) -> None: dt = create_test_datatree() - def f(tree, **attrs): - return tree.assign(arr_with_attrs=xr.Variable("dim0", [], attrs=attrs)) + def f(tree: DataTree, x: int, y: int) -> DataTree: + return tree.assign( + arr_with_attrs=xr.Variable("dim0", [], attrs=dict(x=x, y=y)) + ) + + actual = dt.pipe(f, 1, 2) + assert actual["arr_with_attrs"].attrs == dict(x=1, y=2) + + def test_kwargs(self, create_test_datatree: Callable[[], DataTree]) -> None: + dt = create_test_datatree() + + def f(tree: DataTree, *, x: int, y: int, z: int) -> DataTree: + return tree.assign( + arr_with_attrs=xr.Variable("dim0", [], attrs=dict(x=x, y=y, z=z)) + ) attrs = {"x": 1, "y": 2, "z": 3} actual = dt.pipe(f, **attrs) assert actual["arr_with_attrs"].attrs == attrs - def test_named_self(self, create_test_datatree) -> None: + def test_args_kwargs(self, create_test_datatree: Callable[[], DataTree]) -> None: + dt = create_test_datatree() + + def f(tree: DataTree, x: int, *, y: int, z: int) -> DataTree: + return tree.assign( + arr_with_attrs=xr.Variable("dim0", [], attrs=dict(x=x, y=y, z=z)) + ) + + attrs = {"x": 1, "y": 2, "z": 3} + + actual = dt.pipe(f, attrs["x"], y=attrs["y"], z=attrs["z"]) + assert actual["arr_with_attrs"].attrs == attrs + + def test_named_self(self, create_test_datatree: Callable[[], DataTree]) -> None: dt = create_test_datatree() - def f(x, tree, y): + def f(x: int, tree: DataTree, y: int): tree.attrs.update({"x": x, "y": y}) return tree From 9815d61ce0d99487431139935c47962591ecbe0a Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 7 Feb 2025 16:46:29 -0500 Subject: [PATCH 03/12] More precisely type `pipe` methods. In addition, enhance mypy job configuration to support running it locally via `act`. Fixes #9997 --- .github/workflows/ci-additional.yaml | 80 ++++------- xarray/core/common.py | 33 ++++- xarray/core/datatree.py | 68 +++++++-- xarray/tests/test_dataarray_typing.yml | 190 +++++++++++++++++++++++++ xarray/tests/test_dataset_typing.yml | 190 +++++++++++++++++++++++++ xarray/tests/test_datatree_typing.yml | 190 +++++++++++++++++++++++++ 6 files changed, 676 insertions(+), 75 deletions(-) create mode 100644 xarray/tests/test_dataarray_typing.yml create mode 100644 xarray/tests/test_dataset_typing.yml create mode 100644 xarray/tests/test_datatree_typing.yml diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index 0619fa1a552..b0948dd2e45 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -33,6 +33,15 @@ jobs: with: keyword: "[skip-ci]" + detect-act: + name: Detect 'act' runner + runs-on: ubuntu-latest + outputs: + running: ${{ steps.detect-act.outputs.running }} + steps: + - id: detect-act + run: echo "running=${{ env.ACT || 'false' }}" >> "$GITHUB_OUTPUT" + doctest: name: Doctests runs-on: "ubuntu-latest" @@ -81,15 +90,23 @@ jobs: python -m pytest --doctest-modules xarray --ignore xarray/tests -Werror mypy: - name: Mypy + strategy: + matrix: + include: + - python-version: "3.10" + codecov-flags: mypy-min + - python-version: "3.12" + codecov-flags: mypy + name: Mypy ${{ matrix.python-version }} runs-on: "ubuntu-latest" - needs: detect-ci-trigger + needs: [detect-ci-trigger, detect-act] + if: always() && (needs.detect-ci-trigger.outputs.triggered == 'true' || needs.detect-act.outputs.running == 'true') defaults: run: shell: bash -l {0} env: CONDA_ENV_FILE: ci/requirements/environment.yml - PYTHON_VERSION: "3.12" + PYTHON_VERSION: ${{ matrix.python-version }} steps: - uses: actions/checkout@v4 @@ -116,68 +133,23 @@ jobs: python xarray/util/print_versions.py - name: Install mypy run: | - python -m pip install "mypy==1.15" --force-reinstall + python -m pip install "mypy==1.15" pytest-mypy-plugins --force-reinstall - name: Run mypy run: | python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report - - name: Upload mypy coverage to Codecov - uses: codecov/codecov-action@v5.3.1 - with: - files: mypy_report/cobertura.xml - flags: mypy - env_vars: PYTHON_VERSION - name: codecov-umbrella - fail_ci_if_error: false - - mypy-min: - name: Mypy 3.10 - runs-on: "ubuntu-latest" - needs: detect-ci-trigger - defaults: - run: - shell: bash -l {0} - env: - CONDA_ENV_FILE: ci/requirements/environment.yml - PYTHON_VERSION: "3.10" - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all branches and tags. - - - name: set environment variables + - name: Run mypy tests + # Run pytest with mypy plugin even if mypy analysis in previous step fails. + if: ${{ always() }} run: | - echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v2 - with: - environment-file: ${{env.CONDA_ENV_FILE}} - environment-name: xarray-tests - create-args: >- - python=${{env.PYTHON_VERSION}} - cache-environment: true - cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" - - name: Install xarray - run: | - python -m pip install --no-deps -e . - - name: Version info - run: | - python xarray/util/print_versions.py - - name: Install mypy - run: | - python -m pip install "mypy==1.15" --force-reinstall - - - name: Run mypy - run: | - python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report + python -m pytest -v --mypy-only-local-stub --mypy-pyproject-toml-file=pyproject.toml xarray/**/test_*.yml - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: files: mypy_report/cobertura.xml - flags: mypy-min + flags: ${{ matrix.codecov-flags }} env_vars: PYTHON_VERSION name: codecov-umbrella fail_ci_if_error: false diff --git a/xarray/core/common.py b/xarray/core/common.py index 01c02a8d14f..ceaae42356a 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -6,7 +6,7 @@ from contextlib import suppress from html import escape from textwrap import dedent -from typing import TYPE_CHECKING, Any, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, Union, overload import numpy as np import pandas as pd @@ -60,6 +60,7 @@ T_Resample = TypeVar("T_Resample", bound="Resample") C = TypeVar("C") T = TypeVar("T") +P = ParamSpec("P") class ImplementsArrayReduce: @@ -718,11 +719,27 @@ def assign_attrs(self, *args: Any, **kwargs: Any) -> Self: out.attrs.update(*args, **kwargs) return out + @overload + def pipe( + self, + func: Callable[Concatenate[Self, P], T], + *args: P.args, + **kwargs: P.kwargs, + ) -> T: ... + + @overload def pipe( self, - func: Callable[..., T] | tuple[Callable[..., T], str], + func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any, + ) -> T: ... + + def pipe( + self, + func: Callable[Concatenate[Self, P], T] | tuple[Callable[P, T], str], + *args: P.args, + **kwargs: P.kwargs, ) -> T: """ Apply ``func(self, *args, **kwargs)`` @@ -840,15 +857,19 @@ def pipe( pandas.DataFrame.pipe """ if isinstance(func, tuple): - func, target = func + # Use different var when unpacking function from tuple because the type + # signature of the unpacked function differs from the expected type + # signature in the case where only a function is given, rather than a tuple. + # This makes type checkers happy at both call sites below. + f, target = func if target in kwargs: raise ValueError( f"{target} is both the pipe target and a keyword argument" ) kwargs[target] = self - return func(*args, **kwargs) - else: - return func(self, *args, **kwargs) + return f(*args, **kwargs) + + return func(self, *args, **kwargs) def rolling_exp( self: T_DataWithCoords, diff --git a/xarray/core/datatree.py b/xarray/core/datatree.py index 1a388919f0c..61340ac99ad 100644 --- a/xarray/core/datatree.py +++ b/xarray/core/datatree.py @@ -12,7 +12,17 @@ Mapping, ) from html import escape -from typing import TYPE_CHECKING, Any, Literal, NoReturn, Union, overload +from typing import ( + TYPE_CHECKING, + Any, + Concatenate, + Literal, + NoReturn, + ParamSpec, + TypeVar, + Union, + overload, +) from xarray.core import utils from xarray.core._aggregations import DataTreeAggregations @@ -80,18 +90,23 @@ # """ # DEVELOPERS' NOTE # ---------------- -# The idea of this module is to create a `DataTree` class which inherits the tree structure from TreeNode, and also copies -# the entire API of `xarray.Dataset`, but with certain methods decorated to instead map the dataset function over every -# node in the tree. As this API is copied without directly subclassing `xarray.Dataset` we instead create various Mixin -# classes (in ops.py) which each define part of `xarray.Dataset`'s extensive API. +# The idea of this module is to create a `DataTree` class which inherits the tree +# structure from TreeNode, and also copies the entire API of `xarray.Dataset`, but with +# certain methods decorated to instead map the dataset function over every node in the +# tree. As this API is copied without directly subclassing `xarray.Dataset` we instead +# create various Mixin classes (in ops.py) which each define part of `xarray.Dataset`'s +# extensive API. # -# Some of these methods must be wrapped to map over all nodes in the subtree. Others are fine to inherit unaltered -# (normally because they (a) only call dataset properties and (b) don't return a dataset that should be nested into a new -# tree) and some will get overridden by the class definition of DataTree. +# Some of these methods must be wrapped to map over all nodes in the subtree. Others are +# fine to inherit unaltered (normally because they (a) only call dataset properties and +# (b) don't return a dataset that should be nested into a new tree) and some will get +# overridden by the class definition of DataTree. # """ T_Path = Union[str, NodePath] +T = TypeVar("T") +P = ParamSpec("P") def _collect_data_and_coord_variables( @@ -1465,9 +1480,28 @@ def map_over_datasets( # TODO fix this typing error return map_over_datasets(func, self, *args, kwargs=kwargs) + @overload + def pipe( + self, + func: Callable[Concatenate[Self, P], T], + *args: P.args, + **kwargs: P.kwargs, + ) -> T: ... + + @overload + def pipe( + self, + func: tuple[Callable[..., T], str], + *args: Any, + **kwargs: Any, + ) -> T: ... + def pipe( - self, func: Callable | tuple[Callable, str], *args: Any, **kwargs: Any - ) -> Any: + self, + func: Callable[Concatenate[Self, P], T] | tuple[Callable[..., T], str], + *args: Any, + **kwargs: Any, + ) -> T: """Apply ``func(self, *args, **kwargs)`` This method replicates the pandas method of the same name. @@ -1487,7 +1521,7 @@ def pipe( Returns ------- - object : Any + object : T the return type of ``func``. Notes @@ -1515,15 +1549,19 @@ def pipe( """ if isinstance(func, tuple): - func, target = func + # Use different var when unpacking function from tuple because the type + # signature of the unpacked function differs from the expected type + # signature in the case where only a function is given, rather than a tuple. + # This makes type checkers happy at both call sites below. + f, target = func if target in kwargs: raise ValueError( f"{target} is both the pipe target and a keyword argument" ) kwargs[target] = self - else: - args = (self,) + args - return func(*args, **kwargs) + return f(*args, **kwargs) + + return func(self, *args, **kwargs) # TODO some kind of .collapse() or .flatten() method to merge a subtree diff --git a/xarray/tests/test_dataarray_typing.yml b/xarray/tests/test_dataarray_typing.yml new file mode 100644 index 00000000000..04a6ee452aa --- /dev/null +++ b/xarray/tests/test_dataarray_typing.yml @@ -0,0 +1,190 @@ +- case: test_pipe_lambda_noarg_return_type + main: | + from xarray import DataArray + + da = DataArray().pipe(lambda data: data) + + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + +- case: test_pipe_lambda_posarg_return_type + main: | + from xarray import DataArray + + da = DataArray().pipe(lambda data, arg: arg, "foo") + + reveal_type(da) # N: Revealed type is "builtins.str" + +- case: test_pipe_lambda_chaining_return_type + main: | + from xarray import DataArray + + answer = DataArray().pipe(lambda data, arg: arg, "foo").count("o") + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_lambda_missing_arg + main: | + from xarray import DataArray + + # Call to pipe missing argument for lambda parameter `arg` + da = DataArray().pipe(lambda data, arg: data) + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], Any]" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_lambda_extra_arg + main: | + from xarray import DataArray + + # Call to pipe with extra argument for lambda + da = DataArray().pipe(lambda data: data, "oops!") + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any], Any]", "str" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_posarg + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int) -> DataArray: + return da + + # Call to pipe missing argument for function parameter `arg` + da = DataArray().pipe(f) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[DataArray, int], DataArray]" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_extra_posarg + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int) -> DataArray: + return da + + # Call to pipe missing keyword for kwonly parameter `kwonly` + da = DataArray().pipe(f, 42, "oops!") + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[DataArray, int], DataArray]", "int", "str" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_kwarg + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int, *, kwonly: int) -> DataArray: + return da + + # Call to pipe missing argument for kwonly parameter `kwonly` + da = DataArray().pipe(f, 42) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[DataArray, int, NamedArg(int, 'kwonly')], DataArray]", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_keyword + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int, *, kwonly: int) -> DataArray: + return da + + # Call to pipe missing keyword for kwonly parameter `kwonly` + da = DataArray().pipe(f, 42, 99) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[DataArray, int, NamedArg(int, 'kwonly')], DataArray]", "int", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_unexpected_keyword + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int, *, kwonly: int) -> DataArray: + return da + + # Call to pipe using wrong keyword: `kw` instead of `kwonly` + da = DataArray().pipe(f, 42, kw=99) + out: | + main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] + +- case: test_pipe_tuple_return_type_dataarray + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> DataArray: + return da + + da = DataArray().pipe((f, "da"), 42) + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + +- case: test_pipe_tuple_return_type_other + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> int: + return arg + + answer = DataArray().pipe((f, "da"), 42) + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_tuple_missing_arg + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> DataArray: + return da + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are missing an argument for parameter `arg`, so we get no error here. + + da = DataArray().pipe((f, "da")) + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we failed to pass an argument for `arg`. + + da = DataArray().pipe(lambda data, arg: f(arg, data)) + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], DataArray]" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_tuple_extra_arg + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> DataArray: + return da + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are providing too many args for `f`, so we get no error here. + + da = DataArray().pipe((f, "da"), 42, "foo") + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we passed too many arguments. + + da = DataArray().pipe(lambda data, arg: f(arg, data), 42, "foo") + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any, Any], DataArray]", "int", "str" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T diff --git a/xarray/tests/test_dataset_typing.yml b/xarray/tests/test_dataset_typing.yml new file mode 100644 index 00000000000..99ea4554e84 --- /dev/null +++ b/xarray/tests/test_dataset_typing.yml @@ -0,0 +1,190 @@ +- case: test_pipe_lambda_noarg_return_type + main: | + from xarray import Dataset + + ds = Dataset().pipe(lambda data: data) + + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + +- case: test_pipe_lambda_posarg_return_type + main: | + from xarray import Dataset + + ds = Dataset().pipe(lambda data, arg: arg, "foo") + + reveal_type(ds) # N: Revealed type is "builtins.str" + +- case: test_pipe_lambda_chaining_return_type + main: | + from xarray import Dataset + + answer = Dataset().pipe(lambda data, arg: arg, "foo").count("o") + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_lambda_missing_arg + main: | + from xarray import Dataset + + # Call to pipe missing argument for lambda parameter `arg` + ds = Dataset().pipe(lambda data, arg: data) + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], Any]" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_lambda_extra_arg + main: | + from xarray import Dataset + + # Call to pipe with extra argument for lambda + ds = Dataset().pipe(lambda data: data, "oops!") + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any], Any]", "str" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_posarg + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int) -> Dataset: + return ds + + # Call to pipe missing argument for function parameter `arg` + ds = Dataset().pipe(f) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Dataset, int], Dataset]" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_extra_posarg + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int) -> Dataset: + return ds + + # Call to pipe missing keyword for kwonly parameter `kwonly` + ds = Dataset().pipe(f, 42, "oops!") + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Dataset, int], Dataset]", "int", "str" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_kwarg + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int, *, kwonly: int) -> Dataset: + return ds + + # Call to pipe missing argument for kwonly parameter `kwonly` + ds = Dataset().pipe(f, 42) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Dataset, int, NamedArg(int, 'kwonly')], Dataset]", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_keyword + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int, *, kwonly: int) -> Dataset: + return ds + + # Call to pipe missing keyword for kwonly parameter `kwonly` + ds = Dataset().pipe(f, 42, 99) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Dataset, int, NamedArg(int, 'kwonly')], Dataset]", "int", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_unexpected_keyword + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int, *, kwonly: int) -> Dataset: + return ds + + # Call to pipe using wrong keyword: `kw` instead of `kwonly` + ds = Dataset().pipe(f, 42, kw=99) + out: | + main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] + +- case: test_pipe_tuple_return_type_dataset + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> Dataset: + return ds + + ds = Dataset().pipe((f, "ds"), 42) + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + +- case: test_pipe_tuple_return_type_other + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> int: + return arg + + answer = Dataset().pipe((f, "ds"), 42) + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_tuple_missing_arg + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> Dataset: + return ds + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are missing an argument for parameter `arg`, so we get no error here. + + ds = Dataset().pipe((f, "ds")) + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we failed to pass an argument for `arg`. + + ds = Dataset().pipe(lambda data, arg: f(arg, data)) + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], Dataset]" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_tuple_extra_arg + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> Dataset: + return ds + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are providing too many args for `f`, so we get no error here. + + ds = Dataset().pipe((f, "ds"), 42, "foo") + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we passed too many arguments. + + ds = Dataset().pipe(lambda data, arg: f(arg, data), 42, "foo") + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any, Any], Dataset]", "int", "str" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T diff --git a/xarray/tests/test_datatree_typing.yml b/xarray/tests/test_datatree_typing.yml new file mode 100644 index 00000000000..69d566d07e5 --- /dev/null +++ b/xarray/tests/test_datatree_typing.yml @@ -0,0 +1,190 @@ +- case: test_pipe_lambda_noarg_return_type + main: | + from xarray import DataTree + + dt = DataTree().pipe(lambda data: data) + + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + +- case: test_pipe_lambda_posarg_return_type + main: | + from xarray import DataTree + + dt = DataTree().pipe(lambda data, arg: arg, "foo") + + reveal_type(dt) # N: Revealed type is "builtins.str" + +- case: test_pipe_lambda_chaining_return_type + main: | + from xarray import DataTree + + answer = DataTree().pipe(lambda data, arg: arg, "foo").count("o") + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_lambda_missing_arg + main: | + from xarray import DataTree + + # Call to pipe missing argument for lambda parameter `arg` + dt = DataTree().pipe(lambda data, arg: data) + out: | + main:4: error: No overload variant of "pipe" of "DataTree" matches argument type "Callable[[Any, Any], Any]" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_lambda_extra_arg + main: | + from xarray import DataTree + + # Call to pipe with extra argument for lambda + dt = DataTree().pipe(lambda data: data, "oops!") + out: | + main:4: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[Any], Any]", "str" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_posarg + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int) -> DataTree: + return dt + + # Call to pipe missing argument for function parameter `arg` + dt = DataTree().pipe(f) + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument type "Callable[[DataTree, int], DataTree]" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_extra_posarg + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int) -> DataTree: + return dt + + # Call to pipe missing keyword for kwonly parameter `kwonly` + dt = DataTree().pipe(f, 42, "oops!") + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[DataTree, int], DataTree]", "int", "str" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_kwarg + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int, *, kwonly: int) -> DataTree: + return dt + + # Call to pipe missing argument for kwonly parameter `kwonly` + dt = DataTree().pipe(f, 42) + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[DataTree, int, NamedArg(int, 'kwonly')], DataTree]", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_keyword + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int, *, kwonly: int) -> DataTree: + return dt + + # Call to pipe missing keyword for kwonly parameter `kwonly` + dt = DataTree().pipe(f, 42, 99) + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[DataTree, int, NamedArg(int, 'kwonly')], DataTree]", "int", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_unexpected_keyword + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int, *, kwonly: int) -> DataTree: + return dt + + # Call to pipe using wrong keyword: `kw` instead of `kwonly` + dt = DataTree().pipe(f, 42, kw=99) + out: | + main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataTree" [call-arg] + +- case: test_pipe_tuple_return_type_datatree + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> DataTree: + return dt + + dt = DataTree().pipe((f, "dt"), 42) + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + +- case: test_pipe_tuple_return_type_other + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> int: + return arg + + answer = DataTree().pipe((f, "dt"), 42) + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_tuple_missing_arg + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> DataTree: + return dt + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are missing an argument for parameter `arg`, so we get no error here. + + dt = DataTree().pipe((f, "dt")) + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we failed to pass an argument for `arg`. + + dt = DataTree().pipe(lambda data, arg: f(arg, data)) + out: | + main:17: error: No overload variant of "pipe" of "DataTree" matches argument type "Callable[[Any, Any], DataTree]" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_tuple_extra_arg + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> DataTree: + return dt + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are providing too many args for `f`, so we get no error here. + + dt = DataTree().pipe((f, "dt"), 42, "foo") + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we passed too many arguments. + + dt = DataTree().pipe(lambda data, arg: f(arg, data), 42, "foo") + out: | + main:17: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[Any, Any], DataTree]", "int", "str" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T From e28f100b5f26306ab4d744f94ded79952c03cf36 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 11 Feb 2025 14:16:03 -0500 Subject: [PATCH 04/12] Pin mypy to 1.15 in CI --- .github/workflows/ci-additional.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index b0948dd2e45..d609f7237e7 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -133,7 +133,7 @@ jobs: python xarray/util/print_versions.py - name: Install mypy run: | - python -m pip install "mypy==1.15" pytest-mypy-plugins --force-reinstall + python -m pip install "mypy==1.15" pytest-mypy-plugins - name: Run mypy run: | From 73f92c9661c73b604ee8f4d54df2b89afde51972 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 12 Feb 2025 18:57:21 -0500 Subject: [PATCH 05/12] Revert mypy CI job changes --- .github/workflows/ci-additional.yaml | 82 +++++++++++++++++++--------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index d609f7237e7..fef104a1ddc 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -33,15 +33,6 @@ jobs: with: keyword: "[skip-ci]" - detect-act: - name: Detect 'act' runner - runs-on: ubuntu-latest - outputs: - running: ${{ steps.detect-act.outputs.running }} - steps: - - id: detect-act - run: echo "running=${{ env.ACT || 'false' }}" >> "$GITHUB_OUTPUT" - doctest: name: Doctests runs-on: "ubuntu-latest" @@ -90,23 +81,15 @@ jobs: python -m pytest --doctest-modules xarray --ignore xarray/tests -Werror mypy: - strategy: - matrix: - include: - - python-version: "3.10" - codecov-flags: mypy-min - - python-version: "3.12" - codecov-flags: mypy - name: Mypy ${{ matrix.python-version }} + name: Mypy runs-on: "ubuntu-latest" - needs: [detect-ci-trigger, detect-act] - if: always() && (needs.detect-ci-trigger.outputs.triggered == 'true' || needs.detect-act.outputs.running == 'true') + needs: detect-ci-trigger defaults: run: shell: bash -l {0} env: CONDA_ENV_FILE: ci/requirements/environment.yml - PYTHON_VERSION: ${{ matrix.python-version }} + PYTHON_VERSION: "3.12" steps: - uses: actions/checkout@v4 @@ -133,23 +116,68 @@ jobs: python xarray/util/print_versions.py - name: Install mypy run: | - python -m pip install "mypy==1.15" pytest-mypy-plugins + python -m pip install "mypy==1.15" --force-reinstall - name: Run mypy run: | python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report - - name: Run mypy tests - # Run pytest with mypy plugin even if mypy analysis in previous step fails. - if: ${{ always() }} + - name: Upload mypy coverage to Codecov + uses: codecov/codecov-action@v5.3.1 + with: + file: mypy_report/cobertura.xml + flags: mypy + env_vars: PYTHON_VERSION + name: codecov-umbrella + fail_ci_if_error: false + + mypy-min: + name: Mypy 3.10 + runs-on: "ubuntu-latest" + needs: detect-ci-trigger + defaults: + run: + shell: bash -l {0} + env: + CONDA_ENV_FILE: ci/requirements/environment.yml + PYTHON_VERSION: "3.10" + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags. + + - name: set environment variables run: | - python -m pytest -v --mypy-only-local-stub --mypy-pyproject-toml-file=pyproject.toml xarray/**/test_*.yml + echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v2 + with: + environment-file: ${{env.CONDA_ENV_FILE}} + environment-name: xarray-tests + create-args: >- + python=${{env.PYTHON_VERSION}} + cache-environment: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + - name: Install xarray + run: | + python -m pip install --no-deps -e . + - name: Version info + run: | + python xarray/util/print_versions.py + - name: Install mypy + run: | + python -m pip install "mypy==1.15" --force-reinstall + + - name: Run mypy + run: | + python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: - files: mypy_report/cobertura.xml - flags: ${{ matrix.codecov-flags }} + file: mypy_report/cobertura.xml + flags: mypy-min env_vars: PYTHON_VERSION name: codecov-umbrella fail_ci_if_error: false From 5d859b201d64ffedbd48bf1e1210b4d019120848 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 12 Feb 2025 19:03:07 -0500 Subject: [PATCH 06/12] Add pytest-mypy-plugin and typestub packages --- ci/requirements/environment.yml | 14 +++++++++++++- pyproject.toml | 11 +++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index 321dbe75c38..f1465f5a7e7 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -29,6 +29,7 @@ dependencies: - opt_einsum - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pooch @@ -39,14 +40,25 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present + - types-defusedxml + - types-pexpect diff --git a/pyproject.toml b/pyproject.toml index 32b0bce1322..b1ddbb104f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,12 +40,14 @@ dev = [ "hypothesis", "jinja2", "mypy", + "pandas-stubs", "pre-commit", "pytest", "pytest-cov", "pytest-env", - "pytest-xdist", + "pytest-mypy-plugins", "pytest-timeout", + "pytest-xdist", "ruff>=0.8.0", "sphinx", "sphinx_autosummary_accessors", @@ -304,7 +306,12 @@ known-first-party = ["xarray"] ban-relative-imports = "all" [tool.pytest.ini_options] -addopts = ["--strict-config", "--strict-markers"] +addopts = [ + "--strict-config", + "--strict-markers", + "--mypy-only-local-stub", + "--mypy-pyproject-toml-file=pyproject.toml", +] # We want to forbid warnings from within xarray in our tests — instead we should # fix our own code, or mark the test itself as expecting a warning. So this: From a6f9ef5d5d249e96bebbcb76f4fbc8a7c98a5fa7 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 14 Feb 2025 11:55:10 -0500 Subject: [PATCH 07/12] Add pytest-mypy-plugins to all conda env files --- .github/workflows/ci.yaml | 13 +++++++++++++ ci/requirements/all-but-dask.yml | 3 ++- ci/requirements/all-but-numba.yml | 3 ++- ci/requirements/bare-minimum.yml | 3 ++- ci/requirements/environment-3.14.yml | 14 +++++++++++++- ci/requirements/environment-windows-3.14.yml | 15 ++++++++++++++- ci/requirements/environment-windows.yml | 15 ++++++++++++++- ci/requirements/min-all-deps.yml | 3 ++- 8 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 713da24f577..1f76af94fea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -143,12 +143,25 @@ jobs: enableCrossOsArchive: true save-always: true + # Run all tests in *.py files, which excludes tests in *.yml files that test type + # annotations via the pytest-mypy-plugins plugin, because the type annotations + # tests fail when using multiple processes (the -n option to pytest). - name: Run tests run: python -m pytest -n 4 --timeout 180 --cov=xarray --cov-report=xml --junitxml=pytest.xml + xarray/tests/test_*.py + + # As noted in the comment on the previous step, we must run type annotation tests + # separately, and we will run them even if the preceding tests failed. Further, + # we must restrict these tests to run only when matrix.env is empty, as this is + # the only case when all of the necessary dependencies are included such that + # spurious mypy errors due to missing packages are eliminated. + - name: Run mypy tests + if: ${{ always() && matrix.env == '' }} + run: python -m pytest xarray/tests/test_*.yml - name: Upload test results if: always() diff --git a/ci/requirements/all-but-dask.yml b/ci/requirements/all-but-dask.yml index b7bf167188f..ca4943bddb1 100644 --- a/ci/requirements/all-but-dask.yml +++ b/ci/requirements/all-but-dask.yml @@ -30,8 +30,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn diff --git a/ci/requirements/all-but-numba.yml b/ci/requirements/all-but-numba.yml index 17a657eb32b..fa7ad81f198 100644 --- a/ci/requirements/all-but-numba.yml +++ b/ci/requirements/all-but-numba.yml @@ -43,8 +43,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn diff --git a/ci/requirements/bare-minimum.yml b/ci/requirements/bare-minimum.yml index d9590d95165..02e99d34af2 100644 --- a/ci/requirements/bare-minimum.yml +++ b/ci/requirements/bare-minimum.yml @@ -9,8 +9,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - numpy=1.24 - packaging=23.1 - pandas=2.1 diff --git a/ci/requirements/environment-3.14.yml b/ci/requirements/environment-3.14.yml index cca3a7a746b..cebae38bc83 100644 --- a/ci/requirements/environment-3.14.yml +++ b/ci/requirements/environment-3.14.yml @@ -29,6 +29,7 @@ dependencies: - opt_einsum - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pooch @@ -38,14 +39,25 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn # - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present + - types-defusedxml + - types-pexpect diff --git a/ci/requirements/environment-windows-3.14.yml b/ci/requirements/environment-windows-3.14.yml index c7f67d2efac..31c91b24b6d 100644 --- a/ci/requirements/environment-windows-3.14.yml +++ b/ci/requirements/environment-windows-3.14.yml @@ -25,6 +25,7 @@ dependencies: - numpy - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pre-commit @@ -33,12 +34,24 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn # - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr + - pip: + - types-defusedxml + - types-pexpect diff --git a/ci/requirements/environment-windows.yml b/ci/requirements/environment-windows.yml index a2ecef43d07..f8eb80f6c75 100644 --- a/ci/requirements/environment-windows.yml +++ b/ci/requirements/environment-windows.yml @@ -25,6 +25,7 @@ dependencies: - numpy - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pre-commit @@ -33,12 +34,24 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr + - pip: + - types-defusedxml + - types-pexpect diff --git a/ci/requirements/min-all-deps.yml b/ci/requirements/min-all-deps.yml index f3dab2e5bbf..52c7f9b18e3 100644 --- a/ci/requirements/min-all-deps.yml +++ b/ci/requirements/min-all-deps.yml @@ -46,8 +46,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio=1.3 - scipy=1.11 - seaborn=0.13 From 95084f156cad3201a688abfaa28f4bb1cca54318 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 14 Feb 2025 12:03:03 -0500 Subject: [PATCH 08/12] Remove dup pandas-stubs dep --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b1ddbb104f9..7c621bffe97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ dev = [ "hypothesis", "jinja2", "mypy", - "pandas-stubs", "pre-commit", "pytest", "pytest-cov", From d92b99275948d53b3f2a668710208b41fd739d25 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:33:04 -0500 Subject: [PATCH 09/12] Revert pre-commit config changes --- .pre-commit-config.yaml | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ee39291c3c..f02dbf9dc69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,37 +42,23 @@ repos: - id: prettier args: [--cache-location=.prettier_cache/cache] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.15.0 + rev: v1.14.1 hooks: - id: mypy - files: "^xarray" - exclude: "^xarray/util/generate_.*\\.py$" + # Copied from setup.cfg + exclude: "properties|asv_bench" # This is slow and so we take it out of the fast-path; requires passing # `--hook-stage manual` to pre-commit stages: [manual] - additional_dependencies: - # Type stubs plus additional dependencies from ci/requirements/environment.yml - # required in order to satisfy most (ideally all) type checks. This is rather - # brittle, so it is difficult (if not impossible) to get mypy to succeed in - # this context, even when it succeeds in CI. - - dask - - distributed - - hypothesis - - matplotlib - - numpy==2.1.3 - - pandas-stubs - - pytest - - types-colorama - - types-defusedxml - - types-docutils - - types-pexpect - - types-psutil - - types-Pygments - - types-python-dateutil - - types-pytz - - types-PyYAML - - types-setuptools - - typing-extensions>=4.1.0 + additional_dependencies: [ + # Type stubs + types-python-dateutil, + types-setuptools, + types-PyYAML, + types-pytz, + typing-extensions>=4.1.0, + numpy, + ] - repo: https://github.com/citation-file-format/cff-converter-python rev: ebf0b5e44d67f8beaa1cd13a0d0393ea04c6058d hooks: From a9307cf7cf379877367a48624e0dd0ada914ebe0 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:41:48 -0500 Subject: [PATCH 10/12] Place mypy tests behind pytest mypy marker --- .github/workflows/ci.yaml | 30 ++++++++++++++------------ conftest.py | 18 +++++++++++++++- pyproject.toml | 1 + xarray/tests/test_dataarray_typing.yml | 28 ++++++++++++------------ xarray/tests/test_dataset_typing.yml | 28 ++++++++++++------------ xarray/tests/test_datatree_typing.yml | 28 ++++++++++++------------ 6 files changed, 76 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f76af94fea..8061bfe8434 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,6 +49,7 @@ jobs: # Bookend python versions python-version: ["3.10", "3.13"] env: [""] + numprocesses: [4] include: # Minimum python version: - env: "bare-minimum" @@ -67,6 +68,16 @@ jobs: - env: "flaky" python-version: "3.13" os: ubuntu-latest + # The mypy tests must be executed using only 1 process in order to guarantee + # predictable mypy output messages for comparison to expectations. + - env: "mypy" + python-version: "3.10" + numprocesses: 1 + os: ubuntu-latest + - env: "mypy" + python-version: "3.13" + numprocesses: 1 + os: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -88,6 +99,10 @@ jobs: then echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV echo "PYTEST_ADDOPTS=-m 'flaky or network' --run-flaky --run-network-tests -W default" >> $GITHUB_ENV + elif [[ "${{ matrix.env }}" == "mypy" ]] ; + then + echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV + echo "PYTEST_ADDOPTS=-n 1 -m 'mypy' --run-mypy -W default" >> $GITHUB_ENV else echo "CONDA_ENV_FILE=ci/requirements/${{ matrix.env }}.yml" >> $GITHUB_ENV fi @@ -143,25 +158,12 @@ jobs: enableCrossOsArchive: true save-always: true - # Run all tests in *.py files, which excludes tests in *.yml files that test type - # annotations via the pytest-mypy-plugins plugin, because the type annotations - # tests fail when using multiple processes (the -n option to pytest). - name: Run tests - run: python -m pytest -n 4 + run: python -m pytest -n ${{ matrix.numprocesses }} --timeout 180 --cov=xarray --cov-report=xml --junitxml=pytest.xml - xarray/tests/test_*.py - - # As noted in the comment on the previous step, we must run type annotation tests - # separately, and we will run them even if the preceding tests failed. Further, - # we must restrict these tests to run only when matrix.env is empty, as this is - # the only case when all of the necessary dependencies are included such that - # spurious mypy errors due to missing packages are eliminated. - - name: Run mypy tests - if: ${{ always() && matrix.env == '' }} - run: python -m pytest xarray/tests/test_*.yml - name: Upload test results if: always() diff --git a/conftest.py b/conftest.py index 24b7530b220..532a7badd91 100644 --- a/conftest.py +++ b/conftest.py @@ -3,7 +3,7 @@ import pytest -def pytest_addoption(parser): +def pytest_addoption(parser: pytest.Parser): """Add command-line flags for pytest.""" parser.addoption("--run-flaky", action="store_true", help="runs flaky tests") parser.addoption( @@ -11,6 +11,7 @@ def pytest_addoption(parser): action="store_true", help="runs tests requiring a network connection", ) + parser.addoption("--run-mypy", action="store_true", help="runs mypy tests") def pytest_runtest_setup(item): @@ -21,6 +22,21 @@ def pytest_runtest_setup(item): pytest.skip( "set --run-network-tests to run test requiring an internet connection" ) + if "mypy" in item.keywords and not item.config.getoption("--run-mypy"): + pytest.skip("set --run-mypy option to run mypy tests") + + +# See https://docs.pytest.org/en/stable/example/markers.html#automatically-adding-markers-based-on-test-names +def pytest_collection_modifyitems(items): + for item in items: + if "mypy" in item.nodeid: + # IMPORTANT: mypy type annotation tests leverage the pytest-mypy-plugins + # plugin, and are thus written in test_*.yml files. As such, there are + # no explicit test functions on which we can apply a pytest.mark.mypy + # decorator. Therefore, we mark them via this name-based, automatic + # marking approach, meaning that each test case must contain "mypy" in the + # name. + item.add_marker(pytest.mark.mypy) @pytest.fixture(autouse=True) diff --git a/pyproject.toml b/pyproject.toml index 7c621bffe97..817fda6c328 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -367,6 +367,7 @@ filterwarnings = [ log_cli_level = "INFO" markers = [ "flaky: flaky tests", + "mypy: type annotation tests", "network: tests requiring a network connection", "slow: slow tests", "slow_hypothesis: slow hypothesis tests", diff --git a/xarray/tests/test_dataarray_typing.yml b/xarray/tests/test_dataarray_typing.yml index 04a6ee452aa..ae3356f9d7c 100644 --- a/xarray/tests/test_dataarray_typing.yml +++ b/xarray/tests/test_dataarray_typing.yml @@ -1,4 +1,4 @@ -- case: test_pipe_lambda_noarg_return_type +- case: test_mypy_pipe_lambda_noarg_return_type main: | from xarray import DataArray @@ -6,7 +6,7 @@ reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" -- case: test_pipe_lambda_posarg_return_type +- case: test_mypy_pipe_lambda_posarg_return_type main: | from xarray import DataArray @@ -14,7 +14,7 @@ reveal_type(da) # N: Revealed type is "builtins.str" -- case: test_pipe_lambda_chaining_return_type +- case: test_mypy_pipe_lambda_chaining_return_type main: | from xarray import DataArray @@ -22,7 +22,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_lambda_missing_arg +- case: test_mypy_pipe_lambda_missing_arg main: | from xarray import DataArray @@ -34,7 +34,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_lambda_extra_arg +- case: test_mypy_pipe_lambda_extra_arg main: | from xarray import DataArray @@ -46,7 +46,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_posarg +- case: test_mypy_pipe_function_missing_posarg main: | from xarray import DataArray @@ -61,7 +61,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_extra_posarg +- case: test_mypy_pipe_function_extra_posarg main: | from xarray import DataArray @@ -76,7 +76,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_kwarg +- case: test_mypy_pipe_function_missing_kwarg main: | from xarray import DataArray @@ -91,7 +91,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_keyword +- case: test_mypy_pipe_function_missing_keyword main: | from xarray import DataArray @@ -106,7 +106,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_unexpected_keyword +- case: test_mypy_pipe_function_unexpected_keyword main: | from xarray import DataArray @@ -118,7 +118,7 @@ out: | main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] -- case: test_pipe_tuple_return_type_dataarray +- case: test_mypy_pipe_tuple_return_type_dataarray main: | from xarray import DataArray @@ -128,7 +128,7 @@ da = DataArray().pipe((f, "da"), 42) reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" -- case: test_pipe_tuple_return_type_other +- case: test_mypy_pipe_tuple_return_type_other main: | from xarray import DataArray @@ -139,7 +139,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_tuple_missing_arg +- case: test_mypy_pipe_tuple_missing_arg main: | from xarray import DataArray @@ -164,7 +164,7 @@ main:17: note: def [P`9, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_tuple_extra_arg +- case: test_mypy_pipe_tuple_extra_arg main: | from xarray import DataArray diff --git a/xarray/tests/test_dataset_typing.yml b/xarray/tests/test_dataset_typing.yml index 99ea4554e84..3b62f81d361 100644 --- a/xarray/tests/test_dataset_typing.yml +++ b/xarray/tests/test_dataset_typing.yml @@ -1,4 +1,4 @@ -- case: test_pipe_lambda_noarg_return_type +- case: test_mypy_pipe_lambda_noarg_return_type main: | from xarray import Dataset @@ -6,7 +6,7 @@ reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" -- case: test_pipe_lambda_posarg_return_type +- case: test_mypy_pipe_lambda_posarg_return_type main: | from xarray import Dataset @@ -14,7 +14,7 @@ reveal_type(ds) # N: Revealed type is "builtins.str" -- case: test_pipe_lambda_chaining_return_type +- case: test_mypy_pipe_lambda_chaining_return_type main: | from xarray import Dataset @@ -22,7 +22,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_lambda_missing_arg +- case: test_mypy_pipe_lambda_missing_arg main: | from xarray import Dataset @@ -34,7 +34,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_lambda_extra_arg +- case: test_mypy_pipe_lambda_extra_arg main: | from xarray import Dataset @@ -46,7 +46,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_posarg +- case: test_mypy_pipe_function_missing_posarg main: | from xarray import Dataset @@ -61,7 +61,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_extra_posarg +- case: test_mypy_pipe_function_extra_posarg main: | from xarray import Dataset @@ -76,7 +76,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_kwarg +- case: test_mypy_pipe_function_missing_kwarg main: | from xarray import Dataset @@ -91,7 +91,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_keyword +- case: test_mypy_pipe_function_missing_keyword main: | from xarray import Dataset @@ -106,7 +106,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_unexpected_keyword +- case: test_mypy_pipe_function_unexpected_keyword main: | from xarray import Dataset @@ -118,7 +118,7 @@ out: | main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] -- case: test_pipe_tuple_return_type_dataset +- case: test_mypy_pipe_tuple_return_type_dataset main: | from xarray import Dataset @@ -128,7 +128,7 @@ ds = Dataset().pipe((f, "ds"), 42) reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" -- case: test_pipe_tuple_return_type_other +- case: test_mypy_pipe_tuple_return_type_other main: | from xarray import Dataset @@ -139,7 +139,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_tuple_missing_arg +- case: test_mypy_pipe_tuple_missing_arg main: | from xarray import Dataset @@ -164,7 +164,7 @@ main:17: note: def [P`9, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_tuple_extra_arg +- case: test_mypy_pipe_tuple_extra_arg main: | from xarray import Dataset diff --git a/xarray/tests/test_datatree_typing.yml b/xarray/tests/test_datatree_typing.yml index 69d566d07e5..fac7fe8ab65 100644 --- a/xarray/tests/test_datatree_typing.yml +++ b/xarray/tests/test_datatree_typing.yml @@ -1,4 +1,4 @@ -- case: test_pipe_lambda_noarg_return_type +- case: test_mypy_pipe_lambda_noarg_return_type main: | from xarray import DataTree @@ -6,7 +6,7 @@ reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" -- case: test_pipe_lambda_posarg_return_type +- case: test_mypy_pipe_lambda_posarg_return_type main: | from xarray import DataTree @@ -14,7 +14,7 @@ reveal_type(dt) # N: Revealed type is "builtins.str" -- case: test_pipe_lambda_chaining_return_type +- case: test_mypy_pipe_lambda_chaining_return_type main: | from xarray import DataTree @@ -22,7 +22,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_lambda_missing_arg +- case: test_mypy_pipe_lambda_missing_arg main: | from xarray import DataTree @@ -34,7 +34,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_lambda_extra_arg +- case: test_mypy_pipe_lambda_extra_arg main: | from xarray import DataTree @@ -46,7 +46,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_posarg +- case: test_mypy_pipe_function_missing_posarg main: | from xarray import DataTree @@ -61,7 +61,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_extra_posarg +- case: test_mypy_pipe_function_extra_posarg main: | from xarray import DataTree @@ -76,7 +76,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_kwarg +- case: test_mypy_pipe_function_missing_kwarg main: | from xarray import DataTree @@ -91,7 +91,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_keyword +- case: test_mypy_pipe_function_missing_keyword main: | from xarray import DataTree @@ -106,7 +106,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_unexpected_keyword +- case: test_mypy_pipe_function_unexpected_keyword main: | from xarray import DataTree @@ -118,7 +118,7 @@ out: | main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataTree" [call-arg] -- case: test_pipe_tuple_return_type_datatree +- case: test_mypy_pipe_tuple_return_type_datatree main: | from xarray import DataTree @@ -128,7 +128,7 @@ dt = DataTree().pipe((f, "dt"), 42) reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" -- case: test_pipe_tuple_return_type_other +- case: test_mypy_pipe_tuple_return_type_other main: | from xarray import DataTree @@ -139,7 +139,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_tuple_missing_arg +- case: test_mypy_pipe_tuple_missing_arg main: | from xarray import DataTree @@ -164,7 +164,7 @@ main:17: note: def [P`9, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_tuple_extra_arg +- case: test_mypy_pipe_tuple_extra_arg main: | from xarray import DataTree From 83b59face02de474992c57ba6954f613c23aab00 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:48:24 -0500 Subject: [PATCH 11/12] Set default pytest numprocesses to 4 --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8061bfe8434..1aad039320a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,7 +49,6 @@ jobs: # Bookend python versions python-version: ["3.10", "3.13"] env: [""] - numprocesses: [4] include: # Minimum python version: - env: "bare-minimum" @@ -159,7 +158,7 @@ jobs: save-always: true - name: Run tests - run: python -m pytest -n ${{ matrix.numprocesses }} + run: python -m pytest -n ${{ matrix.numprocesses || 4 }} --timeout 180 --cov=xarray --cov-report=xml From 6d6083ef3e4077116be1c41dba85b9a463a5d3b4 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:53:44 -0500 Subject: [PATCH 12/12] Ignore pytest-mypy-plugins for min version check --- ci/minimum_versions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/minimum_versions.py b/ci/minimum_versions.py index c226e304769..cc115789d0f 100644 --- a/ci/minimum_versions.py +++ b/ci/minimum_versions.py @@ -26,8 +26,9 @@ "pytest", "pytest-cov", "pytest-env", - "pytest-xdist", + "pytest-mypy-plugins", "pytest-timeout", + "pytest-xdist", "hypothesis", ]