Bump ruff, drop autoflake, add darglint back (#1279)

strickvl · AlexejPenner · web-flow · commit a02769d3835e · 2023-02-01T13:26:11.000+01:00
* bump ruff version

* remove autoflake

* remove unused variable assignment

* remove autoflake

* ignore D401 errors

* add noqa to avoid exceptions

* add darglint back :(

* fix missing docstrings

* add darglint dependencies back

* fix docstring indentation

* Update .github/workflows/docstring.yml

Co-authored-by: Alexej Penner &lt;thealexejpenner@gmail.com&gt;

* fix linting errors

---------

Co-authored-by: Alexej Penner &lt;thealexejpenner@gmail.com&gt;
diff --git a/.darglint b/.darglint
@@ -0,0 +1,2 @@
+[darglint]
+docstring_style=google
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -85,5 +85,8 @@ jobs:
   markdown-link-check:
     uses: ./.github/workflows/markdown-link-check.yml
 
+  docstring-check:
+    uses: ./.github/workflows/docstring.yml
+
   spell-check:
     uses: ./.github/workflows/spellcheck.yml
diff --git a/.github/workflows/docstring.yml b/.github/workflows/docstring.yml
@@ -0,0 +1,25 @@
+name: Docstring Check
+
+on: workflow_call
+
+jobs:
+  docstring:
+    name: docstringcheck
+    runs-on: ubuntu-latest
+    env:
+      ZENML_DEBUG: 1
+      ZENML_ANALYTICS_OPT_IN: false
+      PYTHONIOENCODING: "utf-8"
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.7"
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          python -m pip install darglint
+      - name: Docstring Check
+        run: bash scripts/docstring.sh
diff --git a/docs/mkdocstrings_helper.py b/docs/mkdocstrings_helper.py
@@ -205,7 +205,7 @@ def generate_docs(
     )
 
     integration_file_contents = [INTEGRATION_DOCS_TITLE]
-    integrations_file_content = create_entity_docs(
+    create_entity_docs(
         api_doc_file_dir=integrations_dev_doc_file_dir,
         ignored_modules=["__init__.py", "__pycache__"],
         sources_path=path / "integrations",
diff --git a/pyproject.toml b/pyproject.toml
@@ -86,14 +86,14 @@ server = ["fastapi", "uvicorn", "python-multipart", "python-jose", "fastapi-util
 black = "^22.3.0"
 pytest = "^6.2.4"
 mypy = "^0.991"
-ruff = "^0.0.223"
+ruff = "^0.0.238"
 coverage = { extras = ["toml"], version = "^5.5" }
 pre-commit = "^2.14.0"
-autoflake = "^1.4"
 pyment = "^0.3.3"
 tox = "^3.24.3"
 hypothesis = "^6.43.1"
 typing-extensions = ">=3.7.4"
+darglint = "^1.8.1"
 
 # pytest dev dependencies
 pytest-randomly = "^3.10.1"
@@ -204,7 +204,7 @@ exclude = [
 ]
 per-file-ignores = {}
 select = ["D", "E", "F", "I", "I001", "Q"]
-ignore = ["E501", "F401", "F403", "D301", "D403", "D407", "D213", "D203", "S101", "S104", "S105", "S106", "S107"]
+ignore = ["E501", "F401", "F403", "D301", "D401", "D403", "D407", "D213", "D203", "S101", "S104", "S105", "S106", "S107"]
 src = ["src", "test"]
 # use Python 3.7 as the minimum version for autofixing
 target-version = "py37"
diff --git a/scripts/docstring.sh b/scripts/docstring.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+set -x
+
+DOCSTRING_SRC=${1:-"src/zenml tests/harness"}
+
+export ZENML_DEBUG=1
+export ZENML_ANALYTICS_OPT_IN=false
+
+darglint -v 2 $DOCSTRING_SRC
diff --git a/scripts/format.sh b/scripts/format.sh
@@ -5,7 +5,10 @@ SRC=${1:-"src/zenml tests examples docs/mkdocstrings_helper.py"}
 
 export ZENML_DEBUG=1
 export ZENML_ANALYTICS_OPT_IN=false
-autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place $SRC --exclude=__init__.py
+
+# autoflake replacement: removes unused imports and variables
+ruff $SRC --select F401,F841 --fix --exclude "__init__.py" --isolated
+
 # sorts imports
 ruff $SRC --select I --fix --ignore D
 black $SRC
diff --git a/scripts/lint.sh b/scripts/lint.sh
@@ -13,8 +13,8 @@ ruff $SRC_NO_TESTS
 # TODO: Fix docstrings in tests and examples and remove the `--extend-ignore D` flag
 ruff $TESTS_EXAMPLES --extend-ignore D
 
-# TODO: remove this once ruff implements the feature
-autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place $SRC --exclude=__init__.py --check | ( grep -v "No issues detected" || true )
+# autoflake replacement: checks for unused imports and variables
+ruff $SRC --select F401,F841 --exclude "__init__.py" --isolated
 
 black $SRC  --check
 
diff --git a/src/zenml/cli/artifact.py b/src/zenml/cli/artifact.py
@@ -36,7 +36,11 @@ def artifact() -> None:
 @cli_utils.list_options(ArtifactFilterModel)
 @artifact.command("list", help="List all artifacts.")
 def list_artifacts(**kwargs: Any) -> None:
-    """List all artifacts."""
+    """List all artifacts.
+
+    Args:
+        **kwargs: Keyword arguments to filter artifacts.
+    """
     cli_utils.print_active_config()
     artifacts = Client().list_artifacts(**kwargs)
 
diff --git a/src/zenml/cli/pipeline.py b/src/zenml/cli/pipeline.py
@@ -58,7 +58,11 @@ def cli_pipeline_run(python_file: str, config_path: str) -> None:
 @pipeline.command("list", help="List all registered pipelines.")
 @list_options(PipelineFilterModel)
 def list_pipelines(**kwargs: Any) -> None:
-    """List all registered pipelines."""
+    """List all registered pipelines.
+
+    Args:
+        **kwargs: Keyword arguments to filter pipelines.
+    """
     cli_utils.print_active_config()
     client = Client()
     with console.status("Listing pipelines...\n"):
@@ -180,7 +184,11 @@ def runs() -> None:
 @runs.command("list", help="List all registered pipeline runs.")
 @list_options(PipelineRunFilterModel)
 def list_pipeline_runs(**kwargs: Any) -> None:
-    """List all registered pipeline runs for the filter."""
+    """List all registered pipeline runs for the filter.
+
+    Args:
+        **kwargs: Keyword arguments to filter pipeline runs.
+    """
     cli_utils.print_active_config()
 
     client = Client()
diff --git a/src/zenml/cli/role.py b/src/zenml/cli/role.py
@@ -35,7 +35,11 @@ def role() -> None:
 @role.command("list")
 @list_options(RoleFilterModel)
 def list_roles(**kwargs: Any) -> None:
-    """List all roles that fulfill the filter requirements."""
+    """List all roles that fulfill the filter requirements.
+
+    Args:
+        **kwargs: Keyword arguments to filter the list of roles.
+    """
     cli_utils.print_active_config()
     client = Client()
     with console.status("Listing roles...\n"):
@@ -287,7 +291,11 @@ def assignment() -> None:
 @assignment.command("list")
 @list_options(UserRoleAssignmentFilterModel)
 def list_role_assignments(**kwargs: Any) -> None:
-    """List all user role assignments that fulfill the filter requirements."""
+    """List all user role assignments that fulfill the filter requirements.
+
+    Args:
+        kwargs: Keyword arguments.
+    """
     cli_utils.print_active_config()
     client = Client()
     with console.status("Listing roles...\n"):
diff --git a/src/zenml/cli/stack.py b/src/zenml/cli/stack.py
@@ -685,7 +685,11 @@ def rename_stack(
 @stack.command("list")
 @list_options(StackFilterModel)
 def list_stacks(**kwargs: Any) -> None:
-    """List all stacks that fulfill the filter requirements."""
+    """List all stacks that fulfill the filter requirements.
+
+    Args:
+        kwargs: Keyword arguments to filter the stacks.
+    """
     client = Client()
     with console.status("Listing stacks...\n"):
         stacks = client.list_stacks(**kwargs)
diff --git a/src/zenml/cli/stack_components.py b/src/zenml/cli/stack_components.py
@@ -136,7 +136,11 @@ def generate_stack_component_list_command(
 
     @list_options(ComponentFilterModel)
     def list_stack_components_command(**kwargs: Any) -> None:
-        """Prints a table of stack components."""
+        """Prints a table of stack components.
+
+        Args:
+            kwargs: Keyword arguments to filter the components.
+        """
         client = Client()
         with console.status(f"Listing {component_type.plural}..."):
             kwargs["type"] = component_type
diff --git a/src/zenml/cli/user_management.py b/src/zenml/cli/user_management.py
@@ -75,7 +75,11 @@ def describe_user(user_name_or_id: Optional[str] = None) -> None:
 @user.command("list")
 @list_options(UserFilterModel)
 def list_users(**kwargs: Any) -> None:
-    """List all users."""
+    """List all users.
+
+    Args:
+        kwargs: Keyword arguments to filter the list of users.
+    """
     cli_utils.print_active_config()
     client = Client()
     with console.status("Listing stacks...\n"):
@@ -251,7 +255,11 @@ def team() -> None:
 @team.command("list")
 @list_options(TeamFilterModel)
 def list_teams(**kwargs: Any) -> None:
-    """List all teams that fulfill the filter requirements."""
+    """List all teams that fulfill the filter requirements.
+
+    Args:
+        kwargs: The filter options.
+    """
     cli_utils.print_active_config()
     client = Client()
 
diff --git a/src/zenml/cli/utils.py b/src/zenml/cli/utils.py
@@ -1064,7 +1064,11 @@ def warn_unsupported_non_default_workspace() -> None:
 
 
 def print_page_info(page: Page[T]) -> None:
-    """Print all information pertaining to a page to show the amount of items and pages."""
+    """Print all page information showing the number of items and pages.
+
+    Args:
+        page: The page to print the information for.
+    """
     declare(
         f"Page `({page.page}/{page.total_pages})`, `{page.total}` items "
         f"found for the applied filters."
@@ -1174,6 +1178,9 @@ def list_options(filter_model: Type[BaseFilterModel]) -> Callable[[F], F]:
 
     Args:
         filter_model: The filter model based on which to decorate the function.
+
+    Returns:
+        The inner decorator.
     """
 
     def inner_decorator(func: F) -> F:
diff --git a/src/zenml/cli/workspace.py b/src/zenml/cli/workspace.py
@@ -38,7 +38,11 @@ def workspace() -> None:
 @workspace.command("list", hidden=True)
 @list_options(WorkspaceFilterModel)
 def list_workspaces(**kwargs: Any) -> None:
-    """List all workspaces."""
+    """List all workspaces.
+
+    Args:
+        **kwargs: Keyword arguments to filter the list of workspaces.
+    """
     warn_unsupported_non_default_workspace()
     cli_utils.print_active_config()
     client = Client()
diff --git a/src/zenml/client.py b/src/zenml/client.py
@@ -789,6 +789,7 @@ def list_teams(
             created: Use to filter by time of creation
             updated: Use the last updated date for filtering
             name: Use the team name for filtering
+
         Returns:
             The Team
         """
@@ -947,6 +948,7 @@ def list_roles(
             created: Use to filter by time of creation
             updated: Use the last updated date for filtering
             name: Use the role name for filtering
+
         Returns:
             The Role
         """
@@ -1073,9 +1075,6 @@ def get_user_role_assignment(
 
         Returns:
             The role assignment.
-
-        Raises:
-            RuntimeError: If the role assignment does not exist.
         """
         return self.zen_store.get_user_role_assignment(
             user_role_assignment_id=role_assignment_id
@@ -1182,9 +1181,6 @@ def get_team_role_assignment(
 
         Returns:
             The role assignment.
-
-        Raises:
-            RuntimeError: If the role assignment does not exist.
         """
         return self.zen_store.get_team_role_assignment(
             team_role_assignment_id=team_role_assignment_id
@@ -1362,6 +1358,7 @@ def list_workspaces(
             created: Use to filter by time of creation
             updated: Use the last updated date for filtering
             name: Use the team name for filtering
+
         Returns:
             The Team
         """
@@ -1773,6 +1770,7 @@ def activate_stack(
 
         Raises:
             KeyError: If the stack is not registered.
+            ZenKeyError: If the stack is not registered.
         """
         # Make sure the stack is registered
         try:
@@ -2797,6 +2795,7 @@ def list_run_steps(
             cache_key: The cache_key of the run to filter by.
             status: The name of the run to filter by.
             num_outputs: The number of outputs for the step run
+
         Returns:
             A page with Pipeline fitting the filter description
         """
diff --git a/src/zenml/integrations/great_expectations/visualizers/ge_visualizer.py b/src/zenml/integrations/great_expectations/visualizers/ge_visualizer.py
@@ -16,7 +16,7 @@
 
 from typing import Any, cast
 
-import great_expectations as ge  # type: ignore[import]
+import great_expectations as ge  # type: ignore[import] # noqa
 from great_expectations.checkpoint.types.checkpoint_result import (  # type: ignore[import]
     CheckpointResult,
 )
diff --git a/src/zenml/materializers/pandas_materializer.py b/src/zenml/materializers/pandas_materializer.py
@@ -45,7 +45,7 @@ def __init__(self, uri: str):
         """
         super().__init__(uri)
         try:
-            import pyarrow  # type: ignore
+            import pyarrow  # type: ignore # noqa
 
             self.pyarrow_exists = True
         except ImportError:
diff --git a/src/zenml/models/component_models.py b/src/zenml/models/component_models.py
@@ -128,7 +128,11 @@ class ComponentFilterModel(ShareableWorkspaceScopedFilterModel):
     user_id: Union[UUID, str] = Field(None, description="User of the stack")
 
     def set_scope_type(self, component_type: str) -> None:
-        """Set the type of component on which to perform the filtering to scope the response."""
+        """Set the type of component on which to perform the filtering to scope the response.
+
+        Args:
+            component_type: The type of component to scope the query to.
+        """
         self.scope_type = component_type
 
     def generate_filter(
diff --git a/src/zenml/models/filter_models.py b/src/zenml/models/filter_models.py
diff --git a/src/zenml/models/page_model.py b/src/zenml/models/page_model.py
diff --git a/src/zenml/post_execution/base_view.py b/src/zenml/post_execution/base_view.py
diff --git a/src/zenml/steps/base_step.py b/src/zenml/steps/base_step.py
diff --git a/src/zenml/zen_stores/rest_zen_store.py b/src/zenml/zen_stores/rest_zen_store.py
diff --git a/src/zenml/zen_stores/sql_zen_store.py b/src/zenml/zen_stores/sql_zen_store.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+[darglint]`
	`2`	`+docstring_style=google`
Original file line number	Diff line number	Diff line change
`@@ -205,7 +205,7 @@ def generate_docs(`
`205`	`205`	`)`
`206`	`206`
`207`	`207`	`integration_file_contents = [INTEGRATION_DOCS_TITLE]`
`208`		`- integrations_file_content = create_entity_docs(`
	`208`	`+ create_entity_docs(`
`209`	`209`	`api_doc_file_dir=integrations_dev_doc_file_dir,`
`210`	`210`	`ignored_modules=["__init__.py", "__pycache__"],`
`211`	`211`	`sources_path=path / "integrations",`