Add a validate labeler command (#16774)

* Add a `validate labeler` command

* Update ddev/src/ddev/cli/validate/labeler.py

Co-authored-by: Ilia Kurenkov <[email protected]>

---------

Co-authored-by: Ilia Kurenkov <[email protected]>

.github/workflows/run-validations.yml

@@ -54,6 +54,10 @@ on:
         required: false
         default: false
         type: boolean
+      labeler:
+        required: false
+        default: false
+        type: boolean
       legacy-signature:
         required: false
         default: false
@@ -223,6 +227,10 @@ jobs:
       if: inputs.typos
       run: ddev validate typos $TARGET
 
+    - name: Validate labeler configuration
+      if: inputs.labeler
+      run: ddev validate labeler
+
     # Every validation below here is sorted by increasing runtime rather than alphabetically
     - name: Validate third-party license metadata
       if: inputs.licenses

.github/workflows/validate.yml

@@ -26,6 +26,7 @@ jobs:
       imports: true
       integration-style: true
       jmx-metrics: true
+      labeler: true
       legacy-signature: true
       license-headers: true
       licenses: true

ddev/changelog.d/16774.added

@@ -0,0 +1 @@
+Add a `validate labeler` command

ddev/src/ddev/cli/validate/__init__.py

@@ -23,6 +23,7 @@
 
 from ddev.cli.validate.ci import ci
 from ddev.cli.validate.http import http
+from ddev.cli.validate.labeler import labeler
 from ddev.cli.validate.licenses import licenses
 from ddev.cli.validate.manifest import manifest
 from ddev.cli.validate.metadata import metadata
@@ -48,6 +49,7 @@ def validate():
 validate.add_command(imports)
 validate.add_command(integration_style)
 validate.add_command(jmx_metrics)
+validate.add_command(labeler)
 validate.add_command(legacy_signature)
 validate.add_command(license_headers)
 validate.add_command(licenses)

ddev/src/ddev/cli/validate/labeler.py

@@ -0,0 +1,92 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from __future__ import annotations
+
+import copy
+from typing import TYPE_CHECKING
+
+import click
+import yaml
+
+if TYPE_CHECKING:
+    from ddev.cli.application import Application
+
+
+@click.command()
+@click.option('--sync', is_flag=True, help='Update the labeler configuration')
+@click.pass_obj
+def labeler(app: Application, sync: bool):
+    """Validate labeler configuration."""
+
+    is_core = app.repo.name == 'core'
+
+    if not is_core:
+        app.display_info(
+            f"The labeler validation is only enabled for integrations-core, skipping for repo {app.repo.name}"
+        )
+        return
+
+    valid_integrations = dict.fromkeys(i.name for i in app.repo.integrations.iter("all"))
+    # Remove this when we remove the `datadog_checks_tests_helper` package
+    valid_integrations['datadog_checks_tests_helper'] = None
+
+    pr_labels_config_path = app.repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml'
+    if not pr_labels_config_path.exists():
+        app.abort('Unable to find the PR Labels config file')
+
+    pr_labels_config = yaml.safe_load(pr_labels_config_path.read_text())
+    new_pr_labels_config = copy.deepcopy(pr_labels_config)
+
+    tracker = app.create_validation_tracker('labeler')
+
+    for label in pr_labels_config:
+        if label.startswith('integration'):
+            check_name = label.removeprefix('integration/')
+            if check_name not in valid_integrations:
+                if sync:
+                    new_pr_labels_config.pop(label)
+                    app.display_info(f'Removing `{label}` only found in labeler config')
+                    continue
+                message = f'Unknown check label `{label}` found in PR labels config'
+                tracker.error((str(pr_labels_config_path),), message=message)
+
+    # Check if valid integration has a label
+    for check_name in valid_integrations:
+        integration_label = f"integration/{check_name}"
+
+        if integration_label not in pr_labels_config:
+            if sync:
+                new_pr_labels_config[integration_label] = [f'{check_name}/**/*']
+                app.display_info(f'Adding config for `{check_name}`')
+                continue
+
+            message = f'Check `{check_name}` does not have an integration PR label'
+            tracker.error((str(pr_labels_config_path),), message=message)
+            continue
+
+        # Check if label config is properly configured
+        integration_label_config = pr_labels_config.get(integration_label)
+        if integration_label_config != [f'{check_name}/**/*']:
+            if sync:
+                new_pr_labels_config[integration_label] = [f'{check_name}/**/*']
+                app.display_info(f"Fixing label config for `{check_name}`")
+                continue
+            message = (
+                f'Integration PR label `{integration_label}` is not properly configured: `{integration_label_config}`'
+            )
+            tracker.error((str(pr_labels_config_path),), message=message)
+
+    if sync:
+        output = yaml.safe_dump(new_pr_labels_config, default_flow_style=False, sort_keys=True)
+        pr_labels_config_path.write_text(output)
+        app.display_info(f'Successfully fixed {pr_labels_config_path}')
+
+    tracker.display()
+
+    if tracker.errors:  # no cov
+        message = 'Try running `ddev validate labeler --sync`'
+        app.display_info(message)
+        app.abort()
+
+    app.display_success('Labeler configuration is valid')

ddev/tests/cli/validate/conftest.py

@@ -0,0 +1,70 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+from ddev.repo.core import Repository
+
+
+def _fake_repo(tmp_path_factory, config_file, name):
+    repo_path = tmp_path_factory.mktemp(name)
+    repo = Repository(name, str(repo_path))
+
+    config_file.model.repos[name] = str(repo.path)
+    config_file.model.repo = name
+    config_file.save()
+
+    for integration in ('dummy', 'dummy2'):
+        write_file(
+            repo_path / integration,
+            'manifest.json',
+            """We don't need the content for this test, we just need the file""",
+        )
+
+    write_file(
+        repo_path / '.github' / 'workflows' / 'config',
+        'labeler.yml',
+        """changelog/no-changelog:
+- any:
+  - requirements-agent-release.txt
+  - '*/__about__.py'
+- all:
+  - '!*/datadog_checks/**'
+  - '!*/pyproject.toml'
+  - '!ddev/src/**'
+integration/datadog_checks_tests_helper:
+- datadog_checks_tests_helper/**/*
+integration/dummy:
+- dummy/**/*
+integration/dummy2:
+- dummy2/**/*
+release:
+- '*/__about__.py'
+""",
+    )
+
+    return repo
+
+
+@pytest.fixture
+def fake_repo(
+    tmp_path_factory,
+    config_file,
+):
+    yield _fake_repo(tmp_path_factory, config_file, 'core')
+
+
+@pytest.fixture
+def fake_extras_repo(tmp_path_factory, config_file):
+    yield _fake_repo(tmp_path_factory, config_file, 'extras')
+
+
+@pytest.fixture
+def fake_marketplace_repo(tmp_path_factory, config_file):
+    yield _fake_repo(tmp_path_factory, config_file, 'marketplace')
+
+
+def write_file(folder, file, content):
+    folder.mkdir(exist_ok=True, parents=True)
+    file_path = folder / file
+    file_path.write_text(content)

ddev/tests/cli/validate/test_labeler.py

@@ -0,0 +1,167 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import os
+
+import pytest
+import yaml
+
+
+@pytest.mark.parametrize('repo_fixture', ['fake_extras_repo', 'fake_marketplace_repo'])
+def test_labeler_config_not_integrations_core(repo_fixture, ddev, config_file, request):
+    fixture = request.getfixturevalue(repo_fixture)
+    os.remove(fixture.path / '.github' / 'workflows' / 'config' / 'labeler.yml')
+    result = ddev('validate', 'labeler')
+    assert result.exit_code == 0, result.output
+
+
+def test_labeler_config_does_not_exist(fake_repo, ddev):
+    os.remove(fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml')
+    result = ddev('validate', 'labeler')
+
+    assert result.exit_code == 1, result.output
+    assert "Unable to find the PR Labels config file" in result.output
+
+
+def test_labeler_unknown_integration_in_config_file(fake_repo, ddev):
+    (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').write_text(
+        labeler_test_config(["dummy", "dummy2", 'dummy3'])
+    )
+
+    result = ddev('validate', 'labeler')
+
+    assert result.exit_code == 1, result.output
+    assert "Unknown check label `integration/dummy3` found in PR labels config" in result.output
+
+
+def test_labeler_integration_not_in_config_file(fake_repo, ddev):
+    (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').write_text(labeler_test_config(["dummy"]))
+
+    result = ddev('validate', 'labeler')
+
+    assert result.exit_code == 1, result.output
+    assert "Check `dummy2` does not have an integration PR label" in result.output
+
+
+def test_labeler_invalid_configuration(fake_repo, ddev):
+    (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').write_text(
+        """changelog/no-changelog:
+- any:
+  - requirements-agent-release.txt
+  - '*/__about__.py'
+- all:
+  - '!*/datadog_checks/**'
+  - '!*/pyproject.toml'
+  - '!ddev/src/**'
+integration/datadog_checks_tests_helper:
+- datadog_checks_tests_helper/**/*
+integration/dummy:
+- dummy/**/*
+integration/dummy2:
+- something
+release:
+- '*/__about__.py'
+    """,
+    )
+
+    result = ddev('validate', 'labeler')
+
+    assert result.exit_code == 1, result.output
+    assert (
+        "Integration PR label `integration/dummy2` is not properly configured: \n    `['something']`" in result.output
+    )
+
+
+def test_labeler_valid_configuration(fake_repo, ddev):
+    result = ddev('validate', 'labeler')
+
+    assert result.exit_code == 0, result.output
+    assert "Labeler configuration is valid" in result.output
+
+
+def test_labeler_sync_remove_integration_in_config(fake_repo, ddev):
+    (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').write_text(
+        labeler_test_config(["dummy", "dummy2", "dummy3"])
+    )
+
+    result = ddev('validate', 'labeler', '--sync')
+
+    assert result.exit_code == 0, result.output
+    assert 'Removing `integration/dummy3` only found in labeler config' in result.output
+    assert 'Labeler configuration is valid' in result.output
+    assert 'Successfully fixed' in result.output
+
+    assert (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').read_text() == labeler_test_config(
+        ["dummy", "dummy2"]
+    )
+
+
+def test_labeler_sync_add_integration_in_config(fake_repo, ddev):
+    (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').write_text(labeler_test_config(["dummy"]))
+
+    result = ddev('validate', 'labeler', '--sync')
+
+    assert result.exit_code == 0, result.output
+    assert 'Adding config for `dummy2`' in result.output
+    assert 'Labeler configuration is valid' in result.output
+    assert 'Successfully fixed' in result.output
+
+    assert (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').read_text() == labeler_test_config(
+        ["dummy", "dummy2"]
+    )
+
+
+def test_labeler_fix_existing_integration_in_config(fake_repo, ddev):
+    (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').write_text(
+        """changelog/no-changelog:
+- any:
+  - requirements-agent-release.txt
+  - '*/__about__.py'
+- all:
+  - '!*/datadog_checks/**'
+  - '!*/pyproject.toml'
+  - '!ddev/src/**'
+integration/datadog_checks_tests_helper:
+- datadog_checks_tests_helper/**/*
+integration/dummy:
+- dummy/**/*
+integration/dummy2:
+- something
+release:
+- '*/__about__.py'
+    """,
+    )
+
+    result = ddev('validate', 'labeler', '--sync')
+
+    assert result.exit_code == 0, result.output
+    assert 'Fixing label config for `dummy2`' in result.output
+    assert 'Labeler configuration is valid' in result.output
+    assert 'Successfully fixed' in result.output
+    assert (fake_repo.path / '.github' / 'workflows' / 'config' / 'labeler.yml').read_text() == labeler_test_config(
+        ["dummy", "dummy2"]
+    )
+
+
+def labeler_test_config(integrations):
+    config = yaml.safe_load(
+        """
+changelog/no-changelog:
+- any:
+  - requirements-agent-release.txt
+  - '*/__about__.py'
+- all:
+  - '!*/datadog_checks/**'
+  - '!*/pyproject.toml'
+  - '!ddev/src/**'
+integration/datadog_checks_tests_helper:
+- datadog_checks_tests_helper/**/*
+release:
+- '*/__about__.py'
+"""
+    )
+
+    for integration in integrations:
+        config[f"integration/{integration}"] = [f"{integration}/**/*"]
+
+    return yaml.dump(config, default_flow_style=False, sort_keys=True)