|
| 1 | +# (C) Datadog, Inc. 2023-present |
| 2 | +# All rights reserved |
| 3 | +# Licensed under a 3-clause BSD style license (see LICENSE) |
| 4 | +from __future__ import annotations |
| 5 | + |
| 6 | +from typing import TYPE_CHECKING |
| 7 | + |
| 8 | +import click |
| 9 | + |
| 10 | +if TYPE_CHECKING: |
| 11 | + from ddev.cli.application import Application |
| 12 | + from ddev.integration.core import Integration |
| 13 | + from ddev.utils.fs import Path |
| 14 | + |
| 15 | + |
| 16 | +def fix_coverage_report(report_file: Path): |
| 17 | + target_dir = report_file.parent.name |
| 18 | + report = report_file.read_bytes() |
| 19 | + |
| 20 | + # Make every target's `tests` directory path unique so they don't get combined in UI |
| 21 | + report = report.replace(b'"tests/', f'"{target_dir}/tests/'.encode('utf-8')) |
| 22 | + |
| 23 | + report_file.write_bytes(report) |
| 24 | + |
| 25 | + |
| 26 | +@click.command(short_help='Run tests') |
| 27 | +@click.argument('target_spec', required=False) |
| 28 | +@click.argument('args', nargs=-1) |
| 29 | +@click.option('--lint', '-s', is_flag=True, help='Run only lint & style checks') |
| 30 | +@click.option('--fmt', '-fs', is_flag=True, help='Run only the code formatter') |
| 31 | +@click.option('--bench', '-b', is_flag=True, help='Run only benchmarks') |
| 32 | +@click.option('--latest', is_flag=True, help='Only verify support of new product versions') |
| 33 | +@click.option('--cov', '-c', 'coverage', is_flag=True, help='Measure code coverage') |
| 34 | +@click.option('--compat', is_flag=True, help='Check compatibility with the minimum allowed Agent version') |
| 35 | +@click.option('--ddtrace', is_flag=True, envvar='DDEV_TEST_ENABLE_TRACING', help='Enable tracing during test execution') |
| 36 | +@click.option('--memray', is_flag=True, help='Measure memory usage during test execution') |
| 37 | +@click.option('--recreate', '-r', is_flag=True, help='Recreate environments from scratch') |
| 38 | +@click.option('--list', '-l', 'list_envs', is_flag=True, help='Show available test environments') |
| 39 | +@click.option('--python-filter', envvar='PYTHON_FILTER', hidden=True) |
| 40 | +@click.option('--junit', is_flag=True, hidden=True) |
| 41 | +@click.option('--e2e', is_flag=True, hidden=True) |
| 42 | +@click.pass_obj |
| 43 | +def test( |
| 44 | + app: Application, |
| 45 | + target_spec: str | None, |
| 46 | + args: tuple[str, ...], |
| 47 | + lint: bool, |
| 48 | + fmt: bool, |
| 49 | + bench: bool, |
| 50 | + latest: bool, |
| 51 | + coverage: bool, |
| 52 | + compat: bool, |
| 53 | + ddtrace: bool, |
| 54 | + memray: bool, |
| 55 | + recreate: bool, |
| 56 | + list_envs: bool, |
| 57 | + python_filter: str | None, |
| 58 | + junit: bool, |
| 59 | + e2e: bool, |
| 60 | +): |
| 61 | + """ |
| 62 | + Run tests. |
| 63 | + """ |
| 64 | + import json |
| 65 | + import os |
| 66 | + import sys |
| 67 | + |
| 68 | + from ddev.repo.constants import PYTHON_VERSION |
| 69 | + from ddev.testing.constants import EndToEndEnvVars, TestEnvVars |
| 70 | + from ddev.utils.ci import running_in_ci |
| 71 | + |
| 72 | + if target_spec is None: |
| 73 | + target_spec = 'changed' |
| 74 | + |
| 75 | + target_name, _, environments = target_spec.partition(':') |
| 76 | + |
| 77 | + # target name -> target |
| 78 | + targets: dict[str, Integration] = {} |
| 79 | + if target_name == 'changed': |
| 80 | + for integration in app.repo.integrations.iter_changed(): |
| 81 | + if integration.is_testable: |
| 82 | + targets[integration.name] = integration |
| 83 | + else: |
| 84 | + try: |
| 85 | + integration = app.repo.integrations.get(target_name) |
| 86 | + except OSError: |
| 87 | + app.abort(f'Unknown target: {target_name}') |
| 88 | + |
| 89 | + if integration.is_testable: |
| 90 | + targets[integration.name] = integration |
| 91 | + |
| 92 | + if not targets: |
| 93 | + app.abort('No testable targets found') |
| 94 | + |
| 95 | + if list_envs: |
| 96 | + multiple_targets = len(targets) > 1 |
| 97 | + for target in targets.values(): |
| 98 | + with target.path.as_cwd(): |
| 99 | + if multiple_targets: |
| 100 | + app.display_header(target.display_name) |
| 101 | + |
| 102 | + app.platform.check_command([sys.executable, '-m', 'hatch', 'env', 'show']) |
| 103 | + |
| 104 | + return |
| 105 | + |
| 106 | + global_env_vars: dict[str, str] = {} |
| 107 | + |
| 108 | + hatch_verbosity = app.verbosity + 1 |
| 109 | + if hatch_verbosity > 0: |
| 110 | + global_env_vars['HATCH_VERBOSE'] = str(hatch_verbosity) |
| 111 | + elif hatch_verbosity < 0: |
| 112 | + global_env_vars['HATCH_QUIET'] = str(abs(hatch_verbosity)) |
| 113 | + |
| 114 | + api_key = app.config.org.config.get('api_key') |
| 115 | + if api_key and not (lint or fmt): |
| 116 | + global_env_vars['DD_API_KEY'] = api_key |
| 117 | + |
| 118 | + # Only enable certain functionality when running standard tests |
| 119 | + standard_tests = not (lint or fmt or bench or latest) |
| 120 | + |
| 121 | + # Keep track of environments so that they can first be removed if requested |
| 122 | + chosen_environments = [] |
| 123 | + |
| 124 | + base_command = [sys.executable, '-m', 'hatch', 'env', 'run'] |
| 125 | + if environments and not standard_tests: |
| 126 | + app.abort('Cannot specify environments when using specific functionality like linting') |
| 127 | + elif lint: |
| 128 | + chosen_environments.append('lint') |
| 129 | + base_command.extend(('--env', 'lint', '--', 'all')) |
| 130 | + elif fmt: |
| 131 | + chosen_environments.append('lint') |
| 132 | + base_command.extend(('--env', 'lint', '--', 'fmt')) |
| 133 | + elif bench: |
| 134 | + filter_data = json.dumps({'benchmark-env': True}) |
| 135 | + base_command.extend(('--filter', filter_data, '--', 'benchmark')) |
| 136 | + elif latest: |
| 137 | + filter_data = json.dumps({'latest-env': True}) |
| 138 | + base_command.extend(('--filter', filter_data, '--', 'test', '--run-latest-metrics')) |
| 139 | + else: |
| 140 | + if environments: |
| 141 | + for env_name in environments.split(','): |
| 142 | + chosen_environments.append(env_name) |
| 143 | + base_command.extend(('--env', env_name)) |
| 144 | + else: |
| 145 | + chosen_environments.append('default') |
| 146 | + base_command.append('--ignore-compat') |
| 147 | + |
| 148 | + if python_filter: |
| 149 | + filter_data = json.dumps({'python': python_filter}) |
| 150 | + base_command.extend(('--filter', filter_data)) |
| 151 | + |
| 152 | + base_command.extend(('--', 'test-cov' if coverage else 'test')) |
| 153 | + |
| 154 | + if app.verbosity <= 0: |
| 155 | + base_command.extend(('--tb', 'short')) |
| 156 | + |
| 157 | + if memray: |
| 158 | + if app.platform.windows: |
| 159 | + app.abort('Memory profiling with `memray` is not supported on Windows') |
| 160 | + |
| 161 | + base_command.append('--memray') |
| 162 | + |
| 163 | + if e2e: |
| 164 | + base_command.extend(('-m', 'e2e')) |
| 165 | + global_env_vars[EndToEndEnvVars.PARENT_PYTHON] = sys.executable |
| 166 | + |
| 167 | + app.display_debug(f'Targets: {", ".join(targets)}') |
| 168 | + for target in targets.values(): |
| 169 | + app.display_header(target.display_name) |
| 170 | + |
| 171 | + command = base_command.copy() |
| 172 | + env_vars = global_env_vars.copy() |
| 173 | + |
| 174 | + if standard_tests: |
| 175 | + if ddtrace and (target.is_integration or target.name == 'datadog_checks_base'): |
| 176 | + # TODO: remove this once we drop Python 2 |
| 177 | + if app.platform.windows and ( |
| 178 | + (python_filter and python_filter != PYTHON_VERSION) |
| 179 | + or not all(env_name.startswith('py3') for env_name in chosen_environments) |
| 180 | + ): |
| 181 | + app.display_warning('Tracing is only supported on Python 3 on Windows') |
| 182 | + else: |
| 183 | + command.append('--ddtrace') |
| 184 | + env_vars['DDEV_TRACE_ENABLED'] = 'true' |
| 185 | + env_vars['DD_PROFILING_ENABLED'] = 'true' |
| 186 | + env_vars['DD_SERVICE'] = os.environ.get('DD_SERVICE', 'ddev-integrations') |
| 187 | + env_vars['DD_ENV'] = os.environ.get('DD_ENV', 'ddev-integrations') |
| 188 | + |
| 189 | + if junit: |
| 190 | + # In order to handle multiple environments the report files must contain the environment name. |
| 191 | + # Hatch injects the `HATCH_ENV_ACTIVE` environment variable, see: |
| 192 | + # https://hatch.pypa.io/latest/plugins/environment/reference/#hatch.env.plugin.interface.EnvironmentInterface.get_env_vars |
| 193 | + command.extend(('--junit-xml', f'.junit/test-{"e2e" if e2e else "unit"}-$HATCH_ENV_ACTIVE.xml')) |
| 194 | + # Test results class prefix |
| 195 | + command.extend(('--junit-prefix', target.name)) |
| 196 | + |
| 197 | + if ( |
| 198 | + compat |
| 199 | + and target.is_package |
| 200 | + and target.is_integration |
| 201 | + and target.minimum_base_package_version is not None |
| 202 | + ): |
| 203 | + env_vars[TestEnvVars.BASE_PACKAGE_VERSION] = target.minimum_base_package_version |
| 204 | + |
| 205 | + command.extend(args) |
| 206 | + |
| 207 | + with target.path.as_cwd(env_vars=env_vars): |
| 208 | + app.display_debug(f'Command: {command}') |
| 209 | + |
| 210 | + if recreate: |
| 211 | + if bench or latest: |
| 212 | + variable = 'benchmark-env' if bench else 'latest-env' |
| 213 | + env_data = json.loads( |
| 214 | + app.platform.check_command_output([sys.executable, '-m', 'hatch', 'env', 'show', '--json']) |
| 215 | + ) |
| 216 | + for env_name, env_config in env_data.items(): |
| 217 | + if env_config.get(variable, False): |
| 218 | + app.platform.check_command([sys.executable, '-m', 'hatch', 'env', 'remove', env_name]) |
| 219 | + else: |
| 220 | + for env_name in chosen_environments: |
| 221 | + app.platform.check_command([sys.executable, '-m', 'hatch', 'env', 'remove', env_name]) |
| 222 | + |
| 223 | + app.platform.check_command(command) |
| 224 | + if standard_tests and coverage: |
| 225 | + app.display_header('Coverage report') |
| 226 | + app.platform.check_command([sys.executable, '-m', 'coverage', 'report', '--rcfile=../.coveragerc']) |
| 227 | + |
| 228 | + if running_in_ci(): |
| 229 | + app.platform.check_command( |
| 230 | + [sys.executable, '-m', 'coverage', 'xml', '-i', '--rcfile=../.coveragerc'] |
| 231 | + ) |
| 232 | + fix_coverage_report(target.path / 'coverage.xml') |
| 233 | + else: |
| 234 | + (target.path / '.coverage').remove() |
0 commit comments