From 5627522c6a11c7e9f3c1f7f2ddc721fd84cae198 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 17 Feb 2023 18:08:45 +0100 Subject: [PATCH] Add "all-testable" integration alias (#29520) The "all" alias outgrew original meaning with adding statsd integration in #29449 which resulted in a problem fixed by #29517. The "all" tests as result was not "all" but "all that could be tested". This PR adds "all-testable" alias which as opposed to "all" does not contain "statsd" - it also renames some of the vars/arrays to reflect it.7 --- .github/workflows/ci.yml | 8 +- BREEZE.rst | 3 +- TESTING.rst | 13 ++- .../src/airflow_breeze/global_constants.py | 15 ++- .../src/airflow_breeze/params/shell_params.py | 5 +- .../airflow_breeze/utils/common_options.py | 4 +- images/breeze/output-commands-hash.txt | 12 +-- images/breeze/output-commands.svg | 78 ++++++++-------- images/breeze/output_shell.svg | 90 +++++++++--------- images/breeze/output_start-airflow.svg | 92 ++++++++++--------- .../output_testing_integration-tests.svg | 64 +++++++------ images/breeze/output_testing_tests.svg | 64 +++++++------ 12 files changed, 244 insertions(+), 204 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23e795d3823d9..46ea914a2f687 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1058,8 +1058,8 @@ jobs: breeze testing integration-tests --integration trino --integration kerberos breeze stop if: needs.build-info.outputs.runs-on != 'self-hosted' - - name: "Integration Tests Postgres: all" - run: breeze testing integration-tests --integration all + - name: "Integration Tests Postgres: all-testable" + run: breeze testing integration-tests --integration all-testable if: needs.build-info.outputs.runs-on == 'self-hosted' - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}" uses: ./.github/actions/post_tests @@ -1096,8 +1096,8 @@ jobs: - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image if: needs.build-info.outputs.runs-on == 'self-hosted' - - name: "Integration Tests MySQL: all" - run: breeze testing integration-tests --integration all + - name: "Integration Tests MySQL: all-testable" + run: breeze testing integration-tests --integration all-testable if: needs.build-info.outputs.runs-on == 'self-hosted' - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}" uses: ./.github/actions/post_tests diff --git a/BREEZE.rst b/BREEZE.rst index ffbb5fa2e1a48..245bdb60d5cba 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1816,7 +1816,8 @@ By default Breeze starts only airflow container without any integration enabled. that is selected). You can start the additional integrations by passing ``--integration`` flag with appropriate integration name when starting Breeze. You can specify several ``--integration`` flags to start more than one integration at a time. -Finally you can specify ``--integration all`` to start all integrations. +Finally you can specify ``--integration all-testable`` to start all testable integrations and +``--integration all`` to enable all integrations. Once integration is started, it will continue to run until the environment is stopped with ``breeze stop`` command. or restarted via ``breeze restart`` command diff --git a/TESTING.rst b/TESTING.rst index ab62885f6511a..c38d20f9671fa 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -461,7 +461,8 @@ environment with enabled integrations and in the CI. See `CI `_ for deta When you are in the Breeze environment, by default, all integrations are disabled. This enables only true unit tests to be executed in Breeze. You can enable the integration by passing the ``--integration `` switch when starting Breeze. You can specify multiple integrations by repeating the ``--integration`` switch -or using the ``--integration all`` switch that enables all integrations. +or using the ``--integration all-testable`` switch that enables all testable integrations and +``--integration all`` switch that enables all integrations. NOTE: Every integration requires a separate container with the corresponding integration image. These containers take precious resources on your PC, mainly the memory. The started integrations are not stopped @@ -500,11 +501,17 @@ To start ``mongo`` and ``cassandra`` integrations, enter: breeze --integration mongo --integration cassandra +To start all testable integrations, enter: + +.. code-block:: bash + + breeze --integration all-testable + To start all integrations, enter: .. code-block:: bash - breeze --integration all + breeze --integration all-testable Note that Kerberos is a special kind of integration. Some tests run differently when Kerberos integration is enabled (they retrieve and use a Kerberos authentication token) and differently when the @@ -569,7 +576,7 @@ Runs all integration tests: .. code-block:: bash - breeze testing integration-tests --db-reset --integration all + breeze testing integration-tests --db-reset --integration all-testable Runs all mongo DB tests: diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 589c51af4631f..16890305c6ffe 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -42,7 +42,7 @@ ALLOWED_BACKENDS = ["sqlite", "mysql", "postgres", "mssql"] ALLOWED_PROD_BACKENDS = ["mysql", "postgres", "mssql"] DEFAULT_BACKEND = ALLOWED_BACKENDS[0] -ALL_INTEGRATIONS = [ +TESTABLE_INTEGRATIONS = [ "cassandra", "celery", "kerberos", @@ -50,14 +50,23 @@ "pinot", "trino", ] -ALLOWED_INTEGRATIONS = sorted( +OTHER_INTEGRATIONS = ["statsd"] +ALL_INTEGRATIONS = sorted( [ - *ALL_INTEGRATIONS, + *TESTABLE_INTEGRATIONS, + *OTHER_INTEGRATIONS, + ] +) +AUTOCOMPLETE_INTEGRATIONS = sorted( + [ + "all-testable", "all", "otel", "statsd", + *ALL_INTEGRATIONS, ] ) + ALLOWED_KUBERNETES_VERSIONS = ["v1.23.13", "v1.24.7", "v1.25.3", "v1.26.0"] ALLOWED_EXECUTORS = ["KubernetesExecutor", "CeleryExecutor", "LocalExecutor", "CeleryKubernetesExecutor"] ALLOWED_KIND_OPERATIONS = ["start", "stop", "restart", "status", "deploy", "test", "shell", "k9s"] diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index c864280364b68..a28d62bcad553 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -37,6 +37,7 @@ MOUNT_REMOVE, MOUNT_SELECTED, MOUNT_SKIP, + TESTABLE_INTEGRATIONS, get_airflow_version, ) from airflow_breeze.utils.console import get_console @@ -237,7 +238,9 @@ def compose_file(self) -> str: compose_file_list.append(DOCKER_COMPOSE_DIR / "remove-sources.yml") if self.include_mypy_volume: compose_file_list.append(DOCKER_COMPOSE_DIR / "mypy.yml") - if "all" in self.integration: + if "all-testable" in self.integration: + integrations = TESTABLE_INTEGRATIONS + elif "all" in self.integration: integrations = ALL_INTEGRATIONS else: integrations = self.integration diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 9caf98b007dd1..1d6459b7349e4 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -27,7 +27,6 @@ ALLOWED_CONSTRAINTS_MODES_CI, ALLOWED_CONSTRAINTS_MODES_PROD, ALLOWED_INSTALLATION_PACKAGE_FORMATS, - ALLOWED_INTEGRATIONS, ALLOWED_MOUNT_OPTIONS, ALLOWED_MSSQL_VERSIONS, ALLOWED_MYSQL_VERSIONS, @@ -37,6 +36,7 @@ ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, ALLOWED_USE_AIRFLOW_VERSIONS, APACHE_AIRFLOW_GITHUB_REPOSITORY, + AUTOCOMPLETE_INTEGRATIONS, SINGLE_PLATFORMS, get_available_documentation_packages, ) @@ -131,7 +131,7 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, option_integration = click.option( "--integration", help="Integration(s) to enable when running (can be more than one).", - type=BetterChoice(ALLOWED_INTEGRATIONS), + type=BetterChoice(AUTOCOMPLETE_INTEGRATIONS), multiple=True, ) option_postgres_version = click.option( diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 874a992d15c0f..598326d6eb7cb 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,7 +1,7 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:2e1b4e5838838f513750d1352798b006 +main:83de6a9bf2b1afecd1f9ce4cd0493733 build-docs:18235f12f85f8df82f3eb245e429f62d ci:find-newer-dependencies:8fa2b57f5f0523c928743b235ee3ab5a ci:fix-ownership:fee2c9ec9ef19686792002ae054fecdd @@ -53,12 +53,12 @@ setup:regenerate-command-images:15215e52342dd2f2e27a85726f40a820 setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:123b462a421884dc2320ffc5e54b2478 setup:56a2ef337c354362760d247df5d05365 -shell:e34d2512360503b70e6a05e5b8b59986 -start-airflow:e096fa493a810935c012ef3d95775aa2 +shell:ab07ac2d57253e25367a7200ce686703 +start-airflow:5e8460ac38f8e9ea2a0ac7e248fd7bc9 static-checks:12e8fed2acbed0d823efc5121fd0eb58 stop:e5aa686b4e53707ced4039d8414d5cd6 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c testing:helm-tests:94a442e7f3f63b34c4831a84d165690a -testing:integration-tests:fa4f9be38ee380f6f8feeb9dd8ba7669 -testing:tests:8dd0496fddb37b207fc2b74add201d43 -testing:1e4ea0f58b4814342911355d35efd966 +testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98 +testing:tests:ce6d80b3073bafd1f10c50812432885a +testing:0f292a29b2a97fbd95778a72249ee997 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 816cc9693af74..790b9fe55da49 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - +