Move GitDagBundle to providers (apache#47636)

* Move `GitDagBundle` to providers

This commit moves GitDagBundle into the providers package. As part of
the move, a new `GIT` provider is created and like `operators`, `transfers`
etc, we now have `bundles` package. The bundle package is where the `GitDagBundles`
lives.

* fixup! Move `GitDagBundle` to providers

* Update documentation

* Fix static checks

* add git to compatibility tests matrix

* Update doctree

* Update with recent changes

* Fix doc, update version and add provider to prod image preinstalled providers

* fixup! Fix doc, update version and add provider to prod image preinstalled providers

* Fix doc

* update breeze command hash

* Fix incompatible providers regex removal syntax

* Remove providers mistakenly added in pyproject.toml

* Update providers/git/pyproject.toml

Author: ephraimbuddy

.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml

@@ -65,6 +65,7 @@ body:
         - fab
         - facebook
         - ftp
+        - git
         - github
         - google
         - grpc

.github/workflows/test-providers.yml

@@ -205,7 +205,7 @@ jobs:
         run: |
           for provider in ${REMOVE_PROVIDERS}; do
             echo "Removing incompatible provider: ${provider}"
-            rm -vf dist/apache_airflow_providers_${provider/./_}*
+            rm -vf dist/apache_airflow_providers_${provider/./_}-*
           done
         if: matrix.remove-providers != ''
       - name: "Download airflow package: wheel"

INSTALL

@@ -321,7 +321,7 @@ airbyte, alibaba, amazon, apache.beam, apache.cassandra, apache.drill, apache.dr
 apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka, apache.kylin, apache.livy,
 apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana, atlassian.jira, celery, cloudant,
 cncf.kubernetes, cohere, common.compat, common.io, common.messaging, common.sql, databricks,
-datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp,
+datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp, git,
 github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins, microsoft.azure,
 microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc, openai, openfaas,
 openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector, pinecone, postgres,

airflow-core/docs/extra-packages-ref.rst

@@ -266,6 +266,8 @@ Some of those enable Airflow to use executors to run tasks with them - other tha
 +---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
 | fab                 | ``pip install 'apache-airflow[fab]'``               | FAB auth manager                                                |                                              |
 +---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
+| git                 | ``pip install 'apache-airflow[git]'``               | Git bundle and hook                                             |                                              |
++---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
 | github              | ``pip install 'apache-airflow[github]'``            | GitHub operators and hook                                       |                                              |
 +---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
 | influxdb            | ``pip install 'apache-airflow[influxdb]'``          | Influxdb operators and hook                                     |                                              |

airflow-core/src/airflow/config_templates/config.yml

@@ -2478,7 +2478,7 @@ dag_processor:
         [
             {
                 "name": "my-git-repo",
-                "classpath": "airflow.dag_processing.bundles.git.GitDagBundle",
+                "classpath": "airflow.providers.git.bundles.git.GitDagBundle",
                 "kwargs": {
                     "subdir": "dags",
                     "tracking_ref": "main",

airflow-core/src/airflow/dag_processing/bundles/provider.yaml

@@ -16,10 +16,10 @@
 # under the License.
 
 ---
-package-name: apache-airflow-providers-bundles
+package-name: apache-airflow-providers-git
 name: GIT
 description: |
-    `GIT <https://git-scm.com/>`__
+    `Distributed version control system (GIT) <https://git-scm.com/>`__
 
 state: not-ready
 source-date-epoch: 1726861127
@@ -29,13 +29,19 @@ versions:
 
 integrations:
   - integration-name: GIT (Git)
+    external-doc-url: https://git-scm.com/
+    tags: [software]
 
 hooks:
   - integration-name: GIT
     python-modules:
-      - airflow.dag_processing.bundles.git
+      - airflow.providers.git.hooks.git
+bundles:
+  - integration-name: GIT
+    python-modules:
+      - airflow.providers.git.bundles.git
 
 
 connection-types:
-  - hook-class-name: airflow.dag_processing.bundles.git.GitHook
+  - hook-class-name: airflow.providers.git.hooks.git.GitHook
     connection-type: git

airflow-core/src/airflow/provider.yaml.schema.json

@@ -313,6 +313,30 @@
                 ]
             }
         },
+        "bundles": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "integration-name": {
+                        "type": "string",
+                        "description": "Integration name. It must have a matching item in the 'integration' section of any provider."
+                    },
+                    "python-modules": {
+                        "description": "List of python modules containing the bundles.",
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "additionalProperties": false,
+                "required": [
+                    "integration-name",
+                    "python-modules"
+                ]
+            }
+        },
         "connection-types": {
             "type": "array",
             "description": "Array of connection types mapped to hook class names",

airflow-core/src/airflow/provider_info.schema.json

@@ -86,6 +86,30 @@
                 ]
             }
         },
+        "bundles": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "integration-name": {
+                        "type": "string",
+                        "description": "Integration name. It must have a matching item in the 'integration' section of any provider."
+                    },
+                    "python-modules": {
+                        "description": "List of python modules containing the bundles.",
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "additionalProperties": false,
+                "required": [
+                    "integration-name",
+                    "python-modules"
+                ]
+            }
+        },
         "connection-types": {
             "type": "array",
             "description": "Map of connection types mapped to hook class names.",

airflow-core/src/airflow/providers_manager.py

@@ -171,9 +171,6 @@ def _create_customized_form_field_behaviours_schema_validator():
 
 
 def _check_builtin_provider_prefix(provider_package: str, class_name: str) -> bool:
-    if "bundles" in provider_package:
-        # TODO: AIP-66: remove this when this package is moved to providers directory
-        return True
     if provider_package.startswith("apache-airflow"):
         provider_path = provider_package[len("apache-") :].replace("-", ".")
         if not class_name.startswith(provider_path):

airflow-core/tests/unit/api_fastapi/conftest.py

@@ -115,7 +115,7 @@ def configure_git_connection_for_dag_bundle(session):
             (
                 "dag_processor",
                 "dag_bundle_config_list",
-            ): '[{ "name": "dag_maker", "classpath": "airflow.dag_processing.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}, { "name": "another_bundle_name", "classpath": "airflow.dag_processing.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}]'
+            ): '[{ "name": "dag_maker", "classpath": "airflow.providers.git.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}, { "name": "another_bundle_name", "classpath": "airflow.providers.git.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}]'
         }
     ):
         yield