feat(source-google-drive): implementation to sync records (#52099)

New mode to sync File Permissions (ACLs) and Domain Identities.

airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml

@@ -43,5 +43,5 @@ acceptance_tests:
       - spec_path: integration_tests/spec.json
         # changed from "Document File Type Format (Experimental)" to "Unstructured Document Format".
         backward_compatibility_tests_config:
-          disable_for_version: "0.1.0-rc.1"
+          disable_for_version: "0.2.0"
 connector_image: airbyte/source-google-drive:dev

airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json

@@ -400,6 +400,34 @@
             },
             "description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.",
             "required": ["delivery_type"]
+          },
+          {
+            "title": "Replicate Permissions ACL",
+            "type": "object",
+            "properties": {
+              "delivery_type": {
+                "title": "Delivery Type",
+                "default": "use_permissions_transfer",
+                "const": "use_permissions_transfer",
+                "enum": ["use_permissions_transfer"],
+                "type": "string"
+              },
+              "include_identities_stream": {
+                "title": "Include Identity Stream",
+                "description": "This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
+                "default": true,
+                "type": "boolean"
+              },
+              "domain": {
+                "title": "Domain",
+                "description": "The Google domain of the identities.",
+                "airbyte_hidden": false,
+                "order": 1,
+                "type": "string"
+              }
+            },
+            "description": "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source.",
+            "required": ["delivery_type"]
           }
         ]
       },

airbyte-integrations/connectors/source-google-drive/metadata.yaml

@@ -7,7 +7,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.2.0
   dockerRepository: airbyte/source-google-drive
   githubIssueLabel: source-google-drive
   icon: google-drive.svg

airbyte-integrations/connectors/source-google-drive/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.1.2"
+version = "0.2.0"
 name = "source-google-drive"
 description = "Source implementation for Google Drive."
 authors = [ "Airbyte <[email protected]>",]
@@ -21,7 +21,7 @@ google-api-python-client = "==2.104.0"
 google-auth-httplib2 = "==0.1.1"
 google-auth-oauthlib = "==1.1.0"
 google-api-python-client-stubs = "==1.18.0"
-airbyte-cdk = {extras = ["file-based"], version = "^6.18.2"}
+airbyte-cdk = {extras = ["file-based"], version = "^6.33.6"}
 
 
 [tool.poetry.scripts]

airbyte-integrations/connectors/source-google-drive/source_google_drive/schemas/file_permissions.json

@@ -0,0 +1,12 @@
+{
+  "type": "object",
+  "properties": {
+    "id": { "type": "string" },
+    "file_path": { "type": "string" },
+    "allowed_identity_remote_ids": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "publicly_accessible": { "type": "boolean" }
+  }
+}

airbyte-integrations/connectors/source-google-drive/source_google_drive/schemas/identities.json

@@ -0,0 +1,14 @@
+{
+  "type": "object",
+  "properties": {
+    "id": { "type": "string" },
+    "remote_id": { "type": "string" },
+    "parent_id": { "type": ["null", "string"] },
+    "name": { "type": ["null", "string"] },
+    "description": { "type": ["null", "string"] },
+    "email_address": { "type": ["null", "string"] },
+    "member_email_addresses": { "type": ["null", "array"] },
+    "type": { "type": "string" },
+    "modified_at": { "type": "string" }
+  }
+}

airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py

@@ -6,7 +6,7 @@
 from typing import Any, Mapping, Optional
 
 from airbyte_cdk import AdvancedAuth, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification, TState
-from airbyte_cdk.models import AuthFlowType
+from airbyte_cdk.models import AuthFlowType, OauthConnectorInputSpecification
 from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource
 from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor
 from source_google_drive.spec import SourceGoogleDriveSpec
@@ -28,6 +28,11 @@ def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
         """
         Returns the specification describing what fields can be configured by a user when setting up a file-based source.
         """
+        oauth_connector_input_specification = OauthConnectorInputSpecification(
+            consent_url="https://accounts.google.com/o/oauth2/v2/auth?{{client_id_param}}&{{redirect_uri_param}}&response_type=code&{{scope_param}}&access_type=offline&{{state_param}}&include_granted_scopes=true&prompt=consent",
+            access_token_url="https://oauth2.googleapis.com/token?{{client_id_param}}&{{client_secret_param}}&{{auth_code_param}}&{{redirect_uri_param}}&grant_type=authorization_code",
+            scope="https://www.googleapis.com/auth/drive.readonly https://www.googleapis.com/auth/admin.directory.group.readonly https://www.googleapis.com/auth/admin.directory.group.member.readonly https://www.googleapis.com/auth/admin.directory.user.readonly",
+        )
 
         return ConnectorSpecification(
             documentationUrl=self.spec_class.documentation_url(),
@@ -37,10 +42,17 @@ def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
                 predicate_key=["credentials", "auth_type"],
                 predicate_value="Client",
                 oauth_config_specification=OAuthConfigSpecification(
+                    oauth_connector_input_specification=oauth_connector_input_specification,
                     complete_oauth_output_specification={
                         "type": "object",
                         "additionalProperties": False,
-                        "properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}},
+                        "properties": {
+                            "refresh_token": {
+                                "type": "string",
+                                "path_in_connector_config": ["credentials", "refresh_token"],
+                                "path_in_oauth_response": ["refresh_token"],
+                            }
+                        },
                     },
                     complete_oauth_server_input_specification={
                         "type": "object",

airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py

@@ -1,15 +1,52 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
-
-from typing import Any, Dict, Literal, Union
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, Literal, Optional, Union
 
 import dpath.util
 from pydantic.v1 import BaseModel, Field
 
 from airbyte_cdk import OneOfOptionConfig
-from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
+    AbstractFileBasedSpec,
+    DeliverRawFiles,
+    DeliverRecords,
+)
+from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
+    DeliverPermissions as DeliverPermissionsBase,
+)
+
+
+class RemoteIdentityType(Enum):
+    USER = "user"
+    GROUP = "group"
+
+
+class RemoteIdentity(BaseModel):
+    id: uuid.UUID
+    remote_id: str
+    parent_id: str | None = None
+    name: str | None = None
+    description: str | None = None
+    email_address: str | None = None
+    member_email_addresses: list[str] | None = None
+    type: RemoteIdentityType
+    modified_at: datetime
+
+
+class RemotePermissions(BaseModel):
+    id: str
+    file_path: str
+    allowed_identity_remote_ids: list[str] | None = None
+    denied_identity_remote_ids: list[str] | None = None
+    publicly_accessible: bool = False
+
+
+class DeliverPermissions(DeliverPermissionsBase):
+    domain: Optional[str] = Field(title="Domain", description="The Google domain of the identities.", airbyte_hidden=False, order=1)
 
 
 class OAuthCredentials(BaseModel):
@@ -60,7 +97,7 @@ class Config:
         pattern_descriptor="https://drive.google.com/drive/folders/MY-FOLDER-ID",
     )
 
-    delivery_method: DeliverRecords | DeliverRawFiles = Field(
+    delivery_method: DeliverRecords | DeliverRawFiles | DeliverPermissions = Field(
         title="Delivery Method",
         discriminator="delivery_type",
         type="object",