Add git to filesystem source 301

pyproject.toml

@@ -12,7 +12,9 @@ packages = [{include = "sources"}]
 
 [tool.poetry.dependencies]
 python = "^3.8.1"
-dlt = {version = "^0.3.23", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb", "s3", "gs"]}
+# dlt = {version = "^0.3.23", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb", "s3", "gs"]}
+dlt = {path = "../dlt", extras = ["duckdb"], develop = true}
+gitpythonfs = {path = "./sources/filesystem/gitpythonfs", develop = true}
 
 [tool.poetry.group.dev.dependencies]
 mypy = "1.6.1"

sources/filesystem/__init__.py

@@ -73,7 +73,7 @@ def filesystem(
 
     files_chunk: List[FileItem] = []
     for file_model in glob_files(fs_client, bucket_url, file_glob):
-        file_dict = FileItemDict(file_model, credentials)
+        file_dict = FileItemDict(file_model, fs_client)
         if extract_content:
             file_dict["file_content"] = file_dict.read_bytes()
         files_chunk.append(file_dict)  # type: ignore

sources/filesystem/gitpythonfs/README.md

@@ -0,0 +1,6 @@
+# gitpythonfs
+
+Builds on [GitPython](https://gitpython.readthedocs.io/) to provide a Python filesystem interface for git.
+
+The initial use case is to load file contents from git repos into destinations using tools such as [dlt](https://dlthub.com)
+

sources/filesystem/gitpythonfs/__init__.py

@@ -0,0 +1,9 @@
+# file exists to make pytest work.
+#
+# pytest (or maybe just pytest discovery) expects all nested folders to be packages.
+#
+# This __init__.py file not required once gitpythonfs is a completely separate
+# package to dlt-verified-sources or it's tests are moved to /tests.
+#
+# An alternative is to run pytest specifically on a subfolder, e.g.:
+#     pytest sources/filesystem/gitpythonfs