chore: Remove py37 support, and unify compatibility matrix from 3.8-3.9 (amundsen-io#2226)

kristenarmes · web-flow · commit 890bfaf92c4b · 2024-02-22T12:51:26.000-08:00
* Remove py37 support, and unify compatibility matrix from 3.8-3.10

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;

* Remove py3.10 for now, requires other upgrades

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;

* lint

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;

* Upgrade confluent-kafka

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;

* Upgrade feast

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;

* Attempt to fix feast versioning

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;

* lint

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;

---------

Signed-off-by: Kristen Armes &lt;karmes@lyft.com&gt;
diff --git a/.github/workflows/common_pull_request.yml b/.github/workflows/common_pull_request.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ["3.7.x"]
+        python-version: ['3.8.x', '3.9.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1
diff --git a/.github/workflows/databuilder_pull_request.yml b/.github/workflows/databuilder_pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ["3.7.x"]
+        python-version: ['3.8.x', '3.9.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1
diff --git a/.github/workflows/deploy_release.yml b/.github/workflows/deploy_release.yml
@@ -17,10 +17,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: Setup python 3.7
+      - name: Setup python 3.8
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.8
       - name: Add wheel dependency
         run: pip install wheel
       - name: Generate dist
diff --git a/.github/workflows/frontend_pull_request.yml b/.github/workflows/frontend_pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ["3.7.x", "3.8.x"]
+        python-version: ['3.8.x', '3.9.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1
diff --git a/.github/workflows/metadata_pull_request.yml b/.github/workflows/metadata_pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ["3.7.x"]
+        python-version: ['3.8.x', '3.9.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1
diff --git a/.github/workflows/monthly_release.yml b/.github/workflows/monthly_release.yml
@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.8
       - name: Checkout master
         uses: actions/checkout@v2
         with:
diff --git a/.github/workflows/noop_pull_request.yml b/.github/workflows/noop_pull_request.yml
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6.x", "3.7.x", "3.8.x"]
+        python-version: ['3.8.x', '3.9.x']
     steps:
       - name: Do nothing
         run: exit 0
diff --git a/.github/workflows/search_pull_request.yml b/.github/workflows/search_pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ["3.7.x"]
+        python-version: ['3.8.x', '3.9.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1
diff --git a/Dockerfile.frontend.local b/Dockerfile.frontend.local
@@ -14,7 +14,7 @@ RUN npm run dev-build
 
 COPY ./frontend /app
 
-FROM python:3.7-slim
+FROM python:3.8-slim
 WORKDIR /app
 
 COPY ./frontend /app
diff --git a/Dockerfile.frontend.public b/Dockerfile.frontend.public
@@ -8,7 +8,7 @@ RUN npm install
 COPY ./frontend/amundsen_application/static /app/amundsen_application/static
 RUN npm run build
 
-FROM python:3.7-slim as base
+FROM python:3.8-slim as base
 WORKDIR /app
 RUN pip3 install gunicorn
 
diff --git a/Dockerfile.metadata.public b/Dockerfile.metadata.public
@@ -1,4 +1,4 @@
-FROM python:3.7-slim as base
+FROM python:3.8-slim as base
 WORKDIR /app
 RUN pip3 install gunicorn
 
diff --git a/Dockerfile.search.public b/Dockerfile.search.public
@@ -1,4 +1,4 @@
-FROM python:3.7-slim
+FROM python:3.8-slim
 WORKDIR /app
 RUN pip3 install gunicorn
 
diff --git a/README.md b/README.md
@@ -62,7 +62,7 @@ We want your input about what is important, for that, add your votes using the 
 * [Top Questions](https://github.com/amundsen-io/amundsen/issues?q=is%3Aissue+is%3Aclosed+sort%3Areactions-%2B1-desc+label%3Atype%3Aquestion+label%3Astatus%3Aneeds_votes)
 
 ## Requirements
-- Python 3.7
+- Python >= 3.8
 - Node v12
 
 ## User Interface
diff --git a/common/README.md b/common/README.md
@@ -8,7 +8,7 @@ Amundsen Common library holds common codes among micro services in Amundsen.
 For information about Amundsen and our other services, visit the [main repository](https://github.com/amundsen-io/amundsen). Please also see our instructions for a [quick start](https://github.com/amundsen-io/amundsen/blob/master/docs/installation.md#bootstrap-a-default-version-of-amundsen-using-docker) setup  of Amundsen with dummy data, and an [overview of the architecture](https://github.com/amundsen-io/amundsen/blob/master/docs/architecture.md#architecture).
 
 ## Requirements
-- Python >= 3.6
+- Python >= 3.8
 
 ## Doc
 - https://www.amundsen.io/amundsen/
diff --git a/common/amundsen_common/log/action_log_callback.py b/common/amundsen_common/log/action_log_callback.py
@@ -16,8 +16,8 @@
 
 LOGGER = logging.getLogger(__name__)
 
-__pre_exec_callbacks = []  # type: List[Callable[..., Any]]
-__post_exec_callbacks = []  # type: List[Callable[..., Any]]
+__pre_exec_callbacks: List[Callable[..., Any]] = []
+__post_exec_callbacks: List[Callable[..., Any]] = []
 
 
 def register_pre_exec_callback(action_log_callback: Callable[..., Any]) -> None:
diff --git a/common/setup.py b/common/setup.py
@@ -46,6 +46,10 @@
     extras_require={
         'all': requirements_dev
     },
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     package_data={'amundsen_common': ['py.typed']},
+    classifiers=[
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+    ],
 )
diff --git a/databuilder/README.md b/databuilder/README.md
@@ -11,7 +11,7 @@ Amundsen Databuilder is a data ingestion library, which is inspired by [Apache G
 For information about Amundsen and our other services, visit the [main repository](https://github.com/amundsen-io/amundsen#amundsen) `README.md` . Please also see our instructions for a [quick start](https://github.com/amundsen-io/amundsen/blob/master/docs/installation.md#bootstrap-a-default-version-of-amundsen-using-docker) setup  of Amundsen with dummy data, and an [overview of the architecture](https://github.com/amundsen-io/amundsen/blob/master/docs/architecture.md#architecture).
 
 ## Requirements
-- Python >= 3.6.x
+- Python >= 3.8.x
 - elasticsearch 7.x
 
 ## Doc
diff --git a/databuilder/databuilder/extractor/delta_lake_metadata_extractor.py b/databuilder/databuilder/extractor/delta_lake_metadata_extractor.py
@@ -503,8 +503,8 @@ def _fetch_minmax(table: ScrapedTableMetadata, partition_column: str) -> Tuple[s
             _table_columns = table.columns
         else:
             _table_columns = []
-        columns_with_valid_type = list(map(lambda l: l.name,
-                                           filter(lambda l: str(l.data_type).lower() in valid_types, _table_columns)
+        columns_with_valid_type = list(map(lambda n: n.name,
+                                           filter(lambda d: str(d.data_type).lower() in valid_types, _table_columns)
                                            )
                                        )
 
diff --git a/databuilder/setup.cfg b/databuilder/setup.cfg
@@ -40,7 +40,7 @@ output = build/coverage.xml
 directory = build/coverage_html
 
 [mypy]
-python_version = 3.6
+python_version = 3.8
 disallow_untyped_defs = True
 ignore_missing_imports = True
 exclude = example
diff --git a/databuilder/setup.py b/databuilder/setup.py
@@ -17,7 +17,7 @@
 with open(requirements_path, 'r') as requirements_file:
     requirements_dev = requirements_file.readlines()
 
-kafka = ['confluent-kafka==1.0.0']
+kafka = ['confluent-kafka==2.3.0']
 
 cassandra = ['cassandra-driver==3.20.1']
 
@@ -70,7 +70,8 @@
 
 feast = [
     'feast==0.17.0',
-    'fastapi!=0.76.*'
+    'fastapi!=0.76.*',
+    'protobuf<=3.20.1'
 ]
 
 atlas = [
@@ -114,7 +115,7 @@
     include_package_data=True,
     dependency_links=[],
     install_requires=requirements,
-    python_requires='>=3.7',
+    python_requires='>=3.8',
     extras_require={
         'all': all_deps,
         'dev': requirements_dev,
@@ -139,6 +140,7 @@
         'schema_registry': schema_registry,
     },
     classifiers=[
-        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
     ],
 )
diff --git a/frontend/README.md b/frontend/README.md
@@ -14,7 +14,7 @@ The frontend service leverages a separate [search service](./../search) for allo
 For information about Amundsen and our other services, refer to this [README.md](https://github.com/amundsen-io/amundsen). Please also see our instructions for a [quick start](https://www.amundsen.io/amundsen/installation/) setup  of Amundsen with dummy data, and an [overview of the architecture](https://www.amundsen.io/amundsen/architecture/).
 
 ## Requirements
-- Python >= 3.6
+- Python >= 3.8
 - Node = v12
 - npm >= 6.x.x
 
diff --git a/frontend/setup.cfg b/frontend/setup.cfg
@@ -21,7 +21,7 @@ output = build/coverage.xml
 directory = build/coverage_html
 
 [mypy]
-python_version = 3.6
+python_version = 3.8
 disallow_untyped_defs = True
 ignore_missing_imports = True
 strict_optional = True
diff --git a/frontend/setup.py b/frontend/setup.py
@@ -75,12 +75,13 @@ def build_js() -> None:
         'asana': asana,
         'all': all_deps,
     },
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     entry_points="""
         [action_log.post_exec.plugin]
         logging_action_log=amundsen_application.log.action_log_callback:logging_action_log
     """,
     classifiers=[
-        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
     ],
 )
diff --git a/metadata/README.md b/metadata/README.md
@@ -11,7 +11,7 @@ Amundsen Metadata service serves Restful API and is responsible for providing an
 For information about Amundsen and our other services, refer to this [README.md](./../README.md). Please also see our instructions for a [quick start](./../docs/installation.md#bootstrap-a-default-version-of-amundsen-using-docker) setup  of Amundsen with dummy data, and an [overview of the architecture](./../docs/architecture.md#architecture).
 
 ## Requirements
-- Python >= 3.7
+- Python >= 3.8
 
 ## Doc
 - https://www.amundsen.io/amundsen/
diff --git a/metadata/metadata_service/proxy/gremlin_proxy.py b/metadata/metadata_service/proxy/gremlin_proxy.py
@@ -281,7 +281,7 @@ def _safe_get_list(root, *keys, transform: Optional[Callable] = None):
         raise RuntimeError(f'{values} is not a List!  root={root} keys={keys}')
     elif transform is None:
         return sorted(values)
-    elif len(values) > 0 and type(values[0]) == datetime and transform == int:
+    elif len(values) > 0 and type(values[0]) is datetime and transform == int:
         # need to do something special for datetimes we are transforming into int's
         return sorted([transform(value.timestamp()) for value in values])
     else:
diff --git a/metadata/metadata_service/proxy/neo4j_proxy.py b/metadata/metadata_service/proxy/neo4j_proxy.py
@@ -436,7 +436,7 @@ def _exec_table_query(self, table_uri: str) -> Tuple:
 
         resource_reports = self._extract_resource_reports_from_query(table_records.get('resource_reports', []))
 
-        return wmk_results, table_writer, table_apps, timestamp_value,\
+        return wmk_results, table_writer, table_apps, timestamp_value, \
             tags, src, badges, prog_descriptions, resource_reports
 
     @timer_with_counter
diff --git a/metadata/setup.cfg b/metadata/setup.cfg
@@ -32,7 +32,7 @@ exclude_lines =
     import *
 
 [mypy]
-python_version = 3.6
+python_version = 3.8
 disallow_untyped_defs = True
 ignore_missing_imports = True
 strict_optional = True
diff --git a/metadata/setup.py b/metadata/setup.py
@@ -49,8 +49,9 @@
         'rds': rds,
         'gremlin': gremlin
     },
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     classifiers=[
-        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
     ],
 )
diff --git a/metadata/tests/unit/proxy/test_neo4j_proxy.py b/metadata/tests/unit/proxy/test_neo4j_proxy.py
@@ -1524,7 +1524,7 @@ def test_get_lineage_success(self) -> None:
             key = "alpha"
             mock_execute.return_value = [{
                 "upstream_entities": [
-                    {"key": "beta", "source": "gold", "level": 1, "badges": [], "usage":100, "parent": None},
+                    {"key": "beta", "source": "gold", "level": 1, "badges": [], "usage": 100, "parent": None},
                     {"key": "gamma", "source": "dyno", "level": 1,
                      "badges":
                         [
@@ -1541,7 +1541,7 @@ def test_get_lineage_success(self) -> None:
             expected = Lineage(
                 key=key,
                 upstream_entities=[
-                    LineageItem(**{"key": "beta", "source": "gold", "level": 1, "badges": [], "usage":100}),
+                    LineageItem(**{"key": "beta", "source": "gold", "level": 1, "badges": [], "usage": 100}),
                     LineageItem(**{"key": "gamma", "source": "dyno", "level": 1,
                                    "badges":
                                        [
diff --git a/search/README.md b/search/README.md
@@ -19,7 +19,7 @@ For information about Amundsen and our other services, refer to this [README.md]
 
 ## Requirements
 
-- Python >= 3.7
+- Python >= 3.8
 - Elasticsearch, supported versions:
     - 7.x
     - 8.0.0
diff --git a/search/search_service/config.py b/search/search_service/config.py
@@ -52,7 +52,7 @@ class LocalConfig(Config):
                                         PORT=PROXY_PORT)
                                     )
     ES_PROXY_CLIENT = PROXY_CLIENTS[os.environ.get('ES_PROXY_CLIENT', 'ELASTICSEARCH_V2_1')]
-    ELASTICSEARCH_CLIENT = os.environ.get('ELASTICSEARCH_CLIENT')   # type: Optional[Any]
+    ELASTICSEARCH_CLIENT: Optional[Any] = os.environ.get('ELASTICSEARCH_CLIENT')
     PROXY_USER = os.environ.get('CREDENTIALS_PROXY_USER', 'elastic')
     PROXY_PASSWORD = os.environ.get('CREDENTIALS_PROXY_PASSWORD', 'elastic')
 
diff --git a/search/setup.cfg b/search/setup.cfg
@@ -32,7 +32,7 @@ exclude_lines =
     import *
 
 [mypy]
-python_version = 3.6
+python_version = 3.8
 disallow_untyped_defs = True
 ignore_missing_imports = True
 strict_optional = True
diff --git a/search/setup.py b/search/setup.py
@@ -40,5 +40,9 @@
         'dev': requirements_dev,
         'oidc': oidc
     },
-    python_requires=">=3.7"
+    python_requires=">=3.8",
+    classifiers=[
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+    ],
 )
