Merge branch 'main' into force-static-libs

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -12,6 +12,14 @@ body:
       description: What is the problem? A clear and concise description of the bug.
     validations:
       required: true
+  - type: checkboxes
+    id: regression
+    attributes:
+      label: Regression Issue
+      description: What is a regression? If it worked in a previous version but doesn't in the latest version, it's considered a regression. In this case, please provide specific version number in the report.
+      options:
+        - label: Select this option if this issue appears to be a regression.
+          required: false
   - type: textarea
     id: expected
     attributes:

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ env:
   PACKAGE_NAME: aws-crt-python
   LINUX_BASE_IMAGE: ubuntu-18-x64
   RUN: ${{ github.run_id }}-${{ github.run_number }}
-  CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} 
+  CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }}
   AWS_DEFAULT_REGION: us-east-1
 
 jobs:
@@ -231,7 +231,7 @@ jobs:
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
-          python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe"
+          python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.8.10\\${{ matrix.arch }}\\python.exe"
 
 
   macos:
@@ -293,7 +293,7 @@ jobs:
           version: ${{ matrix.version }}
           cpu_count: 4
           shell: bash
-          environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION 
+          environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION
           run: |
             sudo pkg_add awscli py3-pip py3-urllib3
             python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')"
@@ -321,7 +321,7 @@ jobs:
           version: '14.0'
           cpu_count: 4
           shell: bash
-          environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION 
+          environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION
           run: |
             sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake
             python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')"
@@ -335,7 +335,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - name: Run tests 
+      - name: Run tests
         run: |
           python3 -m pip install --upgrade --requirement requirements-dev.txt
           python3 -m pip install . --verbose

.github/workflows/issue-regression-labeler.yml

@@ -0,0 +1,32 @@
+# Apply potential regression label on issues
+name: issue-regression-label
+on:
+  issues:
+    types: [opened, edited]
+jobs:
+  add-regression-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+    - name: Fetch template body
+      id: check_regression
+      uses: actions/github-script@v7
+      env: 
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        TEMPLATE_BODY: ${{ github.event.issue.body }}
+      with:
+        script: |
+          const regressionPattern = /\[x\] Select this option if this issue appears to be a regression\./i;
+          const template = `${process.env.TEMPLATE_BODY}`
+          const match = regressionPattern.test(template);
+          core.setOutput('is_regression', match);
+    - name: Manage regression label
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        if [ "${{ steps.check_regression.outputs.is_regression }}" == "true" ]; then
+          gh issue edit ${{ github.event.issue.number }} --add-label "potential-regression" -R ${{ github.repository }}
+        else
+          gh issue edit ${{ github.event.issue.number }} --remove-label "potential-regression" -R ${{ github.repository }}
+        fi

README.md

@@ -13,7 +13,7 @@ This library is licensed under the Apache 2.0 License.
 
 ## Minimum Requirements:
 
-*   Python 3.7+
+*   Python 3.8+
 
 ## Installation
 

awscrt/eventstream/__init__.py

@@ -5,9 +5,8 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-from collections.abc import ByteString
 from enum import IntEnum
-from typing import Any
+from typing import Any, Union
 from uuid import UUID
 
 __all__ = ['HeaderType', 'Header']
@@ -135,7 +134,7 @@ def from_int64(cls, name: str, value: int) -> 'Header':
         return cls(name, value, HeaderType.INT64)
 
     @classmethod
-    def from_byte_buf(cls, name: str, value: ByteString) -> 'Header':
+    def from_byte_buf(cls, name: str, value: Union[bytes, bytearray]) -> 'Header':
         """Create a Header of type :attr:`~HeaderType.BYTE_BUF`
 
         The value must be a bytes-like object"""
@@ -246,7 +245,7 @@ def value_as_int64(self) -> int:
         Raises an exception if type is not :attr:`~HeaderType.INT64`"""
         return self._value_as(HeaderType.INT64)
 
-    def value_as_byte_buf(self) -> ByteString:
+    def value_as_byte_buf(self) -> Union[bytes, bytearray]:
         """Return value of bytes
 
         Raises an exception if type is not :attr:`~HeaderType.BYTE_BUF`"""

awscrt/eventstream/rpc.py

@@ -11,11 +11,11 @@
 import awscrt.exceptions
 from awscrt.eventstream import Header
 from awscrt.io import ClientBootstrap, SocketOptions, TlsConnectionOptions
-from collections.abc import ByteString, Callable
+from collections.abc import Callable
 from concurrent.futures import Future
 from enum import IntEnum
 from functools import partial
-from typing import Optional, Sequence
+from typing import Optional, Sequence, Union
 
 __all__ = [
     'MessageType',
@@ -381,7 +381,7 @@ def send_protocol_message(
             self,
             *,
             headers: Optional[Sequence[Header]] = None,
-            payload: Optional[ByteString] = None,
+            payload: Optional[Union[bytes, bytearray]] = None,
             message_type: MessageType,
             flags: Optional[int] = None,
             on_flush: Callable = None) -> 'concurrent.futures.Future':
@@ -483,7 +483,7 @@ def activate(
             *,
             operation: str,
             headers: Sequence[Header] = None,
-            payload: ByteString = None,
+            payload: Union[bytes, bytearray] = None,
             message_type: MessageType,
             flags: int = None,
             on_flush: Callable = None):
@@ -553,7 +553,7 @@ def send_message(
             self,
             *,
             headers: Sequence[Header] = None,
-            payload: ByteString = None,
+            payload: Union[bytes, bytearray] = None,
             message_type: MessageType,
             flags: int = None,
             on_flush: Callable = None) -> 'concurrent.futures.Future':

codebuild/cd/manylinux-x64-build.yml

@@ -8,7 +8,7 @@ phases:
     commands:
       - export CC=gcc
       - cd aws-crt-python
-      - /opt/python/cp37-cp37m/bin/python ./continuous-delivery/update-version.py
+      - /opt/python/cp38-cp38/bin/python ./continuous-delivery/update-version.py
   build:
     commands:
       - echo Build started on `date`

codebuild/cd/manylinux-x86-build.yml

@@ -8,7 +8,7 @@ phases:
     commands:
       - export CC=gcc
       - cd aws-crt-python
-      - /opt/python/cp37-cp37m/bin/python ./continuous-delivery/update-version.py
+      - /opt/python/cp38-cp38/bin/python ./continuous-delivery/update-version.py
   build:
     commands:
       - echo Build started on `date`

continuous-delivery/sanity-check-test-pypi.bat

@@ -1,8 +1,8 @@
 FOR /F "delims=" %%A in ('git describe --tags') do ( set TAG_VERSION=%%A )
 set CURRENT_VERSION=%TAG_VERSION:v=%
 
-"C:\Program Files\Python37\python.exe" continuous-delivery\pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error
-"C:\Program Files\Python37\python.exe" continuous-delivery\test-pip-install.py || goto error
+"C:\Program Files\Python38\python.exe" continuous-delivery\pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error
+"C:\Program Files\Python38\python.exe" continuous-delivery\test-pip-install.py || goto error
 
 goto :EOF
 

crt/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.9)
 
 # This CMakeLists.txt exists so we can build all the C libraries we depend on
 # simultaneously. This is much faster than building dependencies one at a time.
@@ -30,21 +30,38 @@ include(CTest)
 # (On Windows and Apple we use the default OS libraries)
 if(UNIX AND NOT APPLE)
     option(USE_OPENSSL "Set this if you want to use your system's OpenSSL compatible libcrypto" OFF)
+    include(AwsPrebuildDependency)
 
     if(NOT USE_OPENSSL)
-        set(DISABLE_GO ON CACHE BOOL "Build without using Go, we don't want the extra dependency")
-        set(BUILD_LIBSSL OFF CACHE BOOL "Don't need libssl, only need libcrypto")
 
-        set(DISABLE_PERL ON CACHE BOOL "Build without using Perl, we don't want the extra dependency")
+        set(AWSLC_CMAKE_ARGUMENTS
+            -DDISABLE_GO=ON  # Build without using Go, we don't want the extra dependency
+            -DDISABLE_PERL=ON  # Build without using Perl, we don't want the extra dependency
+            -DBUILD_LIBSSL=OFF  # Don't need libssl, only need libcrypto
+            -DBUILD_TESTING=OFF
+        )
+
         if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0")
-            set(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX ON CACHE BOOL "Disable AVX512 on old GCC that not supports it")
+            # Disable AVX512 on old GCC that not supports it.
+            list(APPEND AWSLC_CMAKE_ARGUMENTS -DMY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX=ON)
         endif()
 
-        add_subdirectory(aws-lc)
+        # s2n-tls uses libcrypto during its configuration, so we need to prebuild aws-lc.
+        aws_prebuild_dependency(
+            DEPENDENCY_NAME AWSLC
+            SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/aws-lc
+            CMAKE_ARGUMENTS ${AWSLC_CMAKE_ARGUMENTS}
+        )
     endif()
 
-    set(UNSAFE_TREAT_WARNINGS_AS_ERRORS OFF CACHE BOOL "")
-    add_subdirectory(s2n)
+    # prebuild s2n-tls.
+    aws_prebuild_dependency(
+        DEPENDENCY_NAME S2N
+        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/s2n
+        CMAKE_ARGUMENTS
+            -DUNSAFE_TREAT_WARNINGS_AS_ERRORS=OFF
+            -DBUILD_TESTING=OFF
+    )
 endif()
 
 add_subdirectory(aws-c-common)

docsrc/source/api/checksums.rst

@@ -0,0 +1,5 @@
+awscrt.checksums
+================
+
+.. automodule:: awscrt.checksums
+    :members:

docsrc/source/index.rst

@@ -12,6 +12,7 @@ API Reference
    :maxdepth: 2
 
    api/auth
+   api/checksums
    api/common
    api/crypto
    api/exceptions

setup.py

@@ -13,6 +13,12 @@
 import sys
 import sysconfig
 from wheel.bdist_wheel import bdist_wheel
+if sys.platform == 'win32':
+    # distutils is deprecated in Python 3.10 and removed in 3.12. However, it still works because Python defines a compatibility interface as long as setuptools is installed.
+    # We don't have an official alternative for distutils.ccompiler as of September 2024. See: https://github.com/pypa/setuptools/issues/2806
+    # Once that issue is resolved, we can migrate to the official solution.
+    # For now, restrict distutils to Windows only, where it's needed.
+    import distutils.ccompiler
 
 
 def is_64bit():
@@ -76,7 +82,6 @@ def determine_generator_args():
     if sys.platform == 'win32':
         try:
             # See which compiler python picks
-            import distutils.ccompiler
             compiler = distutils.ccompiler.new_compiler()
             compiler.initialize()
 
@@ -332,11 +337,6 @@ def awscrt_ext():
     libraries.reverse()
 
     if sys.platform == 'win32':
-        # distutils is deprecated in Python 3.10 and removed in 3.12. However, it still works because Python defines a compatibility interface as long as setuptools is installed.
-        # We don't have an official alternative for distutils.ccompiler as of September 2024. See: https://github.com/pypa/setuptools/issues/2806
-        # Once that issue is resolved, we can migrate to the official solution.
-        # For now, restrict distutils to Windows only, where it's needed.
-        import distutils.ccompiler
         # the windows apis being used under the hood. Since we're static linking we have to follow the entire chain down
         libraries += ['Secur32', 'Crypt32', 'Advapi32', 'NCrypt', 'BCrypt', 'Kernel32', 'Ws2_32', 'Shlwapi']
         # Ensure that debug info is in the obj files, and that it is linked into the .pyd so that
@@ -471,7 +471,7 @@ def _load_version():
         "Operating System :: Unix",
         "Operating System :: MacOS",
     ],
-    python_requires='>=3.7',
+    python_requires='>=3.8',
     ext_modules=[awscrt_ext()],
     cmdclass={'build_ext': awscrt_build_ext, "bdist_wheel": bdist_wheel_abi3},
     test_suite='test',

test/test_s3.py

@@ -222,8 +222,9 @@ def test_sanity_secure(self):
         self.assertIsNotNone(s3_client)
 
     def test_sanity_network_interface_names(self):
-        s3_client = s3_client_new(True, self.region, network_interface_names=("eth0", "eth1"))
-        self.assertIsNotNone(s3_client)
+        # This is just a sanity test to ensure that we are passing the parameter correctly.
+        with self.assertRaises(Exception):
+            s3_client_new(True, self.region, network_interface_names=("eth0", "invalid-network-interface"))
 
     def test_wait_shutdown(self):
         s3_client = s3_client_new(False, self.region)