S3V2 with path-style request using list-objects API to list bucket (#93)

* S3V2 with path-style request using list-objects API to list bucket

* Update python version for c

* Add lxml dep

---------

Co-authored-by: 杨赫然 <[email protected]>

Author: feiniks

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 1
       - uses: actions/setup-python@v1
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: install dependencies and test
         run: |
           cd $GITHUB_WORKSPACE

ci/requirements.txt

@@ -7,3 +7,4 @@ oss2==2.18.4
 sqlalchemy==2.0.18
 pylibmc==1.6.3
 redis==5.0.8
+lxml==5.3.1

objwrapper/s3.py

@@ -1,6 +1,12 @@
 import boto3
 from botocore.exceptions import ClientError
 from objwrapper.exceptions import InvalidConfigError
+import requests
+from datetime import datetime
+import hmac
+import hashlib
+import base64
+from lxml import etree
 
 class S3Conf(object):
     def __init__(self, key_id, key, bucket_name, host, port, use_v4_sig, aws_region, use_https, path_style_request, sse_c_key):
@@ -24,6 +30,7 @@ def __init__(self, conf):
         self.conf = conf
         self.client = None
         self.bucket = None
+        self.enpoint_url = None;
         self.do_connect()
 
     def do_connect(self):
@@ -37,6 +44,10 @@ def do_connect(self):
             config = boto3.session.Config(signature_version='s3',s3={'addressing_style':addressing_style})
 
         if self.conf.host is None:
+            if self.conf.use_https:
+                self.endpoint_url = f'https://s3.{self.conf.aws_region}.amazonaws.com'
+            else:
+                self.endpoint_url = f'http://s3.{self.conf.aws_region}.amazonaws.com'
             self.client = boto3.client('s3',
                                        region_name=self.conf.aws_region,
                                        aws_access_key_id=self.conf.key_id,
@@ -48,6 +59,7 @@ def do_connect(self):
             endpoint_url = 'https://%s' % self.conf.host if self.conf.use_https else 'http://%s' % self.conf.host
             if self.conf.port:
                 endpoint_url = '%s:%s' % (endpoint_url, self.conf.port)
+            self.endpoint_url = endpoint_url
             self.client = boto3.client('s3',
                                        aws_access_key_id=self.conf.key_id,
                                        aws_secret_access_key=self.conf.key,
@@ -67,6 +79,11 @@ def get_name(self):
         return 'S3 storage backend'
 
     def list_objs(self, prefix=None):
+        if not self.conf.use_v4_sig and self.conf.path_style_request:
+            # When using the S3 v2 protocol and path-style requests, boto3 is unable to list objects properly.
+            # We manually sign the requests and then use the list-objects API to list the objects in the bucket.
+            yield from self.list_objs_v2(prefix)
+            return
         paginator = self.client.get_paginator('list_objects_v2')
         if prefix:
             iterator = paginator.paginate(Bucket=self.bucket, Prefix=prefix)
@@ -79,7 +96,6 @@ def list_objs(self, prefix=None):
                     obj = [tokens, content.get('Size', 0)]
                     yield obj
 
-
     def obj_exists(self, key):
         bucket = self.bucket
         try:
@@ -128,3 +144,65 @@ def get_ctime(self, key):
             return float(ctime)
         except:
             return 0
+
+    def get_signature(self, date):
+        sign_str = f"GET\n\n\n{date}\n/{self.bucket}/"
+
+        hmac_object = hmac.new(self.conf.key.encode('utf-8'), sign_str.encode('utf-8'), hashlib.sha1)
+        hmac_bytes = hmac_object.digest()
+        signature = base64.b64encode(hmac_bytes).decode('utf-8')
+        return signature
+
+    def list_bucket_v2 (self, marker, prefix):
+        now = datetime.utcnow()
+        date = now.strftime('%a, %d %b %Y %H:%M:%S GMT')
+        signature =self.get_signature(date)
+
+        headers = {'Date':date,
+                'Authorization': f"AWS {self.conf.key_id}:{signature}",
+                }
+
+        endpoint_url = self.endpoint_url
+        bucket = self.bucket
+
+        if marker and prefix:
+            url = f'{endpoint_url}/{bucket}/?marker={marker}&prefix={prefix}/'
+        elif marker:
+            url = f'{endpoint_url}/{bucket}/?marker={marker}'
+        elif prefix: 
+            url = f'{endpoint_url}/{bucket}/?prefix={prefix}/'
+        else:
+            url = f'{endpoint_url}/{bucket}/'
+
+        response = requests.get (url, headers=headers, timeout=300)
+        if response.status_code != 200:
+            return None
+        return response.text
+
+    def list_objs_v2(self, prefix):
+        is_truncated = True
+        marker = None
+        while is_truncated:
+            rsp = self.list_bucket_v2(marker, prefix)
+            if not rsp:
+                break
+
+            root = etree.fromstring(rsp.encode('utf-8'))
+            if "ListBucketResult" not in root.tag:
+                break
+            for child in root:
+                if "IsTruncated" in child.tag:
+                    if child.text == "true":
+                        is_truncated = True
+                    else:
+                        is_truncated = False
+                if "Contents" in child.tag:
+                    obj = [] 
+                    for contents in child:
+                        if "Key" in contents.tag:
+                            marker = contents.text
+                            obj.append(contents.text)
+                        if "Size" in contents.tag:
+                            obj.append(int(contents.text))
+                    if obj:
+                        yield obj