RequestTimeTooSkewed: The difference between the request time and the server's time is too large

[forestbat]

I want to list objects in my data directory by minio, however it crashed with this:

# conf.MC is a Minio object
tests/test_gpm_csv.py:7 (test_read_gpm_csv)def test_read_gpm_csv():
        gpm_nc_files = []
        for year in range(2019, 2024):
>           gpm_nc_files_part = ['s3://grids-origin/'+file.object_name for file in conf.MC.list_objects(bucket_name='grids-origin',
                                                                                   prefix='GPM/global/30m/'+str(year), recursive=True) if 'nc4' in file.object_name]
test_gpm_csv.py:11: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
test_gpm_csv.py:11: in <listcomp>
    gpm_nc_files_part = ['s3://grids-origin/'+file.object_name for file in conf.MC.list_objects(bucket_name='grids-origin',
../../../.conda/envs/test_sam/lib/python3.10/site-packages/minio/api.py:3111: in _list_objects
    response = self._execute(
../../../.conda/envs/test_sam/lib/python3.10/site-packages/minio/api.py:440: in _execute
    return self._url_open(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <minio.api.Minio object at 0x7fb9c95f2710>, method = 'GET'
region = 'us-east-1', bucket_name = 'grids-origin', object_name = None
body = None
headers = {'Authorization': 'AWS4-HMAC-SHA256 Credential=forestbat/20240417/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-...6_64) minio-py/7.2.4', 'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', ...}
query_params = {'delimiter': '', 'encoding-type': 'url', 'list-type': '2', 'max-keys': '1000', ...}
preload_content = True, no_body_trace = False
    def _url_open(
            self,
            method: str,
            region: str,
            bucket_name: str | None = None,
            object_name: str | None = None,
            body: bytes | None = None,
            headers: DictType | None = None,
            query_params: DictType | None = None,
            preload_content: bool = True,
            no_body_trace: bool = False,
    ) -> BaseHTTPResponse:
        """Execute HTTP request."""
        creds = self._provider.retrieve() if self._provider else None
        url = self._base_url.build(
            method,
            region,
            bucket_name=bucket_name,
            object_name=object_name,
            query_params=query_params,
        )
        headers, date = self._build_headers(url.netloc, headers, body, creds)
        if creds:
            headers = sign_v4_s3(
                method,
                url,
                region,
                headers,
                creds,
                cast(str, headers.get("x-amz-content-sha256")),
                date,
            )
    
        if self._trace_stream:
            self._trace_stream.write("---------START-HTTP---------\n")
            query = ("?" + url.query) if url.query else ""
            self._trace_stream.write(f"{method} {url.path}{query} HTTP/1.1\n")
            self._trace_stream.write(
                headers_to_strings(headers, titled_key=True),
            )
            self._trace_stream.write("\n")
            if not no_body_trace and body is not None:
                self._trace_stream.write("\n")
                self._trace_stream.write(
                    body.decode() if isinstance(body, bytes) else str(body),
                )
                self._trace_stream.write("\n")
            self._trace_stream.write("\n")
    
        http_headers = HTTPHeaderDict()
        for key, value in (headers or {}).items():
            if isinstance(value, (list, tuple)):
                for val in value:
                    http_headers.add(key, val)
            else:
                http_headers.add(key, value)
    
        response = self._http.urlopen(
            method,
            urlunsplit(url),
            body=body,
            headers=http_headers,
            preload_content=preload_content,
        )
    
        if self._trace_stream:
            self._trace_stream.write(f"HTTP/1.1 {response.status}\n")
            self._trace_stream.write(
                headers_to_strings(response.headers),
            )
            self._trace_stream.write("\n")
    
        if response.status in [200, 204, 206]:
            if self._trace_stream:
                if preload_content:
                    self._trace_stream.write("\n")
                    self._trace_stream.write(response.data.decode())
                    self._trace_stream.write("\n")
                self._trace_stream.write("----------END-HTTP----------\n")
            return response
    
        response.read(cache_content=True)
        if not preload_content:
            response.release_conn()
    
        if self._trace_stream and method != "HEAD" and response.data:
            self._trace_stream.write(response.data.decode())
            self._trace_stream.write("\n")
    
        if (
                method != "HEAD" and
                "application/xml" not in response.headers.get(
                    "content-type", "",
                ).split(";")
        ):
            if self._trace_stream:
                self._trace_stream.write("----------END-HTTP----------\n")
            if response.status == 304 and not response.data:
                raise ServerError(
                    f"server failed with HTTP status code {response.status}",
                    response.status,
                )
            raise InvalidResponseError(
                response.status,
                cast(str, response.headers.get("content-type")),
                response.data.decode() if response.data else None,
            )
    
        if not response.data and method != "HEAD":
            if self._trace_stream:
                self._trace_stream.write("----------END-HTTP----------\n")
            raise InvalidResponseError(
                response.status,
                response.headers.get("content-type"),
                None,
            )
    
        response_error = S3Error.fromxml(response) if response.data else None
    
        if self._trace_stream:
            self._trace_stream.write("----------END-HTTP----------\n")
    
        error_map = {
            301: lambda: self._handle_redirect_response(
                method, bucket_name, response, True,
            ),
            307: lambda: self._handle_redirect_response(
                method, bucket_name, response, True,
            ),
            400: lambda: self._handle_redirect_response(
                method, bucket_name, response, True,
            ),
            403: lambda: ("AccessDenied", "Access denied"),
            404: lambda: (
                ("NoSuchKey", "Object does not exist")
                if object_name
                else ("NoSuchBucket", "Bucket does not exist")
                if bucket_name
                else ("ResourceNotFound", "Request resource not found")
            ),
            405: lambda: (
                "MethodNotAllowed",
                "The specified method is not allowed against this resource",
            ),
            409: lambda: (
                ("NoSuchBucket", "Bucket does not exist")
                if bucket_name
                else ("ResourceConflict", "Request resource conflicts"),
            ),
            501: lambda: (
                "MethodNotAllowed",
                "The specified method is not allowed against this resource",
            ),
        }
    
        if not response_error:
            func = error_map.get(response.status)
            code, message = func() if func else (None, None)
            if not code:
                raise ServerError(
                    f"server failed with HTTP status code {response.status}",
                    response.status,
                )
            response_error = S3Error(
                cast(str, code),
                cast(Union[str, None], message),
                url.path,
                response.headers.get("x-amz-request-id"),
                response.headers.get("x-amz-id-2"),
                response,
                bucket_name=bucket_name,
                object_name=object_name,
            )
    
        if response_error.code in ["NoSuchBucket", "RetryHead"]:
            if bucket_name is not None:
                self._region_map.pop(bucket_name, None)
    
>       raise response_error
E       minio.error.S3Error: S3 operation failed; code: RequestTimeTooSkewed, message: The difference between the request time and the server's time is too large., resource: /grids-origin, request_id: 17C6F2CE13BB749E, host_id: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8
../../../.conda/envs/test_sam/lib/python3.10/site-packages/minio/api.py:423: S3Error

So why it will happen and how to solve it?

[klauspost]

Fix the time on your client/servers.