ML-3730: fix support for client running on windows OS

README.md

@@ -266,7 +266,7 @@ def test_verifier_transport(self):
 
         # verify some stuff from the request
         self.assertEqual(request.container, container_name)
-        self.assertEqual(request.path, os.path.join(os.sep, container_name, 'some/path'))
+        self.assertEqual(request.path, url_join(os.sep, container_name, 'some/path'))
 
         # return a mocked response
         return unittest.mock.MagicMock(status_code=200,
@@ -276,7 +276,7 @@ def test_verifier_transport(self):
 
         # verify some stuff from the request
         self.assertEqual(request.container, container_name)
-        self.assertEqual(request.path, os.path.join(os.sep, container_name, 'some/table/path/some_item_key'))
+        self.assertEqual(request.path, url_join(os.sep, container_name, 'some/table/path/some_item_key'))
 
         # prepare and output mock
         output = unittest.mock.MagicMock(item={

tests/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

tests/test_client.py

@@ -22,11 +22,11 @@
 
 import future.utils
 
-import v3io.common.helpers
 import v3io.dataplane
 import v3io.dataplane.output
 import v3io.dataplane.response
 import v3io.logger
+from v3io.common.helpers import url_join
 
 
 class Test(unittest.TestCase):
@@ -64,7 +64,7 @@ def _delete_dir(self, path):
 class TestContainer(Test):
     def setUp(self):
         super(TestContainer, self).setUp()
-        self._path = os.path.join(self._test_parent_dir, "v3io-py-test-container")
+        self._path = url_join(self._test_parent_dir, "v3io-py-test-container")
 
         # clean up
         self._delete_dir(self._path)
@@ -84,13 +84,13 @@ def test_get_container_contents_valid(self):
         for object_index in range(file_number):
             self._client.object.put(
                 container=self._container,
-                path=os.path.join(self._path, "object-{0}.txt".format(object_index)),
+                path=url_join(self._path, "object-{0}.txt".format(object_index)),
                 body=body,
             )
 
         for object_index in range(dir_number):
             self._client.object.put(
-                container=self._container, path=os.path.join(self._path, "dir-{0}/".format(object_index))
+                container=self._container, path=url_join(self._path, "dir-{0}/".format(object_index))
             )
 
         response = self._client.container.list(
@@ -111,7 +111,7 @@ class TestStream(Test):
     def setUp(self):
         super(TestStream, self).setUp()
 
-        self._path = os.path.join(self._test_parent_dir, "v3io-py-test-stream")
+        self._path = url_join(self._test_parent_dir, "v3io-py-test-stream")
 
         # clean up
         self._client.stream.delete(container=self._container, stream_path=self._path, raise_for_status=[200, 204, 404])
@@ -136,7 +136,7 @@ def test_delete_stream_with_cg(self):
         # write several "consumer group state" files
         for cg_id in range(3):
             self._client.object.put(
-                container=self._container, path=os.path.join(self._path, "cg{}-state.json".format(cg_id))
+                container=self._container, path=url_join(self._path, "cg{}-state.json".format(cg_id))
             )
 
         # check that the stream doesn't exist
@@ -209,7 +209,7 @@ class TestObject(Test):
     def setUp(self):
         super(TestObject, self).setUp()
 
-        self._object_dir = os.path.join(self._test_parent_dir, "v3io-py-test-object")
+        self._object_dir = url_join(self._test_parent_dir, "v3io-py-test-object")
         self._object_path = self._object_dir + "/obj ect.txt"
 
         # clean up
@@ -315,8 +315,8 @@ class TestSchema(Test):
     def setUp(self):
         super(TestSchema, self).setUp()
 
-        self._schema_dir = os.path.join(self._test_parent_dir, "v3io-py-test-schema")
-        self._schema_path = os.path.join(self._schema_dir, ".#schema")
+        self._schema_dir = url_join(self._test_parent_dir, "v3io-py-test-schema")
+        self._schema_path = url_join(self._schema_dir, ".#schema")
 
         # clean up
         self._delete_dir(self._schema_dir)
@@ -371,7 +371,7 @@ class TestKv(Test):
     def setUp(self):
         super(TestKv, self).setUp()
 
-        self._path = os.path.join(self._test_parent_dir, "some_dir/v3io-py-test-emd")
+        self._path = url_join(self._test_parent_dir, "some_dir/v3io-py-test-emd")
         self._delete_dir(self._path)
 
     def test_kv_array(self):
@@ -399,7 +399,7 @@ def test_kv_array(self):
 
     def test_kv_values(self):
         def _get_int_array():
-            int_array = array.array("l")
+            int_array = array.array("q")
             for value in range(10):
                 int_array.append(value)
 
@@ -587,7 +587,7 @@ def _compare_item_values(self, v1, v2):
             v1 = list(v1)
 
         if v1 != v2:
-            self.fail("Values dont match")
+            self.fail(f"Values don't match: {v1} vs {v2}")
 
     def _compare_item_types(self, v1, v2):
         if isinstance(v1, array.array):
@@ -681,7 +681,7 @@ class TestConnectonErrorRecovery(Test):
     def setUp(self):
         super(TestConnectonErrorRecovery, self).setUp()
 
-        self._object_dir = os.path.join(self._test_parent_dir, "v3io-py-test-connection-error")
+        self._object_dir = url_join(self._test_parent_dir, "v3io-py-test-connection-error")
         self._object_path = self._object_dir + "/object.txt"
 
         self._kv_path = "some_dir/v3io-py-test-emd"
@@ -761,15 +761,15 @@ def test_verifier_transport(self):
         def _verify_object_get(request):
             # verify some stuff from the request
             self.assertEqual(request.container, container_name)
-            self.assertEqual(request.path, os.path.join(os.sep, container_name, "some/path"))
+            self.assertEqual(request.path, url_join("/", container_name, "some/path"))
 
             # return a mocked response
             return unittest.mock.MagicMock(status_code=200, body="some body")
 
         def _verify_kv_get(request):
             # verify some stuff from the request
             self.assertEqual(request.container, container_name)
-            self.assertEqual(request.path, os.path.join(os.sep, container_name, "some/table/path/some_item_key"))
+            self.assertEqual(request.path, url_join("/", container_name, "some/table/path/some_item_key"))
 
             # prepare and output mock
             output = unittest.mock.MagicMock(item={"some_key": "some_value"})

tests/test_client_aio.py

@@ -21,6 +21,7 @@
 
 import v3io.aio.dataplane
 import v3io.dataplane
+from v3io.common.helpers import url_join
 
 
 class Test(unittest.IsolatedAsyncioTestCase):
@@ -56,7 +57,7 @@ async def _delete_dir(self, path):
 class TestContainer(Test):
     async def asyncSetUp(self):
         await super(TestContainer, self).asyncSetUp()
-        self._path = os.path.join(self._test_parent_dir, "v3io-py-test-container")
+        self._path = url_join(self._test_parent_dir, "v3io-py-test-container")
 
         # clean up
         await self._delete_dir(self._path)
@@ -74,13 +75,13 @@ async def test_get_container_contents(self):
         for object_index in range(5):
             await self._client.object.put(
                 container=self._container,
-                path=os.path.join(self._path, "object-{0}.txt".format(object_index)),
+                path=url_join(self._path, "object-{0}.txt".format(object_index)),
                 body=body,
             )
 
         for object_index in range(5):
             await self._client.object.put(
-                container=self._container, path=os.path.join(self._path, "dir-{0}/".format(object_index))
+                container=self._container, path=url_join(self._path, "dir-{0}/".format(object_index))
             )
 
         response = await self._client.container.list(
@@ -103,7 +104,7 @@ class TestStream(Test):
     async def asyncSetUp(self):
         await super(TestStream, self).asyncSetUp()
 
-        self._path = os.path.join(self._test_parent_dir, "v3io-py-test-stream")
+        self._path = url_join(self._test_parent_dir, "v3io-py-test-stream")
 
         # clean up
         await self._client.stream.delete(
@@ -130,7 +131,7 @@ async def test_delete_stream_with_cg(self):
         # write several "consumer group state" files
         for cg_id in range(3):
             await self._client.object.put(
-                container=self._container, path=os.path.join(self._path, "cg{}-state.json".format(cg_id))
+                container=self._container, path=url_join(self._path, "cg{}-state.json".format(cg_id))
             )
 
         # check that the stream doesn't exist
@@ -207,7 +208,7 @@ class TestObject(Test):
     async def asyncSetUp(self):
         await super(TestObject, self).asyncSetUp()
 
-        self._object_dir = os.path.join(self._test_parent_dir, "v3io-py-test-object")
+        self._object_dir = url_join(self._test_parent_dir, "v3io-py-test-object")
         self._object_path = self._object_dir + "/obj ect.txt"
 
         # clean up
@@ -290,7 +291,7 @@ async def test_get_offset(self):
 #         await super(TestSchema, self).asyncSetUp()
 #
 #         self._schema_dir = "/v3io-py-test-schemaa"
-#         self._schema_path = os.path.join(self._schema_dir, ".%23schema")
+#         self._schema_path = url_join(self._schema_dir, ".%23schema")
 #
 #         # clean up
 #         await self._delete_dir(self._schema_dir)
@@ -346,7 +347,7 @@ class TestKv(Test):
     async def asyncSetUp(self):
         await super(TestKv, self).asyncSetUp()
 
-        self._path = os.path.join(self._test_parent_dir, "some_dir/v3io-py-test-emd")
+        self._path = url_join(self._test_parent_dir, "some_dir/v3io-py-test-emd")
         await self._delete_dir(self._path)
 
     async def test_kv_array(self):
@@ -374,7 +375,7 @@ async def test_kv_array(self):
 
     async def test_kv_values(self):
         def _get_int_array():
-            int_array = array.array("l")
+            int_array = array.array("q")
             for value in range(10):
                 int_array.append(value)
 

tests/test_client_errors.py

@@ -1,3 +1,17 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 import itertools
 
 import pytest

tests/test_common.py

@@ -0,0 +1,32 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+from v3io.common.helpers import url_join
+
+
+class Test(unittest.TestCase):
+    def test_url_join(self):
+        self.assertEqual(url_join("a", "b"), "a/b")  # add just exactly one "/" between parts
+        self.assertEqual(url_join("/", "a", "b"), "/a/b")
+        self.assertEqual(url_join("/", "a", "/b"), "/a/b")
+        self.assertEqual(url_join("/", "/a", "b"), "/a/b")
+        self.assertEqual(url_join("/", "/a", "/b"), "/a/b")
+        self.assertEqual(url_join("/", "/a/", "b"), "/a/b")
+        self.assertEqual(url_join("/", "/a/", "/b"), "/a/b")
+        self.assertEqual(url_join("a", "b"), "a/b")  # keep suffix "/" exist/not-exist invariant
+        self.assertEqual(url_join("a", "b/"), "a/b/")
+        self.assertEqual(url_join("a", "b//"), "a/b//")
+        self.assertEqual(url_join("a", "b//", "/"), "a/b//")  # suffix "/" count (if > 0) may change (but we don"t care)

v3io/aio/dataplane/kv.py

@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import os
-
 import v3io.aio.dataplane.kv_cursor
 import v3io.dataplane.model
 import v3io.dataplane.output
 import v3io.dataplane.request
+from v3io.common.helpers import url_join
 
 
 class Model(v3io.dataplane.model.Model):
@@ -303,7 +302,7 @@ async def delete(self, container, table_path, key, access_key=None, raise_for_st
         A `Response` object.
         """
         return self._client.delete_object(
-            container, os.path.join(table_path, key), access_key, raise_for_status, transport_actions
+            container, url_join(table_path, key), access_key, raise_for_status, transport_actions
         )
 
     async def create_schema(self, container, table_path, access_key=None, raise_for_status=None, key=None, fields=None):
@@ -343,7 +342,7 @@ async def create_schema(self, container, table_path, access_key=None, raise_for_
         A `Response` object
         """
         put_object_args = locals()
-        put_object_args["path"] = os.path.join(put_object_args["table_path"], ".#schema")
+        put_object_args["path"] = url_join(put_object_args["table_path"], ".#schema")
         put_object_args["offset"] = 0
         put_object_args["append"] = None
         put_object_args["body"] = self._client._get_schema_contents(key, fields)

v3io/aio/dataplane/stream.py

@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import os
-
 import v3io.dataplane.kv_cursor
 import v3io.dataplane.model
 import v3io.dataplane.output
 import v3io.dataplane.request
+from v3io.common.helpers import url_join
 
 
 class Model(v3io.dataplane.model.Model):
@@ -204,7 +203,7 @@ async def seek(
         ----------
         A `Response` object, whose `output` is `SeekShardOutput`.
         """
-        stream_path = self._ensure_path_ends_with_slash(os.path.join(stream_path, str(shard_id)))
+        stream_path = self._ensure_path_ends_with_slash(url_join(stream_path, str(shard_id)))
 
         return await self._transport.request(
             container,
@@ -307,7 +306,7 @@ async def get_records(
         ----------
         A `Response` object, whose `output` is `GetRecordsOutput`.
         """
-        stream_path = self._ensure_path_ends_with_slash(os.path.join(stream_path, str(shard_id)))
+        stream_path = self._ensure_path_ends_with_slash(url_join(stream_path, str(shard_id)))
 
         return await self._transport.request(
             container,

v3io/common/helpers.py

@@ -12,16 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-def url_join(base, *parts):
-    result = base
 
-    if result[0] != "/":
-        result = "/" + base
 
-    for part in parts:
-        if part[0] != "/":
+def url_join(*parts):
+    result = ""
+    slash_suffix = False
+    for part_index, part in enumerate(parts):
+        if part == "":
+            continue
+        # add slash prefix before part if:
+        # 1. slash suffix did not exit in prev part
+        # 2. slash prefix does not exit in this part
+        # 3. part is not the first
+        if not slash_suffix and part[0] != "/" and part_index != 0:
             result += "/" + part
         else:
-            result += part
-
+            # if slash suffix existed in prev trim slash prefix from this part
+            result += part if not slash_suffix else part.lstrip("/")
+        slash_suffix = True if part[-1] == "/" else False
     return result