Tensor. Extended tets, fixes.

Signed-off-by: Michał Zientkiewicz <[email protected]>

dali/c_api_2/c_api_internal_test.cc

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#define DALI_ALLOW_NEW_C_API
-#include "dali/dali.h"  // NOLINT
 #include <gtest/gtest.h>
 #include <stdexcept>
+#define DALI_ALLOW_NEW_C_API
+#include "dali/dali.h"
 #include "dali/c_api_2/error_handling.h"
 #include "dali/core/cuda_error.h"
 

dali/c_api_2/data_objects.cc

@@ -348,3 +348,15 @@ daliResult_t daliTensorListGetTensorDesc(
   *out_tensor = ptr->GetTensorDesc(sample_idx);
   DALI_EPILOG();
 }
+
+daliResult_t daliTensorListViewAsTensor(
+      daliTensorList_h tensor_list,
+      daliTensor_h *out_tensor) {
+  DALI_PROLOG();
+  auto *ptr = ToPointer(tensor_list);
+  if (!out_tensor)
+    throw std::invalid_argument("The output parameter `out_tensor` must not be NULL.");
+  auto t = ptr->ViewAsTensor();
+  *out_tensor = t.release();  // no throwing allowed after this line
+  DALI_EPILOG();
+}

dali/c_api_2/data_objects.h

@@ -18,6 +18,8 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <utility>
+#include <vector>
 #define DALI_ALLOW_NEW_C_API
 #include "dali/dali.h"
 #include "dali/pipeline/data/tensor_list.h"
@@ -148,7 +150,7 @@ struct BufferDeleter {
 template <typename Backend>
 class TensorWrapper : public ITensor {
  public:
-  TensorWrapper(std::shared_ptr<Tensor<Backend>> t) : t_(std::move(t)) {};
+  explicit TensorWrapper(std::shared_ptr<Tensor<Backend>> t) : t_(std::move(t)) {}
 
   void Resize(
       int ndim,
@@ -293,7 +295,7 @@ RefCountedPtr<TensorWrapper<Backend>> Wrap(std::shared_ptr<Tensor<Backend>> tl)
 template <typename Backend>
 class TensorListWrapper : public ITensorList {
  public:
-  TensorListWrapper(std::shared_ptr<TensorList<Backend>> tl) : tl_(std::move(tl)) {};
+  explicit TensorListWrapper(std::shared_ptr<TensorList<Backend>> tl) : tl_(std::move(tl)) {}
 
   void Resize(
       int num_samples,
@@ -351,6 +353,7 @@ class TensorListWrapper : public ITensorList {
     tl_->SetSize(num_samples);
     tl_->set_sample_dim(ndim);
     tl_->SetLayout(new_layout);
+    tl_->set_type(dtype);
     ptrdiff_t next_offset = 0;
     auto type_info = TypeTable::GetTypeInfo(dtype);
     auto element_size = type_info.size();
@@ -375,6 +378,7 @@ class TensorListWrapper : public ITensorList {
     }
 
     if (is_contiguous) {
+      tl_->SetContiguity(BatchContiguity::Contiguous);
       std::vector<int64_t> shape_data(shapes, shapes + ndim * num_samples);
       TensorListShape<> tl_shape(shape_data, num_samples, ndim);
       tl_->ShareData(
@@ -387,6 +391,7 @@ class TensorListWrapper : public ITensorList {
         tl_->order(),
         new_layout);
     } else {
+      tl_->SetContiguity(BatchContiguity::Automatic);
       next_offset = 0;
 
       for (int i = 0; i < num_samples; i++) {
@@ -583,7 +588,7 @@ class TensorListWrapper : public ITensorList {
       tl_->order(),
       tl_->ready_event());
     TensorLayout layout = tl_->GetLayout();
-    if (layout.size() == tshape.sample_dim()) {
+    if (layout.size() == lshape.sample_dim()) {
       t->SetLayout("N" + layout);
     }
     return Wrap(std::move(t));

dali/c_api_2/data_objects_test.cc

@@ -65,7 +65,6 @@ inline auto CreateTensorList(daliBufferPlacement_t placement) {
 void TestTensorListResize(daliStorageDevice_t storage_device) {
   daliBufferPlacement_t placement{};
   placement.device_type = storage_device;
-  auto tl = CreateTensorList(placement);
   int64_t shapes[] = {
     480, 640, 3,
     600, 800, 4,
@@ -74,6 +73,7 @@ void TestTensorListResize(daliStorageDevice_t storage_device) {
   };
   daliDataType_t dtype = DALI_UINT32;
 
+  auto tl = CreateTensorList(placement);
   EXPECT_EQ(daliTensorListResize(tl, 4, 3, nullptr, dtype, nullptr), DALI_ERROR_INVALID_ARGUMENT);
   EXPECT_EQ(daliTensorListResize(tl, -1, 3, shapes, dtype, nullptr), DALI_ERROR_INVALID_ARGUMENT);
   EXPECT_EQ(daliTensorListResize(tl, 4, -1, shapes, dtype, nullptr), DALI_ERROR_INVALID_ARGUMENT);
@@ -115,11 +115,39 @@ void TestTensorListResize(daliStorageDevice_t storage_device) {
   }
 }
 
+struct TestDeleterCtx {
+  void *expected_data;
+  int buffer_delete_count;
+  int context_delete_count;
+};
+
+template <typename element_t>
+inline std::pair<daliDeleter_t, std::unique_ptr<TestDeleterCtx>>
+MakeTestDeleter(element_t *expected_data) {
+  auto ctx = std::unique_ptr<TestDeleterCtx>(new TestDeleterCtx{ expected_data, 0, 0 });
+  daliDeleter_t deleter = {};
+  deleter.deleter_ctx = ctx.get();
+  deleter.delete_buffer = [](void *vctx, void *data, const cudaStream_t *stream) {
+    ASSERT_NE(data, nullptr);
+    auto *ctx = static_cast<TestDeleterCtx *>(vctx);
+    EXPECT_EQ(ctx->context_delete_count, 0);
+    EXPECT_EQ(ctx->buffer_delete_count, 0);
+    EXPECT_EQ(data, ctx->expected_data);
+    ctx->buffer_delete_count++;
+    delete [] static_cast<element_t *>(data);
+  };
+  deleter.destroy_context = [](void *vctx) {
+    auto *ctx = static_cast<TestDeleterCtx *>(vctx);
+    EXPECT_EQ(ctx->context_delete_count, 0);
+    EXPECT_EQ(ctx->buffer_delete_count, 1);
+    ctx->context_delete_count++;
+  };
+  return { deleter, std::move(ctx) };
+}
 
 TEST(CAPI2_TensorListTest, AttachBuffer) {
   daliBufferPlacement_t placement{};
   placement.device_type = DALI_STORAGE_CPU;
-  auto tl = CreateTensorList(placement);
   using element_t = int;
   daliDataType_t dtype = dali::type2id<element_t>::value;
   dali::TensorListShape<> lshape({
@@ -135,30 +163,9 @@ TEST(CAPI2_TensorListTest, AttachBuffer) {
   for (int i = 1; i < 4; i++)
     offsets[i] = offsets[i - 1] + volume(lshape[i - 1]) * sizeof(element_t);
 
-  struct DeleterCtx {
-    void *expected_data;
-    int buffer_delete_count;
-    int context_delete_count;
-  };
-  DeleterCtx ctx = { data.get(), 0, 0 };
-  daliDeleter_t deleter = {};
-  deleter.deleter_ctx = &ctx;
-  deleter.delete_buffer = [](void *vctx, void *data, const cudaStream_t *stream) {
-    ASSERT_NE(data, nullptr);
-    auto *ctx = static_cast<DeleterCtx *>(vctx);
-    EXPECT_EQ(ctx->context_delete_count, 0);
-    EXPECT_EQ(ctx->buffer_delete_count, 0);
-    EXPECT_EQ(data, ctx->expected_data);
-    ctx->buffer_delete_count++;
-    delete [] static_cast<element_t *>(data);
-  };
-  deleter.destroy_context = [](void *vctx) {
-    auto *ctx = static_cast<DeleterCtx *>(vctx);
-    EXPECT_EQ(ctx->context_delete_count, 0);
-    EXPECT_EQ(ctx->buffer_delete_count, 1);
-    ctx->context_delete_count++;
-  };
+  auto [deleter, ctx] = MakeTestDeleter(data.get());
 
+  auto tl = CreateTensorList(placement);
   ASSERT_EQ(daliTensorListAttachBuffer(
       tl,
       lshape.num_samples(),
@@ -189,10 +196,114 @@ TEST(CAPI2_TensorListTest, AttachBuffer) {
 
   tl.reset();
 
-  EXPECT_EQ(ctx.buffer_delete_count, 1) << "Buffer deleter not called";
-  EXPECT_EQ(ctx.context_delete_count, 1) << "Deleter context not destroyed";
+  EXPECT_EQ(ctx->buffer_delete_count, 1) << "Buffer deleter not called";
+  EXPECT_EQ(ctx->context_delete_count, 1) << "Deleter context not destroyed";
 }
 
+
+TEST(CAPI2_TensorListTest, ViewAsTensor) {
+  daliBufferPlacement_t placement{};
+  placement.device_type = DALI_STORAGE_CPU;
+  using element_t = int;
+  daliDataType_t dtype = dali::type2id<element_t>::value;
+  dali::TensorListShape<> lshape = dali::uniform_list_shape(4, { 480, 640, 3 });
+  auto size = lshape.num_elements();
+  std::unique_ptr<element_t> data(new element_t[size]);
+
+  ptrdiff_t sample_size = volume(lshape[0]) * sizeof(element_t);
+
+  ptrdiff_t offsets[4] = {
+    0,
+    1 * sample_size,
+    2 * sample_size,
+    3 * sample_size,
+  };
+
+  auto [deleter, ctx] = MakeTestDeleter(data.get());
+
+  auto tl = CreateTensorList(placement);
+  ASSERT_EQ(daliTensorListAttachBuffer(
+      tl,
+      lshape.num_samples(),
+      lshape.sample_dim(),
+      lshape.data(),
+      dtype,
+      "HWC",
+      data.get(),
+      offsets,
+      deleter), DALI_SUCCESS);
+
+  void *data_ptr = data.release();  // the buffer is now owned by the tensor list
+
+  daliTensor_h ht = nullptr;
+  EXPECT_EQ(daliTensorListViewAsTensor(tl, &ht), DALI_SUCCESS);
+  ASSERT_NE(ht, nullptr);
+  dali::c_api::TensorHandle t(ht);
+
+  daliTensorDesc_t desc{};
+  EXPECT_EQ(daliTensorGetDesc(t, &desc), DALI_SUCCESS);
+  EXPECT_EQ(desc.data, data_ptr);
+  EXPECT_EQ(desc.shape[0], lshape.num_samples());
+  ASSERT_EQ(desc.ndim, 4);
+  ASSERT_NE(desc.shape, nullptr);
+  EXPECT_EQ(desc.shape[1], lshape[0][0]);
+  EXPECT_EQ(desc.shape[2], lshape[0][1]);
+  EXPECT_EQ(desc.shape[3], lshape[0][2]);
+  EXPECT_STREQ(desc.layout, "NHWC");
+  EXPECT_EQ(desc.dtype, dtype);
+
+  tl.reset();
+
+  EXPECT_EQ(ctx->buffer_delete_count, 0) << "Buffer prematurely destroyed";
+  EXPECT_EQ(ctx->context_delete_count, 0) << "Deleter context prematurely destroyed";
+
+  t.reset();
+
+  EXPECT_EQ(ctx->buffer_delete_count, 1) << "Buffer deleter not called";
+  EXPECT_EQ(ctx->context_delete_count, 1) << "Deleter context not destroyed";
+}
+
+
+TEST(CAPI2_TensorListTest, ViewAsTensorError) {
+  daliBufferPlacement_t placement{};
+  placement.device_type = DALI_STORAGE_CPU;
+  using element_t = int;
+  daliDataType_t dtype = dali::type2id<element_t>::value;
+  dali::TensorListShape<> lshape = dali::uniform_list_shape(4, { 480, 640, 3 });
+  auto size = lshape.num_elements();
+  std::unique_ptr<element_t> data(new element_t[size]);
+
+  ptrdiff_t sample_size = volume(lshape[0]) * sizeof(element_t);
+
+  // The samples are not in order
+  ptrdiff_t offsets[4] = {
+    0,
+    2 * sample_size,
+    1 * sample_size,
+    3 * sample_size,
+  };
+
+  auto [deleter, ctx] = MakeTestDeleter(data.get());
+
+  auto tl = CreateTensorList(placement);
+  ASSERT_EQ(daliTensorListAttachBuffer(
+      tl,
+      lshape.num_samples(),
+      lshape.sample_dim(),
+      lshape.data(),
+      dtype,
+      "HWC",
+      data.get(),
+      offsets,
+      deleter), DALI_SUCCESS);
+
+  void *data_ptr = data.release();  // the buffer is now owned by the tensor list
+
+  daliTensor_h ht = nullptr;
+  EXPECT_EQ(daliTensorListViewAsTensor(tl, &ht), DALI_ERROR_INVALID_OPERATION);
+}
+
+
 TEST(CAPI2_TensorListTest, ResizeCPU) {
   TestTensorListResize(DALI_STORAGE_CPU);
 }

dali/c_api_2/error_handling.h

@@ -45,10 +45,10 @@ DLL_PUBLIC daliResult_t HandleError(std::exception_ptr ex);
 DLL_PUBLIC daliResult_t CheckInit();
 
 class InvalidHandle : public std::invalid_argument {
-public:
+ public:
   InvalidHandle() : std::invalid_argument("The handle is invalid") {}
-  InvalidHandle(const std::string &what) : std::invalid_argument(what) {}
-  InvalidHandle(const char *what) : std::invalid_argument(what) {}
+  explicit InvalidHandle(const std::string &what) : std::invalid_argument(what) {}
+  explicit InvalidHandle(const char *what) : std::invalid_argument(what) {}
 };
 
 inline InvalidHandle NullHandle() { return InvalidHandle("The handle must not be NULL."); }

dali/c_api_2/ref_counting.h

@@ -92,7 +92,7 @@ class RefCountedPtr {
   void reset() noexcept {
     if (ptr_)
       ptr_->DecRef();
-    ptr_= nullptr;
+    ptr_ = nullptr;
   }
 
   [[nodiscard]] T *release() noexcept {

dali/c_api_2/validation.h

@@ -15,14 +15,14 @@
 #ifndef DALI_C_API_2_VALIDATION_H_
 #define DALI_C_API_2_VALIDATION_H_
 
+#include <stdexcept>
+#include <optional>
 #define DALI_ALLOW_NEW_C_API
 #include "dali/dali.h"
 #include "dali/core/format.h"
 #include "dali/core/span.h"
 #include "dali/core/tensor_shape_print.h"
 #include "dali/pipeline/data/types.h"
-#include <stdexcept>
-#include <optional>
 
 namespace dali::c_api {