Share usd arrays with arnold core (#2148)

* shared (foreign) arrays wip

* testsuite now passing in hydra mode

* remove iostream and add deletion of points

* test buffer holder to benefit from usd deduplication

* cache the shared AtArray for deduplication

* rename AiArrayMakeSharedXXX to AiArrayMakeShared

* fix potential data race

* remove AtArray map

* moving buffer holder per hdmesh

* replace unordered_map by vector of key value pair

* check bufferMap is empty when deleting HdMesh + comments

* deallocate the shared VtValues by resetting the arnold params

* add an ArrayCopier and allow enabling shared arrays at compilation time

* remove temporary values

* share uvs and normals arrays

* rename functions and add deletion of uvs an normals shared arrays, move the code in dedicated file

* remove unwanted changes on api compatibility

* revert unwanted commits

* revert unwanted commits

* remove unwanted commit

* add function aliases for aiArrayConst*

* fix build on more recent versions

* update changelog

CHANGELOG.md

@@ -3,9 +3,13 @@
 
 ## Next Feature release
 
+### Features
+
+- [usd#2148](https://github.com/Autodesk/arnold-usd/issues/2248) - Leverage new Shared Arrays API in the render delegate.
+
 ### Bug fixes
 
-[usd#2208](https://github.com/Autodesk/arnold-usd/issues/2208) - Fix unnecessary allocations in the instancer and mesh.
+- [usd#2208](https://github.com/Autodesk/arnold-usd/issues/2208) - Fix unnecessary allocations in the instancer and mesh.
 
 ## Pending Feature release
 

SConstruct

@@ -125,6 +125,7 @@ vars.AddVariables(
     BoolVariable('PROC_SCENE_FORMAT', 'Whether or not to build the procedural with a scene format plugin.', True),
     BoolVariable('DISABLE_CXX11_ABI', 'Disable the use of the CXX11 abi for gcc/clang', False),
     BoolVariable('ENABLE_HYDRA_IN_USD_PROCEDURAL', 'Enable building hydra render delegate in the usd procedural', False),
+    BoolVariable('ENABLE_SHARED_ARRAYS', 'Enable the use of shared arrays in hydra', False),
     BoolVariable('BUILD_USDGENSCHEMA_ARNOLD', 'Whether or not to build the simplified usdgenschema', False),
     BoolVariable('IGNORE_ARCH_FLAGS', 'Ignore the arch flags when compiling usdgenschema', False),
     StringVariable('BOOST_LIB_NAME', 'Boost library name pattern', 'boost_%s'),

cmake/utils/options.cmake

@@ -21,6 +21,7 @@ option(BUILD_BOOST_ALL_NO_LIB "Disable linking of boost libraries from boost hea
 option(BUILD_DISABLE_CXX11_ABI "Disable the use of the new CXX11 ABI" OFF)
 option(BUILD_HEADERS_AS_SOURCES "Add headers are source files to the target to help when generating IDE projects." OFF)
 option(ENABLE_HYDRA_IN_USD_PROCEDURAL "Enable hydra in the  procedural (experimental)" OFF)
+option(ENABLE_SHARED_ARRAYS "Enable using shared arrays" OFF)
 set(USD_OVERRIDE_PLUGINPATH_NAME "PXR_PLUGINPATH_NAME" CACHE STRING "Override the plugin path name for the USD libraries. Used when running the testsuite with a static procedural")
 
 option(BUILD_SCHEMAS "Builds the USD Schemas" ON)

libs/common/common_utils.h

@@ -33,6 +33,11 @@
 
 PXR_NAMESPACE_OPEN_SCOPE
 
+#if ARNOLD_VERSION_NUM < 70307
+constexpr auto AiArrayMapConst = AiArrayMap;
+constexpr auto AiArrayUnmapConst = AiArrayUnmap;
+#endif
+
 // Older versions of Arnold had imagers defined as drivers
 #if ARNOLD_VERSION_NUM < 70301
 static const int AI_NODE_IMAGER = AI_NODE_DRIVER;

libs/common/shape_utils.cpp

@@ -271,13 +271,15 @@ AtArray* GenerateVertexIdxs(const VtIntArray& indices, AtArray* vidxs)
         return AiArrayAllocate(0, 1, AI_TYPE_UINT);
     }
     // This primvar has no indices, so we return a copy of vidxs
+    // NOTE that if vidx is a shared array, it will create a shallow copy of it and reference it internally
+    // which can will potentially lead to a call on double free memory error
     if (indices.empty())
         return AiArrayCopy(vidxs);
 
     const auto numIdxs = static_cast<uint32_t>(AiArrayGetNumElements(vidxs));
-    auto* array = AiArrayAllocate(numIdxs, 1, AI_TYPE_UINT);
-    auto* out = static_cast<uint32_t*>(AiArrayMap(array));
-    auto* in = static_cast<uint32_t*>(AiArrayMap(vidxs));
+    AtArray* array = AiArrayAllocate(numIdxs, 1, AI_TYPE_UINT);
+    uint32_t* out = static_cast<uint32_t*>(AiArrayMap(array));
+    const uint32_t* in = static_cast<const uint32_t*>(AiArrayMapConst(vidxs));
 
     for (unsigned int i = 0; i < numIdxs; ++i) {
         if (in[i] >= indices.size()) {
@@ -288,7 +290,7 @@ AtArray* GenerateVertexIdxs(const VtIntArray& indices, AtArray* vidxs)
     }
 
     AiArrayUnmap(array);
-    AiArrayUnmap(vidxs);
+    AiArrayUnmapConst(vidxs);
     return array;
 }
 

libs/common/shape_utils.h

@@ -26,7 +26,8 @@
 
 #include <pxr/base/arch/export.h>
 
-#include <ai.h>
+#include "common_utils.h"
+
 PXR_NAMESPACE_OPEN_SCOPE
 
 /// Read subdivision creases from a Usd or a Hydra mesh.

libs/render_delegate/CMakeLists.txt

@@ -68,7 +68,9 @@ if (${USD_HAS_FULLSCREEN_SHADER})
 endif ()
 
 target_link_libraries(render_delegate INTERFACE common)
-
+if (ENABLE_SHARED_ARRAYS)
+    target_compile_definitions(render_delegate PUBLIC ENABLE_SHARED_ARRAYS=1)
+endif()
 target_compile_definitions(render_delegate PRIVATE "HDARNOLD_EXPORTS=1")
 
 source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SRC} ${HDR})

libs/render_delegate/SConscript

@@ -54,6 +54,9 @@ if local_env['USD_HAS_UPDATED_COMPOSITOR']:
 if local_env['USD_HAS_FULLSCREEN_SHADER']:
     local_env.Append(CPPDEFINES=['USD_HAS_FULLSCREEN_SHADER'])
 
+if local_env['ENABLE_SHARED_ARRAYS']:
+    local_env.Append(CPPDEFINES = ['ENABLE_SHARED_ARRAYS'])
+
 local_env.Append(CPPDEFINES=['HDARNOLD_EXPORTS'])
 local_env.Append(CPPPATH = [os.path.join(env['BUILD_ROOT_DIR'], 'libs', 'render_delegate')])
 local_env.Append(LIBS = ['ai'])