Frame testing multiai ultrasound (nvidia-holoscan#535)

* Enabling frame validation for multiai-ultrasound application

Signed-off-by: Julien Jomier <[email protected]>

* Added validation frame testing for Python

Signed-off-by: Julien Jomier <[email protected]>

* Fix linting

Signed-off-by: Julien Jomier <[email protected]>

---------

Signed-off-by: Julien Jomier <[email protected]>

Author: jjomier

applications/endoscopy_tool_tracking/python/CMakeLists.txt

@@ -61,7 +61,6 @@ if(BUILD_TESTING)
     DEPENDS endoscopy_tool_tracking_python_test
     PASS_REGULAR_EXPRESSION "Valid video output!"
   )
-
 endif()
 
 # Install application and dependencies into the install/ directory for packaging

applications/multiai_ultrasound/cpp/CMakeLists.txt

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,6 +27,7 @@ target_link_libraries(multiai_ultrasound
     PRIVATE
     holoscan::core
     holoscan::ops::aja
+    holoscan::ops::video_stream_recorder
     holoscan::ops::video_stream_replayer
     holoscan::ops::format_converter
     holoscan::ops::holoviz
@@ -57,10 +58,20 @@ add_dependencies(multiai_ultrasound mgpu_multiai_ultrasound_yaml)
 
 # Add testing
 if(BUILD_TESTING)
-  # Configure the yaml file to only play 10 frames
+  set(RECORDING_DIR ${CMAKE_CURRENT_BINARY_DIR}/recording_output)
+  set(SOURCE_VIDEO_BASENAME cpp_multiai_ultrasound_output)
+  set(VALIDATION_FRAMES_DIR ${CMAKE_SOURCE_DIR}/applications/multiai_ultrasound/testing/)
+
+  file(MAKE_DIRECTORY ${RECORDING_DIR})
+
+  # Configure the yaml file for testing
   file(READ "${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound.yaml" CONFIG_FILE)
-  string(REGEX REPLACE "source:[^\n]*" "source: replayer" CONFIG_FILE ${CONFIG_FILE})
   string(REPLACE "count: 0" "count: 10" CONFIG_FILE ${CONFIG_FILE})
+  string(REGEX REPLACE "source:[^\n]*" "source: replayer" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "record_type: \"none\"" "record_type: \"visualizer\"" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "directory: \"/tmp\"" "directory: \"${RECORDING_DIR}\"" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "basename: \"tensor\"" "basename: \"${SOURCE_VIDEO_BASENAME}\"" CONFIG_FILE ${CONFIG_FILE})
+
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/multiai_ultrasound_testing.yaml" ${CONFIG_FILE})
 
   # Add test
@@ -71,4 +82,20 @@ if(BUILD_TESTING)
   set_tests_properties(multiai_ultrasound_cpp_test PROPERTIES
                        PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking."
                        FAIL_REGULAR_EXPRESSION "[^a-z]Error;ERROR;Failed")
+
+  # Add a test to check the validity of the frames
+  add_test(NAME multiai_ultrasound_cpp_render_test
+    COMMAND python3 ${CMAKE_SOURCE_DIR}/utilities/video_validation.py
+    --source_video_dir ${RECORDING_DIR}
+    --source_video_basename ${SOURCE_VIDEO_BASENAME}
+    --output_dir ${RECORDING_DIR}
+    --validation_frames_dir ${VALIDATION_FRAMES_DIR}
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+
+  set_tests_properties(multiai_ultrasound_cpp_render_test PROPERTIES
+    DEPENDS multiai_ultrasound_cpp_test
+    PASS_REGULAR_EXPRESSION "Valid video output!"
+  )
+
 endif()

applications/multiai_ultrasound/cpp/main.cpp

@@ -23,7 +23,9 @@
 #include <holoscan/operators/holoviz/holoviz.hpp>
 #include <holoscan/operators/inference/inference.hpp>
 #include <holoscan/operators/inference_processor/inference_processor.hpp>
+#include <holoscan/operators/video_stream_recorder/video_stream_recorder.hpp>
 #include <holoscan/operators/video_stream_replayer/video_stream_replayer.hpp>
+
 #include <holoscan/version_config.hpp>
 
 #include <visualizer_icardio.hpp>
@@ -37,12 +39,25 @@ class App : public holoscan::Application {
     if (source == "aja") { is_aja_source_ = true; }
   }
 
+  enum class Record { NONE, INPUT, VISUALIZER };
+
+  void set_record(const std::string& record) {
+    if (record == "input") {
+      record_type_ = Record::INPUT;
+    } else if (record == "visualizer") {
+      record_type_ = Record::VISUALIZER;
+    }
+  }
+
   void set_datapath(const std::string& path) { datapath = path; }
 
   void compose() override {
     using namespace holoscan;
 
     std::shared_ptr<Operator> source;
+    std::shared_ptr<Operator> recorder;
+    std::shared_ptr<Operator> recorder_format_converter;
+
     const std::shared_ptr<CudaStreamPool> cuda_stream_pool =
         make_resource<CudaStreamPool>("cuda_stream");
 
@@ -143,10 +158,34 @@ class App : public holoscan::Application {
         Arg("cuda_stream_pool") = cuda_stream_pool);
 
     auto holoviz = make_operator<ops::HolovizOp>(
-        "holoviz", from_config("holoviz"), Arg("cuda_stream_pool") = cuda_stream_pool);
+        "holoviz", from_config("holoviz"),
+        Arg("enable_render_buffer_output") = (record_type_ == Record::VISUALIZER),
+        Arg("allocator") =
+            make_resource<BlockMemoryPool>("visualizer_allocator",
+                                           (int32_t)nvidia::gxf::MemoryStorageType::kDevice,
+                                           // max from VisualizerICardioOp::tensor_to_shape_
+                                           320 * 320 * 4 * sizeof(uint8_t),
+                                           1 * 8),
+        Arg("cuda_stream_pool") = cuda_stream_pool);
 
-    // Flow definition
 
+    // Add recording operators
+    if (record_type_ != Record::NONE) {
+      if (((record_type_ == Record::INPUT) && is_aja_source_) ||
+          (record_type_ == Record::VISUALIZER)) {
+        recorder_format_converter = make_operator<ops::FormatConverterOp>(
+            "recorder_format_converter",
+            from_config("recorder_format_converter"),
+            Arg("pool") =
+                make_resource<BlockMemoryPool>("pool",
+                                           (int32_t)nvidia::gxf::MemoryStorageType::kDevice,
+                                           320 * 320 * 4 * sizeof(uint8_t),
+                                           1 * 8));
+      }
+      recorder = make_operator<ops::VideoStreamRecorderOp>("recorder", from_config("recorder"));
+    }
+
+    // Flow definition
     const std::string source_port_name = is_aja_source_ ? "video_buffer_output" : "";
     add_flow(source, plax_cham_pre, {{source_port_name, ""}});
     add_flow(source, aortic_ste_pre, {{source_port_name, ""}});
@@ -168,10 +207,26 @@ class App : public holoscan::Application {
     add_flow(visualizer_icardio, holoviz, {{"keyarea_5", "receivers"}});
     add_flow(visualizer_icardio, holoviz, {{"lines", "receivers"}});
     add_flow(visualizer_icardio, holoviz, {{"logo", "receivers"}});
+
+
+    if (record_type_ == Record::INPUT) {
+      if (is_aja_source_) {
+        add_flow(source, recorder_format_converter, {{source_port_name, "source_video"}});
+        add_flow(recorder_format_converter, recorder);
+      } else {
+        add_flow(source, recorder);
+      }
+    } else if (record_type_ == Record::VISUALIZER) {
+      add_flow(holoviz,
+               recorder_format_converter,
+               {{"render_buffer_output", "source_video"}});
+      add_flow(recorder_format_converter, recorder);
+    }
   }
 
  private:
   bool is_aja_source_ = false;
+  Record record_type_ = Record::NONE;
   std::string datapath = "data/multiai_ultrasound";
 };
 
@@ -214,6 +269,10 @@ int main(int argc, char** argv) {
 
   auto source = app->from_config("source").as<std::string>();
   app->set_source(source);
+
+  auto record_type = app->from_config("record_type").as<std::string>();
+  app->set_record(record_type);
+
   if (data_path != "") app->set_datapath(data_path);
   app->run();
 

applications/multiai_ultrasound/cpp/multiai_ultrasound.yaml

@@ -15,7 +15,6 @@
 # limitations under the License.
 ---
 source: "replayer" # or "aja"
-do_record: false   # or 'true' if you want to record input video stream.
 
 replayer:
   basename: "icardio_input1"
@@ -24,6 +23,9 @@ replayer:
   realtime: true  # default: true
   count: 0        # default: 0 (no frame count restriction)
 
+record_type: "none"   # or "input" if you want to record input video stream, or "visualizer" if you want
+                      # to record the visualizer output.
+
 aja:  # AJASourceOp
   width: 1920
   height: 1080
@@ -140,3 +142,11 @@ holoviz:
   width: 320
   height: 320
   use_exclusive_display: false
+
+recorder_format_converter:
+  in_dtype: "rgba8888"
+  out_dtype: "rgb888"
+
+recorder:
+  directory: "/tmp"
+  basename: "tensor"

applications/multiai_ultrasound/python/CMakeLists.txt

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,26 +16,50 @@
 # Add testing
 if(BUILD_TESTING)
   # To get the environment path
-  find_package(holoscan 0.6 REQUIRED CONFIG
-               PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install")
+  find_package(holoscan 1.0 REQUIRED CONFIG PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install")
 
-  # Configure the yaml file to only play 10 frames
+  set(RECORDING_DIR ${CMAKE_CURRENT_BINARY_DIR}/recording_output)
+  set(SOURCE_VIDEO_BASENAME python_multiai_ultrasound_output)
+  set(VALIDATION_FRAMES_DIR ${CMAKE_SOURCE_DIR}/applications/multiai_ultrasound/testing/)
+
+  file(MAKE_DIRECTORY ${RECORDING_DIR})
+
+  # Configure the yaml file for testing
   file(READ "${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound.yaml" CONFIG_FILE)
-  string(REGEX REPLACE "source:[^\n]*" "source: replayer" CONFIG_FILE ${CONFIG_FILE})
   string(REPLACE "count: 0" "count: 10" CONFIG_FILE ${CONFIG_FILE})
-  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/multiai_ultrasound_testing.yaml ${CONFIG_FILE})
+  string(REGEX REPLACE "source:[^\n]*" "source: replayer" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "directory: \"/tmp\"" "directory: \"${RECORDING_DIR}\"" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "basename: \"tensor\"" "basename: \"${SOURCE_VIDEO_BASENAME}\"" CONFIG_FILE ${CONFIG_FILE})
+
+  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/multiai_ultrasound_testing.yaml" ${CONFIG_FILE})
 
   # Add test
   add_test(NAME multiai_ultrasound_python_test
-           COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound.py
-                   --config ${CMAKE_CURRENT_BINARY_DIR}/multiai_ultrasound_testing.yaml
-                   --data "${HOLOHUB_DATA_DIR}/multiai_ultrasound"
-           WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+         COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound.py
+                 --config ${CMAKE_CURRENT_BINARY_DIR}/multiai_ultrasound_testing.yaml
+                 --data "${HOLOHUB_DATA_DIR}/multiai_ultrasound"
+                 --record_type visualizer
+         WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
   set_property(TEST multiai_ultrasound_python_test PROPERTY ENVIRONMENT
-               "PYTHONPATH=${GXF_LIB_DIR}/../python/lib:${CMAKE_BINARY_DIR}/python/lib")
+             "PYTHONPATH=${GXF_LIB_DIR}/../python/lib:${CMAKE_BINARY_DIR}/python/lib")
+
+  set_tests_properties(multiai_ultrasound_python_test PROPERTIES
+              PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking.;"
+              FAIL_REGULAR_EXPRESSION "[^a-z]Error;ERROR;Failed")
+
+  # Add a test to check the validity of the frames
+  add_test(NAME multiai_ultrasound_python_render_test
+    COMMAND python3 ${CMAKE_SOURCE_DIR}/utilities/video_validation.py
+    --source_video_dir ${RECORDING_DIR}
+    --source_video_basename ${SOURCE_VIDEO_BASENAME}
+    --output_dir ${RECORDING_DIR}
+    --validation_frames_dir ${VALIDATION_FRAMES_DIR}
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
 
-  set_tests_properties(multiai_ultrasound_python_test
-                PROPERTIES PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking.;"
-                FAIL_REGULAR_EXPRESSION "[^a-z]Error;ERROR;Failed")
+  set_tests_properties(multiai_ultrasound_python_render_test PROPERTIES
+    DEPENDS multiai_ultrasound_python_test
+    PASS_REGULAR_EXPRESSION "Valid video output!"
+  )
 endif()