Skip to content

Commit f1cdff4

Browse files
mocsharpmocsharp
committed
gRPC-based Endoscopy Tool Tracking application
Signed-off-by: Victor Chang <[email protected]>
1 parent 64983e4 commit f1cdff4

20 files changed

+1435
-3
lines changed

‎.vscode/launch.json

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,76 @@
915915
},
916916
//#endregion ucx_h264_endoscopy_tool_tracking
917917

918+
919+
//#region grpc_endoscopy_tool_tracking
920+
{
921+
"name": "(gdb) grpc_endoscopy_tool_tracking/cpp (edge)",
922+
"type": "cppdbg",
923+
"request": "launch",
924+
"preLaunchTask": "Build grpc_endoscopy_tool_tracking (delay 3s)",
925+
"program": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/grpc_endoscopy_tool_tracking_edge",
926+
"environment": [
927+
{
928+
"name": "HOLOSCAN_INPUT_PATH",
929+
"value": "${env:HOLOHUB_DATA_DIR}/endoscopy"
930+
},
931+
{
932+
"name": "HOLOSCAN_LOG_LEVEL",
933+
"value": "INFO"
934+
}
935+
],
936+
"stopAtEntry": false,
937+
"cwd": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp",
938+
"externalConsole": false,
939+
"MIMode": "gdb",
940+
"setupCommands": [
941+
{
942+
"description": "Enable pretty-printing for gdb",
943+
"text": "-enable-pretty-printing",
944+
"ignoreFailures": true
945+
}
946+
],
947+
"presentation": {
948+
"hidden": false,
949+
"group": "grpc_endoscopy_tool_tracking",
950+
"order": 1
951+
}
952+
},
953+
{
954+
"name": "(gdb) grpc_endoscopy_tool_tracking/cpp (cloud)",
955+
"type": "cppdbg",
956+
"request": "launch",
957+
"preLaunchTask": "Build grpc_endoscopy_tool_tracking",
958+
"program": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/grpc_endoscopy_tool_tracking_cloud",
959+
"environment": [
960+
{
961+
"name": "HOLOSCAN_INPUT_PATH",
962+
"value": "${env:HOLOHUB_DATA_DIR}/endoscopy"
963+
},
964+
{
965+
"name": "HOLOSCAN_LOG_LEVEL",
966+
"value": "INFO"
967+
}
968+
],
969+
"stopAtEntry": false,
970+
"cwd": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp",
971+
"externalConsole": false,
972+
"MIMode": "gdb",
973+
"setupCommands": [
974+
{
975+
"description": "Enable pretty-printing for gdb",
976+
"text": "-enable-pretty-printing",
977+
"ignoreFailures": true
978+
}
979+
],
980+
"presentation": {
981+
"hidden": false,
982+
"group": "grpc_endoscopy_tool_tracking",
983+
"order": 1
984+
}
985+
},
986+
//#endregion grpc_endoscopy_tool_tracking
987+
918988
//#region grpc_h264_endoscopy_tool_tracking
919989
{
920990
"name": "(gdb) grpc_h264_endoscopy_tool_tracking/cpp (edge)",
@@ -1198,6 +1268,20 @@
11981268
"order": 11
11991269
}
12001270
},
1271+
{
1272+
"name": "(compound) grpc_endoscopy_tool_tracking/cpp (cloud & edge)",
1273+
"configurations": [
1274+
"(gdb) grpc_endoscopy_tool_tracking/cpp (cloud)",
1275+
"(gdb) grpc_endoscopy_tool_tracking/cpp (edge)"
1276+
],
1277+
"preLaunchTask": "Build grpc_endoscopy_tool_tracking",
1278+
"stopAll": true,
1279+
"presentation": {
1280+
"hidden": false,
1281+
"group": "grpc_endoscopy_tool_tracking",
1282+
"order": 11
1283+
}
1284+
},
12011285
{
12021286
"name": "(compound) grpc_h264_endoscopy_tool_tracking/cpp (cloud & edge)",
12031287
"configurations": [

‎.vscode/tasks.json

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,53 @@
272272
"group": "ucx_endoscopy_tool_tracking"
273273
}
274274
},
275+
{
276+
"type": "shell",
277+
"label": "Build grpc_endoscopy_tool_tracking",
278+
"command": "./run",
279+
"args": [
280+
"build",
281+
"grpc_endoscopy_tool_tracking",
282+
"--type",
283+
"debug"
284+
],
285+
"options": {
286+
"cwd": "${env:WORKSPACE_DIR}"
287+
},
288+
"group": "build",
289+
"problemMatcher": [],
290+
"detail": "CMake template build task",
291+
"presentation": {
292+
"echo": true,
293+
"reveal": "silent",
294+
"focus": true,
295+
"panel": "dedicated",
296+
"showReuseMessage": false,
297+
"clear": true,
298+
"group": "grpc_endoscopy_tool_tracking"
299+
}
300+
},
301+
{
302+
"type": "shell",
303+
"label": "Build grpc_endoscopy_tool_tracking (delay 3s)",
304+
"command": "sleep 3",
305+
"dependsOn": "Build grpc_endoscopy_tool_tracking",
306+
"options": {
307+
"cwd": "${env:WORKSPACE_DIR}"
308+
},
309+
"group": "build",
310+
"problemMatcher": [],
311+
"detail": "CMake template build task",
312+
"presentation": {
313+
"echo": true,
314+
"reveal": "silent",
315+
"focus": true,
316+
"panel": "dedicated",
317+
"showReuseMessage": false,
318+
"clear": true,
319+
"group": "grpc_endoscopy_tool_tracking"
320+
}
321+
},
275322
{
276323
"type": "shell",
277324
"label": "Build h264_endoscopy_tool_tracking",
@@ -332,6 +379,53 @@
332379
"problemMatcher": [],
333380
"detail": "CMake template build task"
334381
},
382+
{
383+
"type": "shell",
384+
"label": "Build grpc_endoscopy_tool_tracking",
385+
"command": "./run",
386+
"args": [
387+
"build",
388+
"grpc_endoscopy_tool_tracking",
389+
"--type",
390+
"debug"
391+
],
392+
"options": {
393+
"cwd": "${env:WORKSPACE_DIR}"
394+
},
395+
"group": "build",
396+
"problemMatcher": [],
397+
"detail": "CMake template build task",
398+
"presentation": {
399+
"echo": true,
400+
"reveal": "silent",
401+
"focus": true,
402+
"panel": "dedicated",
403+
"showReuseMessage": false,
404+
"clear": true,
405+
"group": "grpc_endoscopy_tool_tracking"
406+
}
407+
},
408+
{
409+
"type": "shell",
410+
"label": "Build grpc_endoscopy_tool_tracking (delay 3s)",
411+
"command": "sleep 3",
412+
"dependsOn": "Build grpc_endoscopy_tool_tracking",
413+
"options": {
414+
"cwd": "${env:WORKSPACE_DIR}"
415+
},
416+
"group": "build",
417+
"problemMatcher": [],
418+
"detail": "CMake template build task",
419+
"presentation": {
420+
"echo": true,
421+
"reveal": "silent",
422+
"focus": true,
423+
"panel": "dedicated",
424+
"showReuseMessage": false,
425+
"clear": true,
426+
"group": "grpc_endoscopy_tool_tracking"
427+
}
428+
},
335429
{
336430
"type": "shell",
337431
"label": "Build grpc_h264_endoscopy_tool_tracking",

‎applications/distributed/grpc/CMakeLists.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@
1414
# limitations under the License.
1515

1616

17+
18+
add_holohub_application(grpc_endoscopy_tool_tracking DEPENDS
19+
OPERATORS lstm_tensor_rt_inference
20+
tool_tracking_postprocessor
21+
grpc_operators)
22+
1723
add_holohub_application(grpc_h264_endoscopy_tool_tracking DEPENDS
1824
OPERATORS video_encoder
1925
tensor_to_video_buffer

‎applications/distributed/grpc/grpc_endoscopy_tool_tracking/CMakeLists.txt

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
cmake_minimum_required(VERSION 3.20)
17+
project(grpc_endoscopy_tool_tracking LANGUAGES NONE)
18+
19+
add_subdirectory(cpp)

‎applications/distributed/grpc/grpc_endoscopy_tool_tracking/README.md

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Endoscopy Tool Tracking Application with gRPC
2+
3+
This application demonstrates how to offload heavy workloads to a remote Holoscan application using gRPC.
4+
5+
## Overview
6+
7+
In this sample application, we divided the Endoscopy Tool Tracking application into a server and client application where the two communicate via gRPC.
8+
9+
The client application inputs a video file and streams the video frames to the server application. The server application handles the heavy workloads of inferencing and post-processing of the video frames. It receives the video frames, processes each frame through the endoscopy tool tracking pipeline, and then streams the results to the client.
10+
11+
![Overview](static/overview.png)
12+
*Endoscopy Tool Tracking Application with gRPC*
13+
14+
From the diagram above, we can see that both the App Cloud (the server) and the App Edge (the client) are very similar to the standalone [Endoscopy Tool Tracking](../../../endoscopy_tool_tracking/) application. This section will only describe the differences; for details on inference and post-processing, please refer to the link above.
15+
16+
On the client side, we provided two examples, one using a single fragment and another one using two fragments. When comparing the client side to the standalone [Endoscopy Tool Tracking](../../../endoscopy_tool_tracking/) application, the differences are the queues and the gRPC client. We added the following:
17+
- **Outgoing Requests** operator (`GrpcClientRequestOp`): It converts the video frames (GXF entities) received from the *Video Stream Replayer* operator into `EntityRequest` protobuf messages and queues each frame in the *Request Queue*.
18+
- **gRPC Service & Client** (`EntityClientService` & `EntityClient`): The gRPC Service is responsible for controlling the life cycle of the gRPC client. The client connects to the remote gRPC server and then sends the requests found in the *Request Queue*. When it receives a response, it converts it into a GXF entity and queues it in the *Response Queue*.
19+
- **Incoming Responses** operator (`GrpcClientResponseOp`): This operator is configured with an `AsynchronousCondition` condition to check the availability of the *Response Queue*. When notified of available responses in the queue, it dequeues each item and emits each to the output port.
20+
21+
The App Cloud (the server) application consists of a gRPC server and a few components for managing Holoscan applications. When the server receives a new remote procedure call in this sample application, it launches a new instance of the Endoscopy Tool Tracking application. This is facilitated by the `ApplicationFactory` used for application registration.
22+
23+
Under the hood, the Endoscopy Tool Tracking application here inherits a custom base class (`HoloscanGrpcApplication`) which manages the `Request Queue` and the `Response Queue` as well as the `GrpcServerRequestOp` and `GrpcServerResponseOp` operators for receiving requests and serving results, respectively. When the RPC is complete, the instance of the Endoscopy Tool Tracking application is destroyed and ready to serve the subsequent request.
24+
25+
26+
## Requirements
27+
28+
### Data
29+
30+
[📦️ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data)
31+
32+
The data is automatically downloaded when building the application.
33+
34+
## Building and Running gRPC Endoscopy Tool Tracking Application
35+
36+
* Building and running the application from the top level Holohub directory:
37+
38+
### C++
39+
40+
```bash
41+
# Start the gRPC Server
42+
./dev_container build_and_run grpc_endoscopy_tool_tracking --run_args cloud [--language cpp]
43+
44+
# Start the gRPC Client
45+
./dev_container build_and_run grpc_endoscopy_tool_tracking --run_args edge [--language cpp]
46+
```
47+
48+
49+
## Dev Container
50+
51+
To start the the Dev Container, run the following command from the root directory of Holohub:
52+
53+
```bash
54+
./dev_container vscode
55+
```
56+
57+
### VS Code Launch Profiles
58+
59+
#### C++
60+
61+
The following launch profiles are available:
62+
63+
- **(compound) grpc_endoscopy_tool_tracking/cpp (cloud & edge)**: Launch both the gRPC server and the client.
64+
- **(gdb) grpc_endoscopy_tool_tracking/cpp (cloud)**: Launch the gRPC server.
65+
- **(gdb) grpc_endoscopy_tool_tracking/cpp (edge)**: Launch the gRPC client.
66+
67+
68+
## Limitations & Known Issues
69+
70+
- The connection between the server and the client is controlled by `rpc_timeout`. If no data is received or sent within the configured time, it assumes the call has been completed and hangs up. The `rpc_timeout` value can be configured in the [endoscopy_tool_tracking.yaml](./cpp/endoscopy_tool_tracking.yaml) file with a default of 5 seconds. Increasing this value may help on a slow network.
71+
- The server can serve one request at any given time. Any subsequent call receives a `grpc::StatusCode::RESOURCE_EXHAUSTED` status.
72+
- When debugging using the compound profile, the server may not be ready to serve, resulting in errors with the client application. When this happens, open [tasks.json](../../../.vscode/tasks.json), find `Build grpc_endoscopy_tool_tracking (delay 3s)`, and adjust the `command` field with a higher sleep value.
73+
- The client is expected to exit with the following error. It is how the client application terminates when it completes streaming and displays the entire video.
74+
```bash
75+
[error] [program.cpp:614] Event notification 2 for entity [video_in__outgoing_requests] with id [33] received in an unexpected state [Origin]
76+
```
77+

‎applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/CMakeLists.txt

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
cmake_minimum_required(VERSION 3.20)
17+
project(grpc_endoscopy_tool_tracking CXX)
18+
19+
find_package(holoscan 2.6 REQUIRED CONFIG
20+
PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install")
21+
22+
add_executable(grpc_endoscopy_tool_tracking_cloud
23+
cloud/app_cloud_main.cpp
24+
cloud/grpc_service.hpp
25+
cloud/app_cloud_pipeline.hpp
26+
)
27+
28+
add_executable(grpc_endoscopy_tool_tracking_edge
29+
edge/app_edge_main.cpp
30+
edge/app_edge.hpp
31+
edge/video_input_fragment.hpp
32+
edge/viz_fragment.hpp
33+
)
34+
35+
target_link_libraries(grpc_endoscopy_tool_tracking_cloud
36+
PRIVATE
37+
holoscan::core
38+
holoscan::ops::format_converter
39+
lstm_tensor_rt_inference
40+
tool_tracking_postprocessor
41+
grpc_operators
42+
)
43+
44+
target_link_libraries(grpc_endoscopy_tool_tracking_edge
45+
PRIVATE
46+
holoscan::core
47+
holoscan::ops::holoviz
48+
holoscan::ops::video_stream_replayer
49+
grpc_operators
50+
)
51+
52+
# Copy the config to the binary directory
53+
add_custom_target(grpc_endoscopy_tool_tracking_yaml
54+
COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.yaml" ${CMAKE_CURRENT_BINARY_DIR}
55+
DEPENDS "endoscopy_tool_tracking.yaml"
56+
BYPRODUCTS "endoscopy_tool_tracking.yaml"
57+
)
58+
add_dependencies(grpc_endoscopy_tool_tracking_edge grpc_endoscopy_tool_tracking_yaml)
59+
60+
# Copy the launch script
61+
add_custom_target(grpc_endoscopy_tool_tracking_launch_sh
62+
COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/launch.sh" ${CMAKE_CURRENT_BINARY_DIR}
63+
DEPENDS "launch.sh"
64+
BYPRODUCTS "launch.sh"
65+
)
66+
add_dependencies(grpc_endoscopy_tool_tracking_edge grpc_endoscopy_tool_tracking_launch_sh)
67+
68+
# Default to download datasets
69+
option(HOLOHUB_DOWNLOAD_DATASETS "Download datasets" ON)
70+
71+
# Download the endoscopy sample data
72+
if(HOLOHUB_DOWNLOAD_DATASETS)
73+
include(holoscan_download_data)
74+
holoscan_download_data(endoscopy
75+
URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_endoscopy_sample_data/versions/20230222/zip
76+
DOWNLOAD_NAME holoscan_endoscopy_sample_data_20230222.zip
77+
URL_MD5 d54f84a562d29ed560a87d2607eba973
78+
DOWNLOAD_DIR ${HOLOHUB_DATA_DIR}
79+
)
80+
add_dependencies(grpc_endoscopy_tool_tracking_edge endoscopy_data)
81+
endif()

0 commit comments

Comments
 (0)