From 915dddc7bc629cc5fdd405e82bd4bf6859281ee7 Mon Sep 17 00:00:00 2001 From: DaniAffCH Date: Sun, 10 Mar 2024 10:45:12 +0100 Subject: [PATCH 1/4] add human segmentation c++ demo --- .../CMakeLists.txt | 31 +++ models/human_segmentation_pphumanseg/demo.cpp | 224 ++++++++++++++++++ models/human_segmentation_pphumanseg/demo.py | 5 +- 3 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 models/human_segmentation_pphumanseg/CMakeLists.txt create mode 100644 models/human_segmentation_pphumanseg/demo.cpp diff --git a/models/human_segmentation_pphumanseg/CMakeLists.txt b/models/human_segmentation_pphumanseg/CMakeLists.txt new file mode 100644 index 00000000..a04ec3e3 --- /dev/null +++ b/models/human_segmentation_pphumanseg/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.24) +set(CMAKE_CXX_STANDARD 11) +set(project_name "opencv_zoo_human_segmentation") + +PROJECT (${project_name}) + +set(OPENCV_VERSION "4.9.0") +set(OPENCV_INSTALLATION_PATH "" CACHE PATH "Where to look for OpenCV installation") +find_package(OpenCV ${OPENCV_VERSION} REQUIRED HINTS ${OPENCV_INSTALLATION_PATH}) +# Find OpenCV, you may need to set OpenCV_DIR variable +# to the absolute path to the directory containing OpenCVConfig.cmake file +# via the command line or GUI + +file(GLOB SourceFile + "demo.cpp") +# If the package has been found, several variables will +# be set, you can find the full list with descriptions +# in the OpenCVConfig.cmake file. +# Print some message showing some of them +message(STATUS "OpenCV library status:") +message(STATUS " config: ${OpenCV_DIR}") +message(STATUS " version: ${OpenCV_VERSION}") +message(STATUS " libraries: ${OpenCV_LIBS}") +message(STATUS " include path: ${OpenCV_INCLUDE_DIRS}") + +# Declare the executable target built from your sources +add_executable(${project_name} ${SourceFile}) + +# Link your application with OpenCV libraries +target_link_libraries(${project_name} PRIVATE ${OpenCV_LIBS}) + diff --git a/models/human_segmentation_pphumanseg/demo.cpp b/models/human_segmentation_pphumanseg/demo.cpp new file mode 100644 index 00000000..6971c7e0 --- /dev/null +++ b/models/human_segmentation_pphumanseg/demo.cpp @@ -0,0 +1,224 @@ +#include "opencv2/opencv.hpp" + +#include +#include +#include +#include + +using namespace std; +using namespace cv; +using namespace dnn; + +std::vector> backend_target_pairs = { + {DNN_BACKEND_OPENCV, DNN_TARGET_CPU}, + {DNN_BACKEND_CUDA, DNN_TARGET_CUDA}, + {DNN_BACKEND_CUDA, DNN_TARGET_CUDA_FP16}, + {DNN_BACKEND_TIMVX, DNN_TARGET_NPU}, + {DNN_BACKEND_CANN, DNN_TARGET_NPU} +}; + +class PPHS +{ +private: + Net model; + string modelPath; + + Scalar imageMean = Scalar(0.5,0.5,0.5); + Scalar imageStd = Scalar(0.5,0.5,0.5); + Size modelInputSize = Size(192, 192); + Size currentSize; + + const String inputNames = "x"; + const String outputNames = "save_infer_model/scale_0.tmp_1"; + + int backend_id; + int target_id; + +public: + PPHS(const string& modelPath, + int backend_id = 0, + int target_id = 0) + : modelPath(modelPath), backend_id(backend_id), target_id(target_id) + { + this->model = readNet(modelPath); + this->model.setPreferableBackend(backend_id); + this->model.setPreferableTarget(target_id); + } + + Mat preprocess(const Mat image) + { + this->currentSize = image.size(); + Mat preprocessed = Mat::zeros(this->modelInputSize, image.type()); + resize(image, preprocessed, this->modelInputSize); + + // image normalization + preprocessed.convertTo(preprocessed, CV_32F, 1.0 / 255.0); + preprocessed -= imageMean; + preprocessed /= imageStd; + + return blobFromImage(preprocessed);; + } + + Mat infer(const Mat image) + { + Mat inputBlob = preprocess(image); + + this->model.setInput(inputBlob, this->inputNames); + Mat outputBlob = this->model.forward(this->outputNames); + + return postprocess(outputBlob); + } + + Mat postprocess(Mat image) + { + image = image.reshape(image.size[0], image.size[1]).row(0).reshape(1,image.size[2]); + resize(image, image, this->currentSize, 0, 0, INTER_LINEAR); + image.convertTo(image, CV_8U); + + return image; + } + +}; + + +vector getColorMapList(int num_classes) { + num_classes += 1; + + vector cm(num_classes*3, 0); + + int lab, j; + + for (int i = 0; i < num_classes; ++i) { + lab = i; + j = 0; + + while(lab){ + cm[i] |= (((lab >> 0) & 1) << (7 - j)); + cm[i+num_classes] |= (((lab >> 1) & 1) << (7 - j)); + cm[i+2*num_classes] |= (((lab >> 2) & 1) << (7 - j)); + ++j; + lab >>= 3; + } + + } + + cm.erase(cm.begin(), cm.begin()+3); + + return cm; +}; + +Mat visualize(const Mat& image, const Mat& result, float fps = -1.f, float weight = 0.5) +{ + const Scalar& text_color = Scalar(0, 255, 0); + Mat output_image = image.clone(); + + vector color_map = getColorMapList(256); + + Mat cmm(color_map); + + cmm = cmm.reshape(1,{3,256}); + + if (fps >= 0) + { + putText(output_image, format("FPS: %.2f", fps), Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, text_color, 2); + } + + Mat c1, c2, c3; + + LUT(result, cmm.row(0), c1); + LUT(result, cmm.row(1), c2); + LUT(result, cmm.row(2), c3); + + Mat pseudo_img; + merge(std::vector{c1,c2,c3}, pseudo_img); + + addWeighted(output_image, weight, pseudo_img, 1 - weight, 0, output_image); + + return output_image; +}; + +string keys = +"{ help h | | Print help message. }" +"{ model m | human_segmentation_pphumanseg_2023mar.onnx | Usage: Path to the model, defaults to human_segmentation_pphumanseg_2023mar.onnx }" +"{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera.}" +"{ backend_target t | 0 | Choose one of the backend-target pair to run this demo:\n" + "0: (default) OpenCV implementation + CPU,\n" + "1: CUDA + GPU (CUDA),\n" + "2: CUDA + GPU (CUDA FP16),\n" + "3: TIM-VX + NPU,\n" + "4: CANN + NPU}" +"{ save s | false | Specify to save results.}" +"{ vis v | true | Specify to open a window for result visualization.}" +; + + +int main(int argc, char** argv) +{ + CommandLineParser parser(argc, argv, keys); + + parser.about("Human Segmentation"); + if (parser.has("help")) + { + parser.printMessage(); + return 0; + } + + string modelPath = parser.get("model"); + string inputPath = parser.get("input"); + uint8_t backendTarget = parser.get("backend_target"); + bool saveFlag = parser.get("save"); + bool visFlag = parser.get("vis"); + + if (modelPath.empty()) + CV_Error(Error::StsError, "Model file " + modelPath + " not found"); + + PPHS humanSegmentationModel(modelPath, backend_target_pairs[backendTarget].first, backend_target_pairs[backendTarget].second); + cout << inputPath << endl; + VideoCapture cap; + if (!inputPath.empty()) + cap.open(samples::findFile(inputPath)); + else + cap.open(0); + + if (!cap.isOpened()) + CV_Error(Error::StsError, "Cannot opend video or file"); + + Mat frame; + Mat result; + static const std::string kWinName = "Human Segmentation Demo"; + TickMeter tm; + + while (waitKey(1) < 0) + { + cap >> frame; + + if (frame.empty()) + { + if(inputPath.empty()) + cout << "Frame is empty" << endl; + break; + } + + tm.start(); + result = humanSegmentationModel.infer(frame); + tm.stop(); + + Mat res_frame = visualize(frame, result, tm.getFPS()); + + if(visFlag || inputPath.empty()) + { + imshow(kWinName, res_frame); + if(!inputPath.empty()) + waitKey(0); + } + if(saveFlag) + { + cout << "Results are saved to result.jpg" << endl; + + imwrite("result.jpg", res_frame); + } + } + + return 0; +} + diff --git a/models/human_segmentation_pphumanseg/demo.py b/models/human_segmentation_pphumanseg/demo.py index ea614ac0..313f320c 100644 --- a/models/human_segmentation_pphumanseg/demo.py +++ b/models/human_segmentation_pphumanseg/demo.py @@ -83,8 +83,8 @@ def visualize(image, result, weight=0.6, fps=None): vis_result (np.ndarray): The visualized result. """ color_map = get_color_map_list(256) - color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)] - color_map = np.array(color_map).astype(np.uint8) + color_map = np.array(color_map).reshape(256, 3).astype(np.uint8) + # Use OpenCV LUT for color mapping c1 = cv.LUT(result, color_map[:, 0]) c2 = cv.LUT(result, color_map[:, 1]) @@ -158,3 +158,4 @@ def visualize(image, result, weight=0.6, fps=None): cv.imshow('PPHumanSeg Demo', frame) tm.reset() + From 41f14c2f9997436a1bb46c71dda91746cc2dbb53 Mon Sep 17 00:00:00 2001 From: DaniAffCH Date: Sun, 10 Mar 2024 11:27:46 +0100 Subject: [PATCH 2/4] removed debug print and update README --- .../human_segmentation_pphumanseg/README.md | 19 +++++++++++++++++++ models/human_segmentation_pphumanseg/demo.cpp | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/models/human_segmentation_pphumanseg/README.md b/models/human_segmentation_pphumanseg/README.md index a9ca23c1..61e514b8 100644 --- a/models/human_segmentation_pphumanseg/README.md +++ b/models/human_segmentation_pphumanseg/README.md @@ -4,6 +4,8 @@ This model is ported from [PaddleHub](https://github.com/PaddlePaddle/PaddleHub) ## Demo +### Python + Run the following command to try the demo: ```shell @@ -16,6 +18,23 @@ python demo.py --input /path/to/image -v python demo.py --help ``` +### C++ + +Install latest OpenCV and CMake >= 3.24.0 to get started with: + +```shell +# A typical and default installation path of OpenCV is /usr/local +cmake -B build -D OPENCV_INSTALLATION_PATH=/path/to/opencv/installation . +cmake --build build + +# detect on camera input +./build/opencv_zoo_human_segmentation +# detect on an image +./build/opencv_zoo_human_segmentation -i=/path/to/image +# get help messages +./build/opencv_zoo_human_segmentation -h +``` + ### Example outputs ![webcam demo](./example_outputs/pphumanseg_demo.gif) diff --git a/models/human_segmentation_pphumanseg/demo.cpp b/models/human_segmentation_pphumanseg/demo.cpp index 6971c7e0..022de553 100644 --- a/models/human_segmentation_pphumanseg/demo.cpp +++ b/models/human_segmentation_pphumanseg/demo.cpp @@ -173,7 +173,7 @@ int main(int argc, char** argv) CV_Error(Error::StsError, "Model file " + modelPath + " not found"); PPHS humanSegmentationModel(modelPath, backend_target_pairs[backendTarget].first, backend_target_pairs[backendTarget].second); - cout << inputPath << endl; + VideoCapture cap; if (!inputPath.empty()) cap.open(samples::findFile(inputPath)); From 552b52bac9419e5d1fd5c5f4c0bb1a32179e983a Mon Sep 17 00:00:00 2001 From: DaniAffCH Date: Mon, 11 Mar 2024 12:26:15 +0100 Subject: [PATCH 3/4] inverted colors for consistency --- models/human_segmentation_pphumanseg/demo.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/human_segmentation_pphumanseg/demo.cpp b/models/human_segmentation_pphumanseg/demo.cpp index 022de553..b6c435d4 100644 --- a/models/human_segmentation_pphumanseg/demo.cpp +++ b/models/human_segmentation_pphumanseg/demo.cpp @@ -71,7 +71,9 @@ class PPHS Mat postprocess(Mat image) { - image = image.reshape(image.size[0], image.size[1]).row(0).reshape(1,image.size[2]); + reduceArgMax(image,image,1); + image = image.reshape(1,image.size[2]); + image.convertTo(image, CV_32F); resize(image, image, this->currentSize, 0, 0, INTER_LINEAR); image.convertTo(image, CV_8U); From 5cc258d21a8f5204961129ebd4da6426a098edfc Mon Sep 17 00:00:00 2001 From: DaniAffCH Date: Mon, 11 Mar 2024 12:31:22 +0100 Subject: [PATCH 4/4] adjusted blending weight for visualization --- models/human_segmentation_pphumanseg/demo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/human_segmentation_pphumanseg/demo.cpp b/models/human_segmentation_pphumanseg/demo.cpp index b6c435d4..6408768d 100644 --- a/models/human_segmentation_pphumanseg/demo.cpp +++ b/models/human_segmentation_pphumanseg/demo.cpp @@ -109,7 +109,7 @@ vector getColorMapList(int num_classes) { return cm; }; -Mat visualize(const Mat& image, const Mat& result, float fps = -1.f, float weight = 0.5) +Mat visualize(const Mat& image, const Mat& result, float fps = -1.f, float weight = 0.4) { const Scalar& text_color = Scalar(0, 255, 0); Mat output_image = image.clone();