C++ Demo - Human Segmentation (#243)

* add human segmentation c++ demo

* removed debug print and update README

* inverted colors for consistency

* adjusted blending weight for visualization

CMakeLists.txt

@@ -0,0 +1,31 @@
+cmake_minimum_required(VERSION 3.24)
+set(CMAKE_CXX_STANDARD 11)
+set(project_name "opencv_zoo_human_segmentation")
+
+PROJECT (${project_name})
+
+set(OPENCV_VERSION "4.9.0")
+set(OPENCV_INSTALLATION_PATH "" CACHE PATH "Where to look for OpenCV installation")
+find_package(OpenCV ${OPENCV_VERSION} REQUIRED HINTS ${OPENCV_INSTALLATION_PATH})
+# Find OpenCV, you may need to set OpenCV_DIR variable
+# to the absolute path to the directory containing OpenCVConfig.cmake file
+# via the command line or GUI
+
+file(GLOB SourceFile
+    "demo.cpp")
+# If the package has been found, several variables will
+# be set, you can find the full list with descriptions
+# in the OpenCVConfig.cmake file.
+# Print some message showing some of them
+message(STATUS "OpenCV library status:")
+message(STATUS "    config: ${OpenCV_DIR}")
+message(STATUS "    version: ${OpenCV_VERSION}")
+message(STATUS "    libraries: ${OpenCV_LIBS}")
+message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")
+
+# Declare the executable target built from your sources
+add_executable(${project_name} ${SourceFile})
+
+# Link your application with OpenCV libraries
+target_link_libraries(${project_name} PRIVATE ${OpenCV_LIBS})
+

README.md

@@ -4,6 +4,8 @@ This model is ported from [PaddleHub](https://github.com/PaddlePaddle/PaddleHub)
 
 ## Demo
 
+### Python
+
 Run the following command to try the demo:
 
 ```shell
@@ -16,6 +18,23 @@ python demo.py --input /path/to/image -v
 python demo.py --help
 ```
 
+### C++
+
+Install latest OpenCV and CMake >= 3.24.0 to get started with:
+
+```shell
+# A typical and default installation path of OpenCV is /usr/local
+cmake -B build -D OPENCV_INSTALLATION_PATH=/path/to/opencv/installation .
+cmake --build build
+
+# detect on camera input
+./build/opencv_zoo_human_segmentation
+# detect on an image
+./build/opencv_zoo_human_segmentation -i=/path/to/image
+# get help messages
+./build/opencv_zoo_human_segmentation -h
+```
+
 ### Example outputs
 
 ![webcam demo](./example_outputs/pphumanseg_demo.gif)

demo.cpp

@@ -0,0 +1,226 @@
+#include "opencv2/opencv.hpp"
+
+#include <map>
+#include <vector>
+#include <string>
+#include <iostream>
+
+using namespace std;
+using namespace cv;
+using namespace dnn;
+
+std::vector<std::pair<int, int>> backend_target_pairs = {
+    {DNN_BACKEND_OPENCV, DNN_TARGET_CPU},
+    {DNN_BACKEND_CUDA, DNN_TARGET_CUDA},
+    {DNN_BACKEND_CUDA, DNN_TARGET_CUDA_FP16},
+    {DNN_BACKEND_TIMVX, DNN_TARGET_NPU},
+    {DNN_BACKEND_CANN, DNN_TARGET_NPU}
+};
+
+class PPHS
+{
+private:
+    Net model;
+    string modelPath;
+    
+    Scalar imageMean = Scalar(0.5,0.5,0.5);
+    Scalar imageStd = Scalar(0.5,0.5,0.5);
+    Size modelInputSize = Size(192, 192);
+    Size currentSize;
+
+    const String inputNames = "x";
+    const String outputNames = "save_infer_model/scale_0.tmp_1";
+
+    int backend_id;
+    int target_id;
+    
+public:
+    PPHS(const string& modelPath, 
+        int backend_id = 0, 
+        int target_id = 0) 
+      : modelPath(modelPath), backend_id(backend_id), target_id(target_id)
+    {
+        this->model = readNet(modelPath);
+        this->model.setPreferableBackend(backend_id);
+        this->model.setPreferableTarget(target_id);
+    }
+
+    Mat preprocess(const Mat image)
+    {
+        this->currentSize = image.size();
+        Mat preprocessed = Mat::zeros(this->modelInputSize, image.type());
+        resize(image, preprocessed, this->modelInputSize);
+
+        // image normalization
+        preprocessed.convertTo(preprocessed, CV_32F, 1.0 / 255.0);
+        preprocessed -= imageMean;
+        preprocessed /= imageStd;
+
+        return blobFromImage(preprocessed);;
+    }
+
+    Mat infer(const Mat image)
+    {
+        Mat inputBlob = preprocess(image);
+
+        this->model.setInput(inputBlob, this->inputNames);
+        Mat outputBlob = this->model.forward(this->outputNames);
+
+        return postprocess(outputBlob);
+    }
+
+    Mat postprocess(Mat image) 
+    {
+        reduceArgMax(image,image,1);
+        image = image.reshape(1,image.size[2]);
+        image.convertTo(image, CV_32F);
+        resize(image, image, this->currentSize, 0, 0, INTER_LINEAR);
+        image.convertTo(image, CV_8U);
+
+        return image;
+    }
+
+};
+
+
+vector<uint8_t> getColorMapList(int num_classes) {
+    num_classes += 1;
+
+    vector<uint8_t> cm(num_classes*3, 0);
+
+    int lab, j;
+
+    for (int i = 0; i < num_classes; ++i) {
+        lab = i;
+        j = 0;
+
+        while(lab){
+            cm[i] |= (((lab >> 0) & 1) << (7 - j));
+            cm[i+num_classes] |= (((lab >> 1) & 1) << (7 - j));
+            cm[i+2*num_classes] |= (((lab >> 2) & 1) << (7 - j));
+            ++j;
+            lab >>= 3; 
+        }
+
+    }
+
+    cm.erase(cm.begin(), cm.begin()+3);
+
+    return cm;
+};
+
+Mat visualize(const Mat& image, const Mat& result, float fps = -1.f, float weight = 0.4)
+{
+    const Scalar& text_color = Scalar(0, 255, 0);
+    Mat output_image = image.clone();
+
+    vector<uint8_t> color_map = getColorMapList(256);
+
+    Mat cmm(color_map);
+
+    cmm = cmm.reshape(1,{3,256});
+
+    if (fps >= 0)
+    {
+        putText(output_image, format("FPS: %.2f", fps), Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, text_color, 2);
+    }
+
+    Mat c1, c2, c3;
+
+    LUT(result, cmm.row(0), c1);
+    LUT(result, cmm.row(1), c2);
+    LUT(result, cmm.row(2), c3);
+
+    Mat pseudo_img;
+    merge(std::vector<Mat>{c1,c2,c3}, pseudo_img);
+
+    addWeighted(output_image, weight, pseudo_img, 1 - weight, 0, output_image);
+
+    return output_image;
+};
+
+string keys =
+"{ help  h          |                                                   | Print help message. }"
+"{ model m          | human_segmentation_pphumanseg_2023mar.onnx        | Usage: Path to the model, defaults to human_segmentation_pphumanseg_2023mar.onnx }"
+"{ input i          |                                                   | Path to input image or video file. Skip this argument to capture frames from a camera.}"
+"{ backend_target t | 0                                                 | Choose one of the backend-target pair to run this demo:\n"
+                                                                        "0: (default) OpenCV implementation + CPU,\n"
+                                                                        "1: CUDA + GPU (CUDA),\n"
+                                                                        "2: CUDA + GPU (CUDA FP16),\n"
+                                                                        "3: TIM-VX + NPU,\n"
+                                                                        "4: CANN + NPU}"
+"{ save s           | false                                             | Specify to save results.}"
+"{ vis v            | true                                              | Specify to open a window for result visualization.}"
+;
+
+
+int main(int argc, char** argv)
+{
+    CommandLineParser parser(argc, argv, keys);
+ 
+    parser.about("Human Segmentation");
+    if (parser.has("help"))
+    {
+        parser.printMessage();
+        return 0;
+    }
+    
+    string modelPath = parser.get<string>("model");
+    string inputPath = parser.get<string>("input");
+    uint8_t backendTarget = parser.get<uint8_t>("backend_target");
+    bool saveFlag = parser.get<bool>("save");
+    bool visFlag = parser.get<bool>("vis");
+
+    if (modelPath.empty())
+        CV_Error(Error::StsError, "Model file " + modelPath + " not found");
+
+    PPHS humanSegmentationModel(modelPath, backend_target_pairs[backendTarget].first, backend_target_pairs[backendTarget].second);
+
+    VideoCapture cap;
+    if (!inputPath.empty())
+        cap.open(samples::findFile(inputPath));
+    else
+        cap.open(0);
+    
+    if (!cap.isOpened())
+        CV_Error(Error::StsError, "Cannot opend video or file");    
+
+    Mat frame;
+    Mat result;
+    static const std::string kWinName = "Human Segmentation Demo";
+    TickMeter tm;
+
+    while (waitKey(1) < 0)
+    {
+        cap >> frame;
+
+        if (frame.empty())
+        {
+            if(inputPath.empty())
+                cout << "Frame is empty" << endl;
+            break;
+        }
+
+        tm.start();
+        result = humanSegmentationModel.infer(frame);
+        tm.stop();
+        
+        Mat res_frame = visualize(frame, result, tm.getFPS());
+
+        if(visFlag || inputPath.empty())
+        {
+            imshow(kWinName, res_frame);
+            if(!inputPath.empty())
+                waitKey(0);
+        }
+        if(saveFlag)
+        {
+            cout << "Results are saved to result.jpg" << endl;
+
+            imwrite("result.jpg", res_frame);
+        }
+    }
+    
+    return 0;
+}   
+

demo.py

@@ -83,8 +83,8 @@ def visualize(image, result, weight=0.6, fps=None):
         vis_result (np.ndarray): The visualized result.
     """
     color_map = get_color_map_list(256)
-    color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)]
-    color_map = np.array(color_map).astype(np.uint8)
+    color_map = np.array(color_map).reshape(256, 3).astype(np.uint8)
+
     # Use OpenCV LUT for color mapping
     c1 = cv.LUT(result, color_map[:, 0])
     c2 = cv.LUT(result, color_map[:, 1])
@@ -158,3 +158,4 @@ if __name__ == '__main__':
             cv.imshow('PPHumanSeg Demo', frame)
 
             tm.reset()
+