added evaluation script for PPHumanSeg model (#130)

* added evaluation script for PPHumanSeg

* added quantized model, renamed dataset

* minor spacing changes

* moved _all variables outside loop and updated accuracy

* removed printing for class accuracy and IoU

* added 2 transforms

* evaluation done on same size tensor as input size with mIoU 0.9085

* final changes

* added mIoU and reference

models/human_segmentation_pphumanseg/README.md

@@ -22,6 +22,18 @@ python demo.py --help
 
 ![messi](./examples/messi.jpg)
 
+---
+Results of accuracy evaluation with [tools/eval](../../tools/eval).
+
+| Models             | Accuracy       | mIoU          |
+| ------------------ | -------------- | ------------- |
+| PPHumanSeg         | 0.9581         | 0.8996        |
+| PPHumanSeg quant   | 0.4365         | 0.2788        |
+
+
+\*: 'quant' stands for 'quantized'.
+
+---
 ## License
 
 All files in this directory are licensed under [Apache 2.0 License](./LICENSE).

models/human_segmentation_pphumanseg/pphumanseg.py

@@ -19,6 +19,7 @@ def __init__(self, modelPath, backendId=0, targetId=0):
 
         self._inputNames = ''
         self._outputNames = ['save_infer_model/scale_0.tmp_1']
+        self._currentInputSize = None
         self._inputSize = [192, 192]
         self._mean = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
         self._std = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
@@ -36,28 +37,36 @@ def setTarget(self, target_id):
         self._model.setPreferableTarget(self._targetId)
 
     def _preprocess(self, image):
+
+        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
+
+        self._currentInputSize = image.shape
+        image = cv.resize(image, (192, 192))
+
         image = image.astype(np.float32, copy=False) / 255.0
         image -= self._mean
         image /= self._std
         return cv.dnn.blobFromImage(image)
 
     def infer(self, image):
-        assert image.shape[0] == self._inputSize[1], '{} (height of input image) != {} (preset height)'.format(image.shape[0], self._inputSize[1])
-        assert image.shape[1] == self._inputSize[0], '{} (width of input image) != {} (preset width)'.format(image.shape[1], self._inputSize[0])
 
         # Preprocess
         inputBlob = self._preprocess(image)
 
         # Forward
         self._model.setInput(inputBlob, self._inputNames)
-        outputBlob = self._model.forward(self._outputNames)
+        outputBlob = self._model.forward()
 
         # Postprocess
         results = self._postprocess(outputBlob)
 
         return results
 
     def _postprocess(self, outputBlob):
-        result = np.argmax(outputBlob[0], axis=1).astype(np.uint8)
+
+        outputBlob = outputBlob[0]
+        outputBlob = cv.resize(outputBlob.transpose(1,2,0), (self._currentInputSize[1], self._currentInputSize[0]), interpolation=cv.INTER_LINEAR).transpose(2,0,1)[np.newaxis, ...]
+
+        result = np.argmax(outputBlob, axis=1).astype(np.uint8)
         return result
 

tools/eval/README.md

@@ -21,6 +21,7 @@ Supported datasets:
 - [LFW](#lfw)
 - [ICDAR](#ICDAR2003)
 - [IIIT5K](#iiit5k)
+- [Mini Supervisely](#mini_supervisely)
 
 ## ImageNet
 
@@ -190,4 +191,24 @@ Run evaluation with the following command:
 
 ```shell
 python eval.py -m crnn -d iiit5k -dr /path/to/iiit5k
+```
+
+
+## Mini Supervisely
+
+### Prepare data
+Please download the mini_supervisely data from [here](https://paddleseg.bj.bcebos.com/humanseg/data/mini_supervisely.zip) which includes the validation dataset and unzip it.
+
+### Evaluation
+
+Run evaluation with the following command :
+
+```shell
+python eval.py -m pphumanseg -d mini_supervisely -dr /path/to/pphumanseg
+```
+
+Run evaluation on quantized model with the following command :
+
+```shell
+python eval.py -m pphumanseg_q -d mini_supervisely -dr /path/to/pphumanseg
 ```

tools/eval/datasets/__init__.py

@@ -3,6 +3,7 @@
 from .lfw import LFW
 from .icdar import ICDAR
 from .iiit5k import IIIT5K
+from .minisupervisely import MiniSupervisely
 
 class Registery:
     def __init__(self, name):
@@ -20,4 +21,5 @@ def register(self, item):
 DATASETS.register(WIDERFace)
 DATASETS.register(LFW)
 DATASETS.register(ICDAR)
-DATASETS.register(IIIT5K)
+DATASETS.register(IIIT5K)
+DATASETS.register(MiniSupervisely)

tools/eval/datasets/minisupervisely.py

@@ -0,0 +1,202 @@
+import os
+import cv2 as cv
+import numpy as np
+from tqdm import tqdm
+
+
+class MiniSupervisely : 
+
+    '''
+        Refer to https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.7/paddleseg/core/val.py
+        for official evaluation implementation.
+    '''
+
+    def __init__(self, root) : 
+        self.root = root
+        self.val_path = os.path.join(root, 'val.txt')
+        self.image_set = self.load_data(self.val_path)
+        self.num_classes = 2
+        self.miou = -1
+        self.class_miou = -1
+        self.acc = -1
+        self.class_acc = -1
+
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+
+    def load_data(self, val_path) : 
+        """
+        Load validation image set from val.txt file
+        Args :
+            val_path (str) : path to val.txt file
+        Returns :
+            image_set (list) : list of image path of input and expected image
+        """
+
+        image_set = []
+        with open(val_path, 'r') as f : 
+            for line in f.readlines() : 
+                image_set.append(line.strip().split())
+
+        return image_set
+
+
+    def eval(self, model) : 
+        """
+        Evaluate model on validation set
+        Args :
+            model (object) : PP_HumanSeg model object
+        """
+
+        intersect_area_all = np.zeros([1], dtype=np.int64)
+        pred_area_all = np.zeros([1], dtype=np.int64)
+        label_area_all = np.zeros([1], dtype=np.int64)
+
+        pbar = tqdm(self.image_set)
+
+        pbar.set_description(
+            "Evaluating {} with {} val set".format(model.name, self.name))
+
+        for input_image, expected_image in pbar : 
+
+            input_image = cv.imread(os.path.join(self.root, input_image)).astype('float32')
+
+            expected_image = cv.imread(os.path.join(self.root, expected_image), cv.IMREAD_GRAYSCALE)[np.newaxis, :, :]           
+
+            output_image = model.infer(input_image) 
+
+            intersect_area, pred_area, label_area = self.calculate_area(
+                output_image.astype('uint32'),
+                expected_image.astype('uint32'),
+                self.num_classes)
+
+            intersect_area_all = intersect_area_all + intersect_area
+            pred_area_all = pred_area_all + pred_area
+            label_area_all = label_area_all + label_area
+
+        self.class_iou, self.miou = self.mean_iou(intersect_area_all, pred_area_all,
+                                           label_area_all)
+        self.class_acc, self.acc = self.accuracy(intersect_area_all, pred_area_all)
+
+
+    def get_results(self) :
+        """
+        Get evaluation results
+        Returns :
+            miou (float) : mean iou
+            class_miou (list) : iou on all classes
+            acc (float) : mean accuracy
+            class_acc (list) : accuracy on all classes
+        """
+        return self.miou, self.class_miou, self.acc, self.class_acc
+
+
+    def print_result(self) : 
+        """
+        Print evaluation results
+        """
+        print("Mean IoU : ", self.miou)
+        print("Mean Accuracy : ", self.acc)
+        print("Class IoU : ", self.class_iou)
+        print("Class Accuracy : ", self.class_acc)
+
+
+    def calculate_area(self,pred, label, num_classes, ignore_index=255):
+        """
+        Calculate intersect, prediction and label area
+        Args:
+            pred (Tensor): The prediction by model.
+            label (Tensor): The ground truth of image.
+            num_classes (int): The unique number of target classes.
+            ignore_index (int): Specifies a target value that is ignored. Default: 255.
+        Returns:
+            Tensor: The intersection area of prediction and the ground on all class.
+            Tensor: The prediction area on all class.
+            Tensor: The ground truth area on all class
+        """
+
+
+        if len(pred.shape) == 4:
+            pred = np.squeeze(pred, axis=1)
+        if len(label.shape) == 4:
+            label = np.squeeze(label, axis=1)
+        if not pred.shape == label.shape:
+            raise ValueError('Shape of `pred` and `label should be equal, '
+                            'but there are {} and {}.'.format(pred.shape,
+                                                            label.shape))
+
+        mask = label != ignore_index
+        pred_area = []
+        label_area = []
+        intersect_area = []
+
+        #iterate over all classes and calculate their respective areas
+        for i in range(num_classes):
+            pred_i = np.logical_and(pred == i, mask)
+            label_i = label == i
+            intersect_i = np.logical_and(pred_i, label_i)
+            pred_area.append(np.sum(pred_i.astype('int32')))
+            label_area.append(np.sum(label_i.astype('int32')))
+            intersect_area.append(np.sum(intersect_i.astype('int32')))
+
+        return intersect_area, pred_area, label_area
+
+
+    def mean_iou(self,intersect_area, pred_area, label_area):
+        """
+        Calculate iou.
+        Args:
+            intersect_area (Tensor): The intersection area of prediction and ground truth on all classes.
+            pred_area (Tensor): The prediction area on all classes.
+            label_area (Tensor): The ground truth area on all classes.
+        Returns:
+            np.ndarray: iou on all classes.
+            float: mean iou of all classes.
+        """
+        intersect_area = np.array(intersect_area)
+        pred_area = np.array(pred_area)
+        label_area = np.array(label_area)
+
+        union = pred_area + label_area - intersect_area
+
+        class_iou = []
+        for i in range(len(intersect_area)):
+            if union[i] == 0:
+                iou = 0
+            else:
+                iou = intersect_area[i] / union[i]
+            class_iou.append(iou)
+
+        miou = np.mean(class_iou)
+
+        return np.array(class_iou), miou
+
+
+    def accuracy(self,intersect_area, pred_area):
+        """
+        Calculate accuracy
+        Args:
+            intersect_area (Tensor): The intersection area of prediction and ground truth on all classes..
+            pred_area (Tensor): The prediction area on all classes.
+        Returns:
+            np.ndarray: accuracy on all classes.
+            float: mean accuracy.
+        """
+
+        intersect_area = np.array(intersect_area)
+        pred_area = np.array(pred_area)
+
+        class_acc = []
+        for i in range(len(intersect_area)):
+            if pred_area[i] == 0:
+                acc = 0
+            else:
+                acc = intersect_area[i] / pred_area[i]
+            class_acc.append(acc)
+
+        macc = np.sum(intersect_area) / np.sum(pred_area)
+
+        return np.array(class_acc), macc

tools/eval/eval.py

@@ -77,6 +77,14 @@
         name="CRNN",
         topic="text_recognition",
         modelPath=os.path.join(root_dir, "models/text_recognition_crnn/text_recognition_CRNN_EN_2021sep.onnx")),
+    pphumanseg=dict(
+        name="PPHumanSeg",
+        topic="human_segmentation",
+        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct.onnx")),
+    pphumanseg_q=dict(
+        name="PPHumanSeg",
+        topic="human_segmentation",
+        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct-act_int8-wt_int8-quantized.onnx")),
 )
 
 datasets = dict(
@@ -97,6 +105,9 @@
         iiit5k=dict(
             name="IIIT5K",
             topic="text_recognition"),
+        mini_supervisely=dict(
+            name="MiniSupervisely",
+            topic="human_segmentation"),
 )
 
 def main(args):