From 0d0875cec93ecf23627da7fff886e83df88a2bc2 Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Fri, 10 Feb 2023 09:52:10 +0530
Subject: [PATCH 1/9] added evaluation script for PPHumanSeg

---
 tools/eval/README.md               |  17 ++-
 tools/eval/datasets/__init__.py    |   4 +-
 tools/eval/datasets/pp_humanseg.py | 209 +++++++++++++++++++++++++++++
 tools/eval/eval.py                 |   8 +-
 4 files changed, 235 insertions(+), 3 deletions(-)
 create mode 100644 tools/eval/datasets/pp_humanseg.py

diff --git a/tools/eval/README.md b/tools/eval/README.md
index 650e4d50..eea69fd2 100644
--- a/tools/eval/README.md
+++ b/tools/eval/README.md
@@ -21,6 +21,7 @@ Supported datasets:
 - [LFW](#lfw)
 - [ICDAR](#ICDAR2003)
 - [IIIT5K](#iiit5k)
+- [PPHumanSeg](#PPHumanSeg)
 
 ## ImageNet
 
@@ -181,7 +182,7 @@ python eval.py -m crnn -d icdar -dr /path/to/icdar
 
 ### Prepare data
 
-Please visit https://github.com/cv-small-snails/Text-Recognition-Material to download the IIIT5K dataset and the labels.
+Please visit https://github.com/cv-small-snails/Text-Recognition-Material to download the IIIT5K dataset and the labels. 
 
 ### Evaluation
 
@@ -190,4 +191,18 @@ Run evaluation with the following command:
 
 ```shell
 python eval.py -m crnn -d iiit5k -dr /path/to/iiit5k
+```
+
+
+## PPHumanSeg
+
+### Prepare data
+Please download the mini_supervisely data from [here](https://paddleseg.bj.bcebos.com/humanseg/data/mini_supervisely.zip) which includes the validation dataset and unzip it.
+
+### Evaluation
+
+Run evaluation with the following command
+
+```shell
+python eval.py -m pphumanseg -d pphumanseg -dr /path/to/pphumanseg
 ```
\ No newline at end of file
diff --git a/tools/eval/datasets/__init__.py b/tools/eval/datasets/__init__.py
index 84388375..5331797d 100644
--- a/tools/eval/datasets/__init__.py
+++ b/tools/eval/datasets/__init__.py
@@ -3,6 +3,7 @@
 from .lfw import LFW
 from .icdar import ICDAR
 from .iiit5k import IIIT5K
+from .pp_humanseg import PPHumanSeg
 
 class Registery:
     def __init__(self, name):
@@ -20,4 +21,5 @@ def register(self, item):
 DATASETS.register(WIDERFace)
 DATASETS.register(LFW)
 DATASETS.register(ICDAR)
-DATASETS.register(IIIT5K)
\ No newline at end of file
+DATASETS.register(IIIT5K)
+DATASETS.register(PPHumanSeg)
\ No newline at end of file
diff --git a/tools/eval/datasets/pp_humanseg.py b/tools/eval/datasets/pp_humanseg.py
new file mode 100644
index 00000000..528356e9
--- /dev/null
+++ b/tools/eval/datasets/pp_humanseg.py
@@ -0,0 +1,209 @@
+import os
+import cv2 as cv
+import numpy as np
+from tqdm import tqdm
+
+
+class PPHumanSeg : 
+    def __init__(self, root) : 
+        self.root = root
+        self.val_path = os.path.join(root, 'val.txt')
+        self.image_set = self.load_data(self.val_path)
+        self.num_classes = 2
+        self.miou = -1
+        self.class_miou = -1
+        self.acc = -1
+        self.class_acc = -1
+
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+    
+
+    def load_data(self, val_path) : 
+        """
+        Load validation image set from val.txt file
+        Args :
+            val_path (str) : path to val.txt file
+        Returns :
+            image_set (list) : list of image path of input and expected image
+        """
+
+        image_set = []
+        with open(val_path, 'r') as f : 
+            for line in f.readlines() : 
+                image_set.append(line.strip().split())
+
+        return image_set
+    
+    
+    def eval(self, model) : 
+        """
+        Evaluate model on validation set
+        Args :
+            model (object) : PP_HumanSeg model object
+        """
+
+        pbar = tqdm(self.image_set)
+
+        for input_image, expected_image in pbar : 
+            pbar.set_description(
+                "Evaluating {} with {} val set".format(model.name, self.name))
+            
+            input_image = cv.imread(os.path.join(self.root, input_image))
+            input_image = cv.resize(input_image, (192, 192))
+
+            expected_image = cv.imread(os.path.join(self.root, expected_image),cv.IMREAD_GRAYSCALE)
+            expected_image = cv.resize(expected_image, (192, 192))[np.newaxis, :, :]
+
+
+            output_image = model.infer(input_image)
+            
+            intersect_area_all = []
+            pred_area_all = []
+            label_area_all = []
+
+
+            intersect_area, pred_area, label_area = self.calculate_area(
+                output_image,
+                expected_image,
+                self.num_classes)
+            intersect_area_all = intersect_area_all + intersect_area
+            pred_area_all = pred_area_all + pred_area
+            label_area_all = label_area_all + label_area
+
+
+            
+        self.class_iou, self.miou = self.mean_iou(intersect_area_all, pred_area_all,
+                                           label_area_all)
+        self.class_acc, self.acc = self.accuracy(intersect_area_all, pred_area_all)
+    
+
+    def get_results(self) :
+        """
+        Get evaluation results
+        Returns :
+            miou (float) : mean iou
+            class_miou (list) : iou on all classes
+            acc (float) : mean accuracy
+            class_acc (list) : accuracy on all classes
+        """
+        return self.miou, self.class_miou, self.acc, self.class_acc
+    
+
+    def print_result(self) : 
+        """
+        Print evaluation results
+        """
+        print("Mean IoU : ", self.miou)
+        print("Class IoU : ", self.class_iou)
+        print("Mean Accuracy : ", self.acc)
+        print("Class Accuracy : ", self.class_acc)
+
+    
+    def one_hot(self, arr, max_size) :
+        return np.eye(max_size)[arr]
+
+
+    def calculate_area(self,pred, label, num_classes, ignore_index=255):
+        """
+        Calculate intersect, prediction and label area
+        Args:
+            pred (Tensor): The prediction by model.
+            label (Tensor): The ground truth of image.
+            num_classes (int): The unique number of target classes.
+            ignore_index (int): Specifies a target value that is ignored. Default: 255.
+        Returns:
+            Tensor: The intersection area of prediction and the ground on all class.
+            Tensor: The prediction area on all class.
+            Tensor: The ground truth area on all class
+        """
+        
+        # Delete ignore_index
+        mask = label != ignore_index
+        pred = pred + 1
+        label = label + 1
+        pred = pred * mask
+        label = label * mask
+
+
+        pred = self.one_hot(pred, num_classes + 1)
+        label = self.one_hot(label, num_classes + 1)
+
+        pred = pred[:, :, :, 1:]
+        label = label[:, :, :, 1:]
+
+        pred_area = []
+        label_area = []
+        intersect_area = []
+
+        #iterate over all classes and calculate their respective areas
+        for i in range(num_classes):
+            pred_i = pred[:, :, :, i]
+            label_i = label[:, :, :, i]
+            pred_area_i = np.sum(pred_i)
+            label_area_i = np.sum(label_i)
+            intersect_area_i = np.sum(pred_i * label_i)
+            pred_area.append(pred_area_i)
+            label_area.append(label_area_i)
+            intersect_area.append(intersect_area_i)
+        
+        return intersect_area, pred_area, label_area
+    
+
+    def mean_iou(self,intersect_area, pred_area, label_area):
+        """
+        Calculate iou.
+        Args:
+            intersect_area (Tensor): The intersection area of prediction and ground truth on all classes.
+            pred_area (Tensor): The prediction area on all classes.
+            label_area (Tensor): The ground truth area on all classes.
+        Returns:
+            np.ndarray: iou on all classes.
+            float: mean iou of all classes.
+        """
+        intersect_area = np.array(intersect_area)
+        pred_area = np.array(pred_area)
+        label_area = np.array(label_area)
+
+        union = pred_area + label_area - intersect_area
+
+        class_iou = []
+        for i in range(len(intersect_area)):
+            if union[i] == 0:
+                iou = 0
+            else:
+                iou = intersect_area[i] / union[i]
+            class_iou.append(iou)
+
+        miou = np.mean(class_iou)
+
+        return np.array(class_iou), miou
+    
+
+    def accuracy(self,intersect_area, pred_area):
+        """
+        Calculate accuracy
+        Args:
+            intersect_area (Tensor): The intersection area of prediction and ground truth on all classes..
+            pred_area (Tensor): The prediction area on all classes.
+        Returns:
+            np.ndarray: accuracy on all classes.
+            float: mean accuracy.
+        """
+
+        intersect_area = np.array(intersect_area)
+        pred_area = np.array(pred_area)
+
+        class_acc = []
+        for i in range(len(intersect_area)):
+            if pred_area[i] == 0:
+                acc = 0
+            else:
+                acc = intersect_area[i] / pred_area[i]
+            class_acc.append(acc)
+
+        macc = np.sum(intersect_area) / np.sum(pred_area)
+
+        return np.array(class_acc), macc
diff --git a/tools/eval/eval.py b/tools/eval/eval.py
index 9c25f5ed..d24b1b71 100644
--- a/tools/eval/eval.py
+++ b/tools/eval/eval.py
@@ -77,6 +77,10 @@
         name="CRNN",
         topic="text_recognition",
         modelPath=os.path.join(root_dir, "models/text_recognition_crnn/text_recognition_CRNN_EN_2021sep.onnx")),
+    pphumanseg=dict(
+        name="PPHumanSeg",
+        topic="human_segmentation",
+        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct.onnx")),
 )
 
 datasets = dict(
@@ -97,6 +101,9 @@
         iiit5k=dict(
             name="IIIT5K",
             topic="text_recognition"),
+        pphumanseg=dict(
+            name="PPHumanSeg",
+            topic="human_segmentation"),
 )
 
 def main(args):
@@ -108,7 +115,6 @@ def main(args):
     model_topic = models[model_key].pop("topic")
     model_handler, _ = MODELS.get(model_name)
     model = model_handler(**models[model_key])
-
     # Instantiate dataset
     dataset_key = args.dataset.lower()
     assert dataset_key in datasets

From db4aaca68d51b16ed3e9652f289b4482f263a9eb Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Fri, 10 Feb 2023 16:29:09 +0530
Subject: [PATCH 2/9] added quantized model, renamed dataset

---
 models/human_segmentation_pphumanseg/README.md     | 12 ++++++++++++
 tools/eval/README.md                               | 14 ++++++++++----
 tools/eval/datasets/__init__.py                    |  4 ++--
 .../{pp_humanseg.py => minisupervisely.py}         |  2 +-
 tools/eval/eval.py                                 |  9 +++++++--
 5 files changed, 32 insertions(+), 9 deletions(-)
 rename tools/eval/datasets/{pp_humanseg.py => minisupervisely.py} (99%)

diff --git a/models/human_segmentation_pphumanseg/README.md b/models/human_segmentation_pphumanseg/README.md
index c59f253a..3d89d16b 100644
--- a/models/human_segmentation_pphumanseg/README.md
+++ b/models/human_segmentation_pphumanseg/README.md
@@ -22,6 +22,18 @@ python demo.py --help
 
 ![messi](./examples/messi.jpg)
 
+---
+Results of accuracy evaluation with [tools/eval](../../tools/eval).
+
+| Models             | Accuracy | 
+| ------------------ | -------------- |
+| PPHumanSeg         | 0.9023         |
+| PPHumanSeg quant   | 0.4940         |
+
+
+\*: 'quant' stands for 'quantized'.
+
+---
 ## License
 
 All files in this directory are licensed under [Apache 2.0 License](./LICENSE).
diff --git a/tools/eval/README.md b/tools/eval/README.md
index eea69fd2..c8332f7e 100644
--- a/tools/eval/README.md
+++ b/tools/eval/README.md
@@ -21,7 +21,7 @@ Supported datasets:
 - [LFW](#lfw)
 - [ICDAR](#ICDAR2003)
 - [IIIT5K](#iiit5k)
-- [PPHumanSeg](#PPHumanSeg)
+- [Mini Supervisely](#mini_supervisely)
 
 ## ImageNet
 
@@ -182,7 +182,7 @@ python eval.py -m crnn -d icdar -dr /path/to/icdar
 
 ### Prepare data
 
-Please visit https://github.com/cv-small-snails/Text-Recognition-Material to download the IIIT5K dataset and the labels. 
+Please visit https://github.com/cv-small-snails/Text-Recognition-Material to download the IIIT5K dataset and the labels.
 
 ### Evaluation
 
@@ -194,7 +194,7 @@ python eval.py -m crnn -d iiit5k -dr /path/to/iiit5k
 ```
 
 
-## PPHumanSeg
+## mini_supervisely
 
 ### Prepare data
 Please download the mini_supervisely data from [here](https://paddleseg.bj.bcebos.com/humanseg/data/mini_supervisely.zip) which includes the validation dataset and unzip it.
@@ -204,5 +204,11 @@ Please download the mini_supervisely data from [here](https://paddleseg.bj.bcebo
 Run evaluation with the following command
 
 ```shell
-python eval.py -m pphumanseg -d pphumanseg -dr /path/to/pphumanseg
+python eval.py -m pphumanseg -d mini_supervisely -dr /path/to/pphumanseg
+```
+
+Run evaluation on quantized model with the following command
+
+```shell
+python eval.py -m pphumanseg_q -d mini_supervisely -dr /path/to/pphumanseg
 ```
\ No newline at end of file
diff --git a/tools/eval/datasets/__init__.py b/tools/eval/datasets/__init__.py
index 5331797d..24687d23 100644
--- a/tools/eval/datasets/__init__.py
+++ b/tools/eval/datasets/__init__.py
@@ -3,7 +3,7 @@
 from .lfw import LFW
 from .icdar import ICDAR
 from .iiit5k import IIIT5K
-from .pp_humanseg import PPHumanSeg
+from .minisupervisely import MiniSupervisely
 
 class Registery:
     def __init__(self, name):
@@ -22,4 +22,4 @@ def register(self, item):
 DATASETS.register(LFW)
 DATASETS.register(ICDAR)
 DATASETS.register(IIIT5K)
-DATASETS.register(PPHumanSeg)
\ No newline at end of file
+DATASETS.register(MiniSupervisely)
\ No newline at end of file
diff --git a/tools/eval/datasets/pp_humanseg.py b/tools/eval/datasets/minisupervisely.py
similarity index 99%
rename from tools/eval/datasets/pp_humanseg.py
rename to tools/eval/datasets/minisupervisely.py
index 528356e9..9adf2b77 100644
--- a/tools/eval/datasets/pp_humanseg.py
+++ b/tools/eval/datasets/minisupervisely.py
@@ -4,7 +4,7 @@
 from tqdm import tqdm
 
 
-class PPHumanSeg : 
+class MiniSupervisely : 
     def __init__(self, root) : 
         self.root = root
         self.val_path = os.path.join(root, 'val.txt')
diff --git a/tools/eval/eval.py b/tools/eval/eval.py
index d24b1b71..a3038778 100644
--- a/tools/eval/eval.py
+++ b/tools/eval/eval.py
@@ -81,6 +81,10 @@
         name="PPHumanSeg",
         topic="human_segmentation",
         modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct.onnx")),
+    pphumanseg_q=dict(
+        name="PPHumanSeg",
+        topic="human_segmentation",
+        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct-act_int8-wt_int8-quantized.onnx")),
 )
 
 datasets = dict(
@@ -101,8 +105,8 @@
         iiit5k=dict(
             name="IIIT5K",
             topic="text_recognition"),
-        pphumanseg=dict(
-            name="PPHumanSeg",
+        mini_supervisely=dict(
+            name="MiniSupervisely",
             topic="human_segmentation"),
 )
 
@@ -115,6 +119,7 @@ def main(args):
     model_topic = models[model_key].pop("topic")
     model_handler, _ = MODELS.get(model_name)
     model = model_handler(**models[model_key])
+    
     # Instantiate dataset
     dataset_key = args.dataset.lower()
     assert dataset_key in datasets

From 4193e38741c66fbfde1662d902c7e70fb8a35aa0 Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Fri, 10 Feb 2023 20:05:42 +0530
Subject: [PATCH 3/9] minor spacing changes

---
 models/human_segmentation_pphumanseg/README.md | 2 +-
 tools/eval/eval.py                             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/models/human_segmentation_pphumanseg/README.md b/models/human_segmentation_pphumanseg/README.md
index 3d89d16b..760a11cf 100644
--- a/models/human_segmentation_pphumanseg/README.md
+++ b/models/human_segmentation_pphumanseg/README.md
@@ -25,7 +25,7 @@ python demo.py --help
 ---
 Results of accuracy evaluation with [tools/eval](../../tools/eval).
 
-| Models             | Accuracy | 
+| Models             | Accuracy       | 
 | ------------------ | -------------- |
 | PPHumanSeg         | 0.9023         |
 | PPHumanSeg quant   | 0.4940         |
diff --git a/tools/eval/eval.py b/tools/eval/eval.py
index a3038778..9125045d 100644
--- a/tools/eval/eval.py
+++ b/tools/eval/eval.py
@@ -119,7 +119,7 @@ def main(args):
     model_topic = models[model_key].pop("topic")
     model_handler, _ = MODELS.get(model_name)
     model = model_handler(**models[model_key])
-    
+
     # Instantiate dataset
     dataset_key = args.dataset.lower()
     assert dataset_key in datasets

From e4b799b73e4c534adc5741c0e6abe1f59be926d2 Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Mon, 13 Feb 2023 17:58:42 +0530
Subject: [PATCH 4/9] moved _all variables outside loop and updated accuracy

---
 models/human_segmentation_pphumanseg/README.md |  4 ++--
 tools/eval/datasets/minisupervisely.py         | 11 +++++------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/models/human_segmentation_pphumanseg/README.md b/models/human_segmentation_pphumanseg/README.md
index 760a11cf..312eb171 100644
--- a/models/human_segmentation_pphumanseg/README.md
+++ b/models/human_segmentation_pphumanseg/README.md
@@ -27,8 +27,8 @@ Results of accuracy evaluation with [tools/eval](../../tools/eval).
 
 | Models             | Accuracy       | 
 | ------------------ | -------------- |
-| PPHumanSeg         | 0.9023         |
-| PPHumanSeg quant   | 0.4940         |
+| PPHumanSeg         | 0.9570         |
+| PPHumanSeg quant   | 0.4557         |
 
 
 \*: 'quant' stands for 'quantized'.
diff --git a/tools/eval/datasets/minisupervisely.py b/tools/eval/datasets/minisupervisely.py
index 9adf2b77..4c4eccc6 100644
--- a/tools/eval/datasets/minisupervisely.py
+++ b/tools/eval/datasets/minisupervisely.py
@@ -45,6 +45,10 @@ def eval(self, model) :
             model (object) : PP_HumanSeg model object
         """
 
+        intersect_area_all = []
+        pred_area_all = []
+        label_area_all = []
+
         pbar = tqdm(self.image_set)
 
         for input_image, expected_image in pbar : 
@@ -58,12 +62,7 @@ def eval(self, model) :
             expected_image = cv.resize(expected_image, (192, 192))[np.newaxis, :, :]
 
 
-            output_image = model.infer(input_image)
-            
-            intersect_area_all = []
-            pred_area_all = []
-            label_area_all = []
-
+            output_image = model.infer(input_image)   
 
             intersect_area, pred_area, label_area = self.calculate_area(
                 output_image,

From 6073f5d9504c9f946f159b20761c69aa117e50ca Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Mon, 13 Feb 2023 18:02:01 +0530
Subject: [PATCH 5/9] removed printing for class accuracy and IoU

---
 tools/eval/datasets/minisupervisely.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tools/eval/datasets/minisupervisely.py b/tools/eval/datasets/minisupervisely.py
index 4c4eccc6..a307e81b 100644
--- a/tools/eval/datasets/minisupervisely.py
+++ b/tools/eval/datasets/minisupervisely.py
@@ -96,9 +96,7 @@ def print_result(self) :
         Print evaluation results
         """
         print("Mean IoU : ", self.miou)
-        print("Class IoU : ", self.class_iou)
         print("Mean Accuracy : ", self.acc)
-        print("Class Accuracy : ", self.class_acc)
 
     
     def one_hot(self, arr, max_size) :

From a59e39c11155a16f0c56ba0c4ae2a10fa72fca25 Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Wed, 15 Feb 2023 02:57:09 +0530
Subject: [PATCH 6/9] added 2 transforms

---
 tools/eval/datasets/minisupervisely.py | 58 ++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 3 deletions(-)

diff --git a/tools/eval/datasets/minisupervisely.py b/tools/eval/datasets/minisupervisely.py
index a307e81b..b7907452 100644
--- a/tools/eval/datasets/minisupervisely.py
+++ b/tools/eval/datasets/minisupervisely.py
@@ -4,6 +4,53 @@
 from tqdm import tqdm
 
 
+class Normalize:
+    """
+    Normalize an image.
+    Args:
+        mean (list, optional): The mean value of a data set. Default: [0.5, 0.5, 0.5].
+        std (list, optional): The standard deviation of a data set. Default: [0.5, 0.5, 0.5].
+    Raises:
+        ValueError: When mean/std is not list or any value in std is 0.
+    """
+
+    def __init__(self, mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)):
+        self.mean = mean
+        self.std = std
+        if not (isinstance(self.mean, (list, tuple))
+                and isinstance(self.std, (list, tuple))):
+            raise ValueError(
+                "{}: input type is invalid. It should be list or tuple".format(
+                    self))
+        from functools import reduce
+        if reduce(lambda x, y: x * y, self.std) == 0:
+            raise ValueError('{}: std is invalid!'.format(self))
+
+    def normalize(self, im, mean, std):
+        im = im.astype(np.float32, copy=False) / 255.0
+        im -= mean
+        im /= std
+        return im
+    
+    def __call__(self, im, label=None):
+        """
+        Args:
+            im (np.ndarray): The Image data.
+            label (np.ndarray, optional): The label data. Default: None.
+        Returns:
+            (tuple). When label is None, it returns (im, ), otherwise it returns (im, label).
+        """
+
+        mean = np.array(self.mean)[np.newaxis, np.newaxis, :]
+        std = np.array(self.std)[np.newaxis, np.newaxis, :]
+        im = self.normalize(im, mean, std)
+
+        if label is None:
+            return (im, )
+        else:
+            return (im, label)
+
+
 class MiniSupervisely : 
     def __init__(self, root) : 
         self.root = root
@@ -51,16 +98,21 @@ def eval(self, model) :
 
         pbar = tqdm(self.image_set)
 
+        pbar.set_description(
+            "Evaluating {} with {} val set".format(model.name, self.name))
+        
+        normalize = Normalize()
+
         for input_image, expected_image in pbar : 
-            pbar.set_description(
-                "Evaluating {} with {} val set".format(model.name, self.name))
             
             input_image = cv.imread(os.path.join(self.root, input_image))
             input_image = cv.resize(input_image, (192, 192))
+            input_image = cv.cvtColor(input_image, cv.COLOR_BGR2RGB)
 
             expected_image = cv.imread(os.path.join(self.root, expected_image),cv.IMREAD_GRAYSCALE)
             expected_image = cv.resize(expected_image, (192, 192))[np.newaxis, :, :]
-
+            
+            input_image, expected_image = normalize(input_image, expected_image)
 
             output_image = model.infer(input_image)   
 

From 086a7177fc9731ea86867ed339eede2758fd9b4f Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Thu, 16 Feb 2023 23:35:08 +0530
Subject: [PATCH 7/9] evaluation done on same size tensor as input size with
 mIoU 0.9085

---
 .../pphumanseg.py                             |  11 +-
 tools/eval/datasets/minisupervisely.py        | 103 ++++--------------
 2 files changed, 31 insertions(+), 83 deletions(-)

diff --git a/models/human_segmentation_pphumanseg/pphumanseg.py b/models/human_segmentation_pphumanseg/pphumanseg.py
index 2f20c7d3..8aebf8bd 100644
--- a/models/human_segmentation_pphumanseg/pphumanseg.py
+++ b/models/human_segmentation_pphumanseg/pphumanseg.py
@@ -19,6 +19,7 @@ def __init__(self, modelPath, backendId=0, targetId=0):
 
         self._inputNames = ''
         self._outputNames = ['save_infer_model/scale_0.tmp_1']
+        self._currentInputSize = None
         self._inputSize = [192, 192]
         self._mean = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
         self._std = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
@@ -36,14 +37,15 @@ def setTarget(self, target_id):
         self._model.setPreferableTarget(self._targetId)
 
     def _preprocess(self, image):
+        self._currentInputSize = image.shape
+        image = cv.resize(image, (192, 192))
+        
         image = image.astype(np.float32, copy=False) / 255.0
         image -= self._mean
         image /= self._std
         return cv.dnn.blobFromImage(image)
 
     def infer(self, image):
-        assert image.shape[0] == self._inputSize[1], '{} (height of input image) != {} (preset height)'.format(image.shape[0], self._inputSize[1])
-        assert image.shape[1] == self._inputSize[0], '{} (width of input image) != {} (preset width)'.format(image.shape[1], self._inputSize[0])
 
         # Preprocess
         inputBlob = self._preprocess(image)
@@ -58,6 +60,9 @@ def infer(self, image):
         return results
 
     def _postprocess(self, outputBlob):
-        result = np.argmax(outputBlob[0], axis=1).astype(np.uint8)
+        outputBlob = outputBlob[0][0]
+        outputBlob = cv.resize(outputBlob.transpose(1,2,0), (self._currentInputSize[1], self._currentInputSize[0]), interpolation=cv.INTER_LINEAR).transpose(2,0,1)[np.newaxis, ...]
+
+        result = np.argmax(outputBlob, axis=1).astype(np.uint8)
         return result
 
diff --git a/tools/eval/datasets/minisupervisely.py b/tools/eval/datasets/minisupervisely.py
index b7907452..58b4397e 100644
--- a/tools/eval/datasets/minisupervisely.py
+++ b/tools/eval/datasets/minisupervisely.py
@@ -4,53 +4,6 @@
 from tqdm import tqdm
 
 
-class Normalize:
-    """
-    Normalize an image.
-    Args:
-        mean (list, optional): The mean value of a data set. Default: [0.5, 0.5, 0.5].
-        std (list, optional): The standard deviation of a data set. Default: [0.5, 0.5, 0.5].
-    Raises:
-        ValueError: When mean/std is not list or any value in std is 0.
-    """
-
-    def __init__(self, mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)):
-        self.mean = mean
-        self.std = std
-        if not (isinstance(self.mean, (list, tuple))
-                and isinstance(self.std, (list, tuple))):
-            raise ValueError(
-                "{}: input type is invalid. It should be list or tuple".format(
-                    self))
-        from functools import reduce
-        if reduce(lambda x, y: x * y, self.std) == 0:
-            raise ValueError('{}: std is invalid!'.format(self))
-
-    def normalize(self, im, mean, std):
-        im = im.astype(np.float32, copy=False) / 255.0
-        im -= mean
-        im /= std
-        return im
-    
-    def __call__(self, im, label=None):
-        """
-        Args:
-            im (np.ndarray): The Image data.
-            label (np.ndarray, optional): The label data. Default: None.
-        Returns:
-            (tuple). When label is None, it returns (im, ), otherwise it returns (im, label).
-        """
-
-        mean = np.array(self.mean)[np.newaxis, np.newaxis, :]
-        std = np.array(self.std)[np.newaxis, np.newaxis, :]
-        im = self.normalize(im, mean, std)
-
-        if label is None:
-            return (im, )
-        else:
-            return (im, label)
-
-
 class MiniSupervisely : 
     def __init__(self, root) : 
         self.root = root
@@ -100,26 +53,21 @@ def eval(self, model) :
 
         pbar.set_description(
             "Evaluating {} with {} val set".format(model.name, self.name))
-        
-        normalize = Normalize()
 
         for input_image, expected_image in pbar : 
             
-            input_image = cv.imread(os.path.join(self.root, input_image))
-            input_image = cv.resize(input_image, (192, 192))
+            input_image = cv.imread(os.path.join(self.root, input_image)).astype('float32')
             input_image = cv.cvtColor(input_image, cv.COLOR_BGR2RGB)
-
-            expected_image = cv.imread(os.path.join(self.root, expected_image),cv.IMREAD_GRAYSCALE)
-            expected_image = cv.resize(expected_image, (192, 192))[np.newaxis, :, :]
+            expected_image = cv.imread(os.path.join(self.root, expected_image), cv.IMREAD_GRAYSCALE)[np.newaxis, :, :]
+           
             
-            input_image, expected_image = normalize(input_image, expected_image)
-
-            output_image = model.infer(input_image)   
+            output_image = model.infer(input_image) 
 
             intersect_area, pred_area, label_area = self.calculate_area(
-                output_image,
-                expected_image,
+                output_image.astype('uint32'),
+                expected_image.astype('uint32'),
                 self.num_classes)
+            
             intersect_area_all = intersect_area_all + intersect_area
             pred_area_all = pred_area_all + pred_area
             label_area_all = label_area_all + label_area
@@ -169,34 +117,29 @@ def calculate_area(self,pred, label, num_classes, ignore_index=255):
             Tensor: The ground truth area on all class
         """
         
-        # Delete ignore_index
-        mask = label != ignore_index
-        pred = pred + 1
-        label = label + 1
-        pred = pred * mask
-        label = label * mask
-
-
-        pred = self.one_hot(pred, num_classes + 1)
-        label = self.one_hot(label, num_classes + 1)
-
-        pred = pred[:, :, :, 1:]
-        label = label[:, :, :, 1:]
+       
+        if len(pred.shape) == 4:
+            pred = np.squeeze(pred, axis=1)
+        if len(label.shape) == 4:
+            label = np.squeeze(label, axis=1)
+        if not pred.shape == label.shape:
+            raise ValueError('Shape of `pred` and `label should be equal, '
+                            'but there are {} and {}.'.format(pred.shape,
+                                                            label.shape))
 
+        mask = label != ignore_index
         pred_area = []
         label_area = []
         intersect_area = []
 
         #iterate over all classes and calculate their respective areas
         for i in range(num_classes):
-            pred_i = pred[:, :, :, i]
-            label_i = label[:, :, :, i]
-            pred_area_i = np.sum(pred_i)
-            label_area_i = np.sum(label_i)
-            intersect_area_i = np.sum(pred_i * label_i)
-            pred_area.append(pred_area_i)
-            label_area.append(label_area_i)
-            intersect_area.append(intersect_area_i)
+            pred_i = np.logical_and(pred == i, mask)
+            label_i = label == i
+            intersect_i = np.logical_and(pred_i, label_i)
+            pred_area.append(np.sum(pred_i.astype('int32')))
+            label_area.append(np.sum(label_i.astype('int32')))
+            intersect_area.append(np.sum(intersect_i.astype('int32')))
         
         return intersect_area, pred_area, label_area
     

From d449064e206d895c96e2e00652dd9df200992a43 Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Mon, 20 Feb 2023 15:55:37 +0530
Subject: [PATCH 8/9] final changes

---
 .../human_segmentation_pphumanseg/README.md   |  4 ++--
 .../pphumanseg.py                             | 10 +++++---
 tools/eval/datasets/__init__.py               |  2 +-
 tools/eval/datasets/minisupervisely.py        | 23 ++++++++-----------
 4 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/models/human_segmentation_pphumanseg/README.md b/models/human_segmentation_pphumanseg/README.md
index 312eb171..b9954104 100644
--- a/models/human_segmentation_pphumanseg/README.md
+++ b/models/human_segmentation_pphumanseg/README.md
@@ -27,8 +27,8 @@ Results of accuracy evaluation with [tools/eval](../../tools/eval).
 
 | Models             | Accuracy       | 
 | ------------------ | -------------- |
-| PPHumanSeg         | 0.9570         |
-| PPHumanSeg quant   | 0.4557         |
+| PPHumanSeg         | 0.9581         |
+| PPHumanSeg quant   | 0.4365        |
 
 
 \*: 'quant' stands for 'quantized'.
diff --git a/models/human_segmentation_pphumanseg/pphumanseg.py b/models/human_segmentation_pphumanseg/pphumanseg.py
index 8aebf8bd..4a20c840 100644
--- a/models/human_segmentation_pphumanseg/pphumanseg.py
+++ b/models/human_segmentation_pphumanseg/pphumanseg.py
@@ -37,6 +37,9 @@ def setTarget(self, target_id):
         self._model.setPreferableTarget(self._targetId)
 
     def _preprocess(self, image):
+
+        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
+
         self._currentInputSize = image.shape
         image = cv.resize(image, (192, 192))
         
@@ -52,15 +55,16 @@ def infer(self, image):
 
         # Forward
         self._model.setInput(inputBlob, self._inputNames)
-        outputBlob = self._model.forward(self._outputNames)
-
+        outputBlob = self._model.forward()
+       
         # Postprocess
         results = self._postprocess(outputBlob)
 
         return results
 
     def _postprocess(self, outputBlob):
-        outputBlob = outputBlob[0][0]
+        
+        outputBlob = outputBlob[0]
         outputBlob = cv.resize(outputBlob.transpose(1,2,0), (self._currentInputSize[1], self._currentInputSize[0]), interpolation=cv.INTER_LINEAR).transpose(2,0,1)[np.newaxis, ...]
 
         result = np.argmax(outputBlob, axis=1).astype(np.uint8)
diff --git a/tools/eval/datasets/__init__.py b/tools/eval/datasets/__init__.py
index 24687d23..5ed59faa 100644
--- a/tools/eval/datasets/__init__.py
+++ b/tools/eval/datasets/__init__.py
@@ -22,4 +22,4 @@ def register(self, item):
 DATASETS.register(LFW)
 DATASETS.register(ICDAR)
 DATASETS.register(IIIT5K)
-DATASETS.register(MiniSupervisely)
\ No newline at end of file
+DATASETS.register(MiniSupervisely)
diff --git a/tools/eval/datasets/minisupervisely.py b/tools/eval/datasets/minisupervisely.py
index 58b4397e..979a8e09 100644
--- a/tools/eval/datasets/minisupervisely.py
+++ b/tools/eval/datasets/minisupervisely.py
@@ -45,9 +45,9 @@ def eval(self, model) :
             model (object) : PP_HumanSeg model object
         """
 
-        intersect_area_all = []
-        pred_area_all = []
-        label_area_all = []
+        intersect_area_all = np.zeros([1], dtype=np.int64)
+        pred_area_all = np.zeros([1], dtype=np.int64)
+        label_area_all = np.zeros([1], dtype=np.int64)
 
         pbar = tqdm(self.image_set)
 
@@ -57,12 +57,11 @@ def eval(self, model) :
         for input_image, expected_image in pbar : 
             
             input_image = cv.imread(os.path.join(self.root, input_image)).astype('float32')
-            input_image = cv.cvtColor(input_image, cv.COLOR_BGR2RGB)
-            expected_image = cv.imread(os.path.join(self.root, expected_image), cv.IMREAD_GRAYSCALE)[np.newaxis, :, :]
-           
             
-            output_image = model.infer(input_image) 
+            expected_image = cv.imread(os.path.join(self.root, expected_image), cv.IMREAD_GRAYSCALE)[np.newaxis, :, :]           
 
+            output_image = model.infer(input_image) 
+            
             intersect_area, pred_area, label_area = self.calculate_area(
                 output_image.astype('uint32'),
                 expected_image.astype('uint32'),
@@ -71,8 +70,6 @@ def eval(self, model) :
             intersect_area_all = intersect_area_all + intersect_area
             pred_area_all = pred_area_all + pred_area
             label_area_all = label_area_all + label_area
-
-
             
         self.class_iou, self.miou = self.mean_iou(intersect_area_all, pred_area_all,
                                            label_area_all)
@@ -97,10 +94,8 @@ def print_result(self) :
         """
         print("Mean IoU : ", self.miou)
         print("Mean Accuracy : ", self.acc)
-
-    
-    def one_hot(self, arr, max_size) :
-        return np.eye(max_size)[arr]
+        print("Class IoU : ", self.class_iou)
+        print("Class Accuracy : ", self.class_acc)
 
 
     def calculate_area(self,pred, label, num_classes, ignore_index=255):
@@ -140,7 +135,7 @@ def calculate_area(self,pred, label, num_classes, ignore_index=255):
             pred_area.append(np.sum(pred_i.astype('int32')))
             label_area.append(np.sum(label_i.astype('int32')))
             intersect_area.append(np.sum(intersect_i.astype('int32')))
-        
+
         return intersect_area, pred_area, label_area
     
 

From c7c2700bc9340fa7e6b9beff509303dec10514b3 Mon Sep 17 00:00:00 2001
From: Labib Asari <asari.r.labeeb7@gmail.com>
Date: Tue, 21 Feb 2023 14:19:04 +0530
Subject: [PATCH 9/9] added mIoU and reference

---
 models/human_segmentation_pphumanseg/README.md     | 8 ++++----
 models/human_segmentation_pphumanseg/pphumanseg.py | 2 +-
 tools/eval/README.md                               | 6 +++---
 tools/eval/datasets/minisupervisely.py             | 6 ++++++
 4 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/models/human_segmentation_pphumanseg/README.md b/models/human_segmentation_pphumanseg/README.md
index b9954104..018bb56f 100644
--- a/models/human_segmentation_pphumanseg/README.md
+++ b/models/human_segmentation_pphumanseg/README.md
@@ -25,10 +25,10 @@ python demo.py --help
 ---
 Results of accuracy evaluation with [tools/eval](../../tools/eval).
 
-| Models             | Accuracy       | 
-| ------------------ | -------------- |
-| PPHumanSeg         | 0.9581         |
-| PPHumanSeg quant   | 0.4365        |
+| Models             | Accuracy       | mIoU          |
+| ------------------ | -------------- | ------------- |
+| PPHumanSeg         | 0.9581         | 0.8996        |
+| PPHumanSeg quant   | 0.4365         | 0.2788        |
 
 
 \*: 'quant' stands for 'quantized'.
diff --git a/models/human_segmentation_pphumanseg/pphumanseg.py b/models/human_segmentation_pphumanseg/pphumanseg.py
index 4a20c840..2f38159e 100644
--- a/models/human_segmentation_pphumanseg/pphumanseg.py
+++ b/models/human_segmentation_pphumanseg/pphumanseg.py
@@ -56,7 +56,7 @@ def infer(self, image):
         # Forward
         self._model.setInput(inputBlob, self._inputNames)
         outputBlob = self._model.forward()
-       
+
         # Postprocess
         results = self._postprocess(outputBlob)
 
diff --git a/tools/eval/README.md b/tools/eval/README.md
index c8332f7e..6c715637 100644
--- a/tools/eval/README.md
+++ b/tools/eval/README.md
@@ -194,20 +194,20 @@ python eval.py -m crnn -d iiit5k -dr /path/to/iiit5k
 ```
 
 
-## mini_supervisely
+## Mini Supervisely
 
 ### Prepare data
 Please download the mini_supervisely data from [here](https://paddleseg.bj.bcebos.com/humanseg/data/mini_supervisely.zip) which includes the validation dataset and unzip it.
 
 ### Evaluation
 
-Run evaluation with the following command
+Run evaluation with the following command :
 
 ```shell
 python eval.py -m pphumanseg -d mini_supervisely -dr /path/to/pphumanseg
 ```
 
-Run evaluation on quantized model with the following command
+Run evaluation on quantized model with the following command :
 
 ```shell
 python eval.py -m pphumanseg_q -d mini_supervisely -dr /path/to/pphumanseg
diff --git a/tools/eval/datasets/minisupervisely.py b/tools/eval/datasets/minisupervisely.py
index 979a8e09..63008dab 100644
--- a/tools/eval/datasets/minisupervisely.py
+++ b/tools/eval/datasets/minisupervisely.py
@@ -5,6 +5,12 @@
 
 
 class MiniSupervisely : 
+
+    '''
+        Refer to https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.7/paddleseg/core/val.py
+        for official evaluation implementation.
+    '''
+
     def __init__(self, root) : 
         self.root = root
         self.val_path = os.path.join(root, 'val.txt')