From 05a4cc2170c5521d281838a0b81d79bd29bd473d Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Thu, 8 Aug 2024 16:01:57 +0200 Subject: [PATCH 01/17] working on perturbation detectors --- giskard_vision/core/detectors/metrics.py | 9 ++ giskard_vision/core/detectors/perturbation.py | 114 ++++++++++++++++++ .../transformation_blurring_detector.py | 24 ++++ .../transformation_color_detector.py | 20 +++ giskard_vision/core/models/base.py | 2 +- .../dataloaders/loaders.py | 6 +- 6 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 giskard_vision/core/detectors/metrics.py create mode 100644 giskard_vision/core/detectors/perturbation.py create mode 100644 giskard_vision/core/detectors/transformation_blurring_detector.py create mode 100644 giskard_vision/core/detectors/transformation_color_detector.py diff --git a/giskard_vision/core/detectors/metrics.py b/giskard_vision/core/detectors/metrics.py new file mode 100644 index 00000000..206e34d0 --- /dev/null +++ b/giskard_vision/core/detectors/metrics.py @@ -0,0 +1,9 @@ +from giskard_vision.image_classification.tests.performance import Accuracy +from giskard_vision.landmark_detection.tests.performance import NMEMean +from giskard_vision.object_detection.tests.performance import IoU + +detector_metrics = { + "image_classification": Accuracy, + "landmark": NMEMean, + "object_detection": IoU, +} diff --git a/giskard_vision/core/detectors/perturbation.py b/giskard_vision/core/detectors/perturbation.py new file mode 100644 index 00000000..f84b0bdf --- /dev/null +++ b/giskard_vision/core/detectors/perturbation.py @@ -0,0 +1,114 @@ +import os +from abc import abstractmethod +from pathlib import Path +from typing import Any, Sequence + +import cv2 + +from giskard_vision.core.dataloaders.wrappers import FilteredDataLoader +from giskard_vision.core.detectors.base import ( + DetectorVisionBase, + IssueGroup, + ScanResult, +) +from giskard_vision.landmark_detection.tests.base import TestDiff +from giskard_vision.utils.errors import GiskardImportError + +from .metrics import detector_metrics + +Cropping = IssueGroup( + "Cropping", description="Cropping involves evaluating the landmark detection model on specific face areas." +) + +Ethical = IssueGroup( + "Ethical", + description="The data are filtered by ethnicity to detect ethical biases in the landmark detection model.", +) + +Pose = IssueGroup( + "Head Pose", + description="The data are filtered by head pose to detect biases in the landmark detection model.", +) + +Robustness = IssueGroup( + "Robustness", + description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", +) + + +class PerturbationBaseDetector(DetectorVisionBase): + """ + Abstract class for Landmark Detection Detectors + + Methods: + get_dataloaders(dataset: Any) -> Sequence[Any]: + Abstract method that returns a list of dataloaders corresponding to + slices or transformations + + get_results(model: Any, dataset: Any) -> Sequence[ScanResult]: + Returns a list of ScanResult containing the evaluation results + + get_scan_result(self, test_result) -> ScanResult: + Convert TestResult to ScanResult + """ + + @abstractmethod + def get_dataloaders(self, dataset: Any) -> Sequence[Any]: ... + + def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: + dataloaders = self.get_dataloaders(dataset) + + results = [] + for dl in dataloaders: + test_result = TestDiff(metric=detector_metrics[model.model_type], threshold=1).run( + model=model, + dataloader=dl, + dataloader_ref=dataset, + ) + + # Save example images from dataloader and dataset + current_path = str(Path()) + os.makedirs(f"{current_path}/examples_images", exist_ok=True) + filename_examples = [] + + index_worst = 0 if test_result.indexes_examples is None else test_result.indexes_examples[0] + + if isinstance(dl, FilteredDataLoader): + filename_example_dataloader_ref = str(Path() / "examples_images" / f"{dataset.name}_{index_worst}.png") + cv2.imwrite( + filename_example_dataloader_ref, cv2.resize(dataset[index_worst][0][0], (0, 0), fx=0.3, fy=0.3) + ) + filename_examples.append(filename_example_dataloader_ref) + + filename_example_dataloader = str(Path() / "examples_images" / f"{dl.name}_{index_worst}.png") + cv2.imwrite(filename_example_dataloader, cv2.resize(dl[index_worst][0][0], (0, 0), fx=0.3, fy=0.3)) + filename_examples.append(filename_example_dataloader) + results.append(self.get_scan_result(test_result, filename_examples, dl.name, len(dl))) + + return results + + def get_scan_result(self, test_result, filename_examples, name, size_data) -> ScanResult: + try: + from giskard.scanner.issues import IssueLevel + except (ImportError, ModuleNotFoundError) as e: + raise GiskardImportError(["giskard"]) from e + + relative_delta = (test_result.metric_value_test - test_result.metric_value_ref) / test_result.metric_value_ref + + if relative_delta > self.issue_level_threshold + self.deviation_threshold: + issue_level = IssueLevel.MAJOR + elif relative_delta > self.issue_level_threshold: + issue_level = IssueLevel.MEDIUM + else: + issue_level = IssueLevel.MINOR + + return ScanResult( + name=name, + metric_name=test_result.metric_name, + metric_value=test_result.metric_value_test, + metric_reference_value=test_result.metric_value_ref, + issue_level=issue_level, + slice_size=size_data, + filename_examples=filename_examples, + relative_delta=relative_delta, + ) diff --git a/giskard_vision/core/detectors/transformation_blurring_detector.py b/giskard_vision/core/detectors/transformation_blurring_detector.py new file mode 100644 index 00000000..481a50ea --- /dev/null +++ b/giskard_vision/core/detectors/transformation_blurring_detector.py @@ -0,0 +1,24 @@ +from giskard_vision.core.dataloaders.wrappers import BlurredDataLoader + +from ...core.detectors.decorator import maybe_detector +from .perturbation import PerturbationBaseDetector, Robustness + + +@maybe_detector("blurring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) +class TransformationBlurringDetectorLandmark(PerturbationBaseDetector): + """ + Detector that evaluates models performance on blurred images + """ + + issue_group = Robustness + + def __init__(self, kernel_size=(11, 11), sigma=(3, 3)): + self.kernel_size = kernel_size + self.sigma = sigma + + def get_dataloaders(self, dataset): + dl = BlurredDataLoader(dataset, self.kernel_size, self.sigma) + + dls = [dl] + + return dls diff --git a/giskard_vision/core/detectors/transformation_color_detector.py b/giskard_vision/core/detectors/transformation_color_detector.py new file mode 100644 index 00000000..70488ef0 --- /dev/null +++ b/giskard_vision/core/detectors/transformation_color_detector.py @@ -0,0 +1,20 @@ +from giskard_vision.core.dataloaders.wrappers import ColoredDataLoader + +from ...core.detectors.decorator import maybe_detector +from .perturbation import PerturbationBaseDetector, Robustness + + +@maybe_detector("coloring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) +class TransformationColorDetectorLandmark(PerturbationBaseDetector): + """ + Detector that evaluates models performance depending on images in grayscale + """ + + issue_group = Robustness + + def get_dataloaders(self, dataset): + dl = ColoredDataLoader(dataset) + + dls = [dl] + + return dls diff --git a/giskard_vision/core/models/base.py b/giskard_vision/core/models/base.py index 211e295b..b8bede60 100644 --- a/giskard_vision/core/models/base.py +++ b/giskard_vision/core/models/base.py @@ -40,7 +40,7 @@ def predict_batch(self, idx: int, images: List[np.ndarray]) -> np.ndarray: except Exception: res.append(None) logger.warning( - f"{self.__class__.__name__}: Face not detected in processed image of batch {idx} and index {i}." + f"{self.__class__.__name__}: Prediction failed in processed image of batch {idx} and index {i}." ) # logger.warning(e) # OpenCV's exception is very misleading diff --git a/giskard_vision/image_classification/dataloaders/loaders.py b/giskard_vision/image_classification/dataloaders/loaders.py index 4a8a78f2..ba22aa3c 100644 --- a/giskard_vision/image_classification/dataloaders/loaders.py +++ b/giskard_vision/image_classification/dataloaders/loaders.py @@ -45,7 +45,7 @@ def get_image(self, idx: int) -> np.ndarray: Returns: np.ndarray: The image data. """ - return self.get_row(idx)["image"] + return np.array(self.get_row(idx)["image"]) def get_labels(self, idx: int) -> Optional[np.ndarray]: """ @@ -124,7 +124,7 @@ def get_image(self, idx: int) -> Any: Returns: np.ndarray: The image data. """ - return self.ds[idx]["image"] + return np.array(self.ds[idx]["image"]) def get_labels(self, idx: int) -> Optional[np.ndarray]: """ @@ -201,7 +201,7 @@ def get_image(self, idx: int) -> Any: Returns: np.ndarray: The image data. """ - return self.ds[idx]["img"] + return np.array(self.ds[idx]["img"]) def get_labels(self, idx: int) -> Optional[np.ndarray]: """ From cf2a8c456bc6f145ebd6ae42e7c18d82fe0dcd89 Mon Sep 17 00:00:00 2001 From: Weixuan Xiao Date: Mon, 5 Aug 2024 17:15:27 +0200 Subject: [PATCH 02/17] Refactor HF ppl model to convert numpy array to PIL image --- giskard_vision/image_classification/models/base.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/giskard_vision/image_classification/models/base.py b/giskard_vision/image_classification/models/base.py index 0a6d04a9..1f95750c 100644 --- a/giskard_vision/image_classification/models/base.py +++ b/giskard_vision/image_classification/models/base.py @@ -1,6 +1,7 @@ from typing import Optional import numpy as np +from PIL import Image from giskard_vision.core.models.hf_pipeline import HFPipelineModelBase, HFPipelineTask from giskard_vision.image_classification.types import Types @@ -57,25 +58,30 @@ class SingleLabelImageClassificationHFModelWrapper(ImageClassificationHFModel): classification_labels: list of classification labels, where the position of the label corresponds to the class index """ - def predict_probas(self, image: np.ndarray) -> np.ndarray: + def predict_probas(self, image: np.ndarray, mode="RGB") -> np.ndarray: """method that takes one image as input and outputs the prediction of probabilities for each class Args: image (np.ndarray): input image + mode (str): mode of the image """ + pil_image = Image.fromarray(image, mode=mode) + + # Pipeline takes a PIL image as input _raw_prediction = self.pipeline( - image, + pil_image, top_k=len(self.classification_labels), # Get probabilities for all labels ) _prediction = {p["label"]: p["score"] for p in _raw_prediction} return np.array([_prediction[label] for label in self.classification_labels]) - def predict_image(self, image) -> Types.label: + def predict_image(self, image, mode="RGB") -> Types.label: """method that takes one image as input and outputs one class label Args: image (np.ndarray): input image + mode (str): mode of the image """ - probas = self.predict_probas(image) + probas = self.predict_probas(image, mode=mode) return self.classification_labels[np.argmax(probas)] From 7ba35b6ef6e04a5d7e2352fc18b863b2094cf4db Mon Sep 17 00:00:00 2001 From: Weixuan Xiao Date: Mon, 5 Aug 2024 17:24:15 +0200 Subject: [PATCH 03/17] Allow to set global mode for an HF ppl model for PIL conversion --- giskard_vision/core/dataloaders/hf.py | 6 ++++- giskard_vision/core/models/hf_pipeline.py | 6 ++--- .../image_classification/models/base.py | 24 +++++++++++-------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/giskard_vision/core/dataloaders/hf.py b/giskard_vision/core/dataloaders/hf.py index e746ab02..a615a47a 100644 --- a/giskard_vision/core/dataloaders/hf.py +++ b/giskard_vision/core/dataloaders/hf.py @@ -31,7 +31,11 @@ class HFDataLoader(DataIteratorBase): """ def __init__( - self, hf_id: str, hf_config: Optional[str] = None, hf_split: str = "test", name: Optional[str] = None + self, + hf_id: str, + hf_config: Optional[str] = None, + hf_split: str = "test", + name: Optional[str] = None, ) -> None: """ Initializes the general HuggingFace Datasets instance. diff --git a/giskard_vision/core/models/hf_pipeline.py b/giskard_vision/core/models/hf_pipeline.py index 95bb483d..8c63094e 100644 --- a/giskard_vision/core/models/hf_pipeline.py +++ b/giskard_vision/core/models/hf_pipeline.py @@ -19,9 +19,9 @@ def __init__( """init method that accepts a model object, number of landmarks and dimensions Args: - model_id (str): Hugging Face model ID - name (Optional[str]): name of the model - pipeline_task (HFPipelineTask): HuggingFace pipeline task + model_id (str): Hugging Face model ID. + name (Optional[str]): name of the model. + pipeline_task (HFPipelineTask): HuggingFace pipeline task. Raises: GiskardImportError: If there are missing Hugging Face dependencies. diff --git a/giskard_vision/image_classification/models/base.py b/giskard_vision/image_classification/models/base.py index 1f95750c..92c7bbfe 100644 --- a/giskard_vision/image_classification/models/base.py +++ b/giskard_vision/image_classification/models/base.py @@ -11,9 +11,10 @@ class ImageClassificationHFModel(HFPipelineModelBase): """Hugging Face pipeline wrapper class that serves as a template for image classification predictions Args: - model_id (str): Hugging Face model ID - name (Optional[str]): name of the model - device (str): device to run the model on + model_id (str): Hugging Face model ID. + name (Optional[str]): name of the model. + device (str): device to run the model on. + mode (str): The mode to convert the numpy image data to PIL image, defaulting to "RGB". Attributes: classification_labels: list of classification labels, where the position of the label corresponds to the class index @@ -22,13 +23,14 @@ class ImageClassificationHFModel(HFPipelineModelBase): model_type = "image_classification" prediction_result_cls = Types.prediction_result - def __init__(self, model_id: str, name: Optional[str] = None, device: str = "cpu"): + def __init__(self, model_id: str, name: Optional[str] = None, device: str = "cpu", mode: str = "RGB"): """init method that accepts a model id, name and device Args: - model_id (str): Hugging Face model ID - name (Optional[str]): name of the model - device (str): device to run the model on + model_id (str): Hugging Face model ID. + name (Optional[str]): name of the model. + device (str): device to run the model on. + mode (str): The mode to convert the numpy image data to PIL image, defaulting to "RGB". """ super().__init__( @@ -39,6 +41,7 @@ def __init__(self, model_id: str, name: Optional[str] = None, device: str = "cpu ) self._classification_labels = list(self.pipeline.model.config.id2label.values()) + self._mode = mode @property def classification_labels(self): @@ -58,14 +61,15 @@ class SingleLabelImageClassificationHFModelWrapper(ImageClassificationHFModel): classification_labels: list of classification labels, where the position of the label corresponds to the class index """ - def predict_probas(self, image: np.ndarray, mode="RGB") -> np.ndarray: + def predict_probas(self, image: np.ndarray, mode=None) -> np.ndarray: """method that takes one image as input and outputs the prediction of probabilities for each class Args: image (np.ndarray): input image mode (str): mode of the image """ - pil_image = Image.fromarray(image, mode=mode) + m = mode or self._mode + pil_image = Image.fromarray(image, mode=m) # Pipeline takes a PIL image as input _raw_prediction = self.pipeline( @@ -76,7 +80,7 @@ def predict_probas(self, image: np.ndarray, mode="RGB") -> np.ndarray: return np.array([_prediction[label] for label in self.classification_labels]) - def predict_image(self, image, mode="RGB") -> Types.label: + def predict_image(self, image, mode=None) -> Types.label: """method that takes one image as input and outputs one class label Args: From 6c3ffc9d8247461653093e2bf5db69b99330c6c5 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Thu, 8 Aug 2024 16:23:32 +0200 Subject: [PATCH 04/17] mode switch in hf models --- giskard_vision/image_classification/models/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/giskard_vision/image_classification/models/base.py b/giskard_vision/image_classification/models/base.py index 92c7bbfe..57cd82f6 100644 --- a/giskard_vision/image_classification/models/base.py +++ b/giskard_vision/image_classification/models/base.py @@ -87,5 +87,7 @@ def predict_image(self, image, mode=None) -> Types.label: image (np.ndarray): input image mode (str): mode of the image """ + if len(image.shape) == 2: + mode = "L" # Grayscale - Needed to run the color robustness detector probas = self.predict_probas(image, mode=mode) return self.classification_labels[np.argmax(probas)] From 7e1479599469536d86d9d68e70d905b1a3d2c1e8 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Thu, 8 Aug 2024 17:01:55 +0200 Subject: [PATCH 05/17] supporting gray scale --- giskard_vision/core/models/base.py | 26 +++++++++++++++++-- .../image_classification/models/base.py | 14 ++++------ .../landmark_detection/models/wrappers.py | 4 +-- .../object_detection/models/wrappers.py | 2 +- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/giskard_vision/core/models/base.py b/giskard_vision/core/models/base.py index b8bede60..9f02c78d 100644 --- a/giskard_vision/core/models/base.py +++ b/giskard_vision/core/models/base.py @@ -18,8 +18,8 @@ class ModelBase(ABC): prediction_result_cls = TypesBase.prediction_result @abstractmethod - def predict_image(self, image: np.ndarray) -> Any: - """abstract method that takes one image as input and outputs the prediction + def predict_rgb_image(self, image: np.ndarray) -> Any: + """abstract method that takes one RGB image as input and outputs the prediction Args: image (np.ndarray): input image @@ -27,6 +27,28 @@ def predict_image(self, image: np.ndarray) -> Any: ... + def predict_gray_image(self, image: np.ndarray) -> Any: + """abstract method that takes one gray image as input and outputs the prediction + + Args: + image (np.ndarray): input image + """ + + raise NotImplementedError("predict_gray_image method is not implemented") + + def predict_image(self, image: np.ndarray) -> Any: + """abstract method that takes one image as input and outputs the prediction + + Args: + image (np.ndarray): input image + """ + if image.shape[-1] == 3: + return self.predict_rgb_image(image) + elif image.shape[-1] == 1 or len(image.shape) == 2: + return self.predict_gray_image(image) + else: + raise ValueError("predict_image: image shape not supported.") + def predict_batch(self, idx: int, images: List[np.ndarray]) -> np.ndarray: """method that should be implemented if the passed dataloader has batch_size != 1 diff --git a/giskard_vision/image_classification/models/base.py b/giskard_vision/image_classification/models/base.py index 57cd82f6..0ebb2cb2 100644 --- a/giskard_vision/image_classification/models/base.py +++ b/giskard_vision/image_classification/models/base.py @@ -80,14 +80,10 @@ def predict_probas(self, image: np.ndarray, mode=None) -> np.ndarray: return np.array([_prediction[label] for label in self.classification_labels]) - def predict_image(self, image, mode=None) -> Types.label: - """method that takes one image as input and outputs one class label + def predict_rgb_image(self, image: np.ndarray) -> Types.label: + probas = self.predict_probas(image, mode=None) + return self.classification_labels[np.argmax(probas)] - Args: - image (np.ndarray): input image - mode (str): mode of the image - """ - if len(image.shape) == 2: - mode = "L" # Grayscale - Needed to run the color robustness detector - probas = self.predict_probas(image, mode=mode) + def predict_gray_image(self, image: np.ndarray) -> Types.label: + probas = self.predict_probas(image, mode="L") return self.classification_labels[np.argmax(probas)] diff --git a/giskard_vision/landmark_detection/models/wrappers.py b/giskard_vision/landmark_detection/models/wrappers.py index 937d56c6..f9e85d42 100644 --- a/giskard_vision/landmark_detection/models/wrappers.py +++ b/giskard_vision/landmark_detection/models/wrappers.py @@ -29,7 +29,7 @@ def __init__(self, model): super().__init__(n_landmarks=68, n_dimensions=2, name="FaceAlignment") self.model = model - def predict_image(self, image): + def predict_rgb_image(self, image): """ Predict facial landmarks for a given image using the wrapped face alignment model. @@ -100,7 +100,7 @@ def __init__(self): self.landmark_detector = cv2.face.createFacemarkLBF() self.landmark_detector.loadModel(LBFmodel) - def predict_image(self, image): + def predict_rgb_image(self, image): """ Predict facial landmarks for a given image using the wrapped OpenCV face landmarks model. diff --git a/giskard_vision/object_detection/models/wrappers.py b/giskard_vision/object_detection/models/wrappers.py index 735ff096..b350f22f 100644 --- a/giskard_vision/object_detection/models/wrappers.py +++ b/giskard_vision/object_detection/models/wrappers.py @@ -96,7 +96,7 @@ def shape_rescale(self, image, boxes): def positive_constraint(self, boxes): return np.clip(boxes, 0, None) - def predict_image(self, image: np.ndarray): + def predict_rgb_image(self, image: np.ndarray): try: from keras.applications.mobilenet import preprocess_input except ImportError: From 98baa6d08b9766d2428d9c71ee6525abdb700e27 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Mon, 12 Aug 2024 13:10:30 +0200 Subject: [PATCH 06/17] added missing predict_rgb_image --- giskard_vision/object_detection/models/wrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giskard_vision/object_detection/models/wrappers.py b/giskard_vision/object_detection/models/wrappers.py index a762dab6..b7f461a7 100644 --- a/giskard_vision/object_detection/models/wrappers.py +++ b/giskard_vision/object_detection/models/wrappers.py @@ -259,7 +259,7 @@ def __init__(self, name: str = None, device: str = "cpu"): device=device, ) - def predict_image(self, image: np.ndarray) -> Any: + def predict_rgb_image(self, image: np.ndarray) -> Any: raw_predictions = super().predict_raw(image) # Filter out predictions with a highest score From a9fa22f149f0004eb0d2301fc0b9ae3190bfcbfd Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Mon, 12 Aug 2024 13:12:05 +0200 Subject: [PATCH 07/17] ensuring backward compatibility with predict_image --- giskard_vision/core/models/base.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/giskard_vision/core/models/base.py b/giskard_vision/core/models/base.py index 9f02c78d..ae655d26 100644 --- a/giskard_vision/core/models/base.py +++ b/giskard_vision/core/models/base.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +from abc import ABC from logging import getLogger from time import time from typing import Any, List, Optional @@ -17,18 +17,17 @@ class ModelBase(ABC): model_type: str prediction_result_cls = TypesBase.prediction_result - @abstractmethod def predict_rgb_image(self, image: np.ndarray) -> Any: - """abstract method that takes one RGB image as input and outputs the prediction + """method that takes one RGB image as input and outputs the prediction Args: image (np.ndarray): input image """ - ... + raise NotImplementedError("predict_rgb_image method is not implemented") def predict_gray_image(self, image: np.ndarray) -> Any: - """abstract method that takes one gray image as input and outputs the prediction + """method that takes one gray image as input and outputs the prediction Args: image (np.ndarray): input image From e9198ce6a0449b7674d4015284b59642cff9bd74 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Mon, 12 Aug 2024 15:24:51 +0200 Subject: [PATCH 08/17] updating detectors --- giskard_vision/core/detectors/perturbation.py | 20 ++++--------------- .../transformation_blurring_detector.py | 4 +--- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/giskard_vision/core/detectors/perturbation.py b/giskard_vision/core/detectors/perturbation.py index f84b0bdf..61a767f8 100644 --- a/giskard_vision/core/detectors/perturbation.py +++ b/giskard_vision/core/detectors/perturbation.py @@ -11,25 +11,11 @@ IssueGroup, ScanResult, ) -from giskard_vision.landmark_detection.tests.base import TestDiff +from giskard_vision.core.tests.base import TestDiffBase from giskard_vision.utils.errors import GiskardImportError from .metrics import detector_metrics -Cropping = IssueGroup( - "Cropping", description="Cropping involves evaluating the landmark detection model on specific face areas." -) - -Ethical = IssueGroup( - "Ethical", - description="The data are filtered by ethnicity to detect ethical biases in the landmark detection model.", -) - -Pose = IssueGroup( - "Head Pose", - description="The data are filtered by head pose to detect biases in the landmark detection model.", -) - Robustness = IssueGroup( "Robustness", description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", @@ -52,6 +38,8 @@ class PerturbationBaseDetector(DetectorVisionBase): Convert TestResult to ScanResult """ + issue_group = Robustness + @abstractmethod def get_dataloaders(self, dataset: Any) -> Sequence[Any]: ... @@ -60,7 +48,7 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: results = [] for dl in dataloaders: - test_result = TestDiff(metric=detector_metrics[model.model_type], threshold=1).run( + test_result = TestDiffBase(metric=detector_metrics[model.model_type], threshold=1).run( model=model, dataloader=dl, dataloader_ref=dataset, diff --git a/giskard_vision/core/detectors/transformation_blurring_detector.py b/giskard_vision/core/detectors/transformation_blurring_detector.py index 481a50ea..be1b912c 100644 --- a/giskard_vision/core/detectors/transformation_blurring_detector.py +++ b/giskard_vision/core/detectors/transformation_blurring_detector.py @@ -1,7 +1,7 @@ from giskard_vision.core.dataloaders.wrappers import BlurredDataLoader from ...core.detectors.decorator import maybe_detector -from .perturbation import PerturbationBaseDetector, Robustness +from .perturbation import PerturbationBaseDetector @maybe_detector("blurring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) @@ -10,8 +10,6 @@ class TransformationBlurringDetectorLandmark(PerturbationBaseDetector): Detector that evaluates models performance on blurred images """ - issue_group = Robustness - def __init__(self, kernel_size=(11, 11), sigma=(3, 3)): self.kernel_size = kernel_size self.sigma = sigma From 4dd46b4fb4b725caf88e0eb91b529d90d07fcef9 Mon Sep 17 00:00:00 2001 From: bmalezieux <40993275+bmalezieux@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:25:15 +0200 Subject: [PATCH 09/17] Adding noise perturbation detector with Gaussian noise (#52) * ENH: adding noise perturbation detector with Gaussian noise * FIX: docstring * FIX: the IoU metric now handles prediction batches * renaming metric IoU to IoUMean --------- Co-authored-by: Rabah Khalek --- giskard_vision/core/dataloaders/wrappers.py | 72 +++++++++++++++++++ giskard_vision/core/detectors/metrics.py | 4 +- .../transformation_noise_detector.py | 23 ++++++ giskard_vision/core/tests/base.py | 2 +- .../detectors/metadata_detector.py | 4 +- .../object_detection/tests/performance.py | 61 +++++++++------- 6 files changed, 136 insertions(+), 30 deletions(-) create mode 100644 giskard_vision/core/detectors/transformation_noise_detector.py diff --git a/giskard_vision/core/dataloaders/wrappers.py b/giskard_vision/core/dataloaders/wrappers.py index 0682dbbe..ff39ed9f 100644 --- a/giskard_vision/core/dataloaders/wrappers.py +++ b/giskard_vision/core/dataloaders/wrappers.py @@ -228,6 +228,78 @@ def get_image(self, idx: int) -> np.ndarray: return cv2.GaussianBlur(image, self._kernel_size, *self._sigma) +class NoisyDataLoader(DataLoaderWrapper): + """Wrapper class for a DataIteratorBase, providing noisy images. + + Args: + dataloader (DataIteratorBase): The data loader to be wrapped. + sigma (float): Standard deviation of the Gaussian noise. + + Returns: + NoisyDataLoader: Noisy data loader instance. + """ + + def __init__( + self, + dataloader: DataIteratorBase, + sigma: float = 0.1, + ) -> None: + """ + Initializes the BlurredDataLoader. + + Args: + dataloader (DataIteratorBase): The data loader to be wrapped. + sigma (float): Standard deviation of the Gaussian noise. + """ + super().__init__(dataloader) + self._sigma = sigma + + @property + def name(self): + """ + Gets the name of the blurred data loader. + + Returns: + str: The name of the blurred data loader. + """ + return "noisy" + + def get_image(self, idx: int) -> np.ndarray: + """ + Gets a blurred image using Gaussian blur. + + Args: + idx (int): Index of the data. + + Returns: + np.ndarray: Blurred image data. + """ + image = super().get_image(idx) + return self.add_gaussian_noise(image, self._sigma * 255) + + def add_gaussian_noise(self, image, std_dev): + """ + Add Gaussian noise to the image + + Args: + image (np.ndarray): Image + std_dev (float): Standard deviation of the Gaussian noise. + + Returns: + np.ndarray: Noisy image + """ + # Generate Gaussian noise + noise = np.random.normal(0, std_dev, image.shape).astype(np.float32) + + # Add the noise to the image + noisy_image = cv2.add(image.astype(np.float32), noise) + + # Clip the values to stay within valid range (0-255 for uint8) + noisy_image = np.clip(noisy_image, 0, 255).astype(np.uint8) + + return noisy_image + + class ColoredDataLoader(DataLoaderWrapper): """Wrapper class for a DataIteratorBase, providing color-altered images using OpenCV color conversion. diff --git a/giskard_vision/core/detectors/metrics.py b/giskard_vision/core/detectors/metrics.py index 206e34d0..744ee1f7 100644 --- a/giskard_vision/core/detectors/metrics.py +++ b/giskard_vision/core/detectors/metrics.py @@ -1,9 +1,9 @@ from giskard_vision.image_classification.tests.performance import Accuracy from giskard_vision.landmark_detection.tests.performance import NMEMean -from giskard_vision.object_detection.tests.performance import IoU +from giskard_vision.object_detection.tests.performance import IoUMean detector_metrics = { "image_classification": Accuracy, "landmark": NMEMean, - "object_detection": IoU, + "object_detection": IoUMean, } diff --git a/giskard_vision/core/detectors/transformation_noise_detector.py b/giskard_vision/core/detectors/transformation_noise_detector.py new file mode 100644 index 00000000..e5c03652 --- /dev/null +++ b/giskard_vision/core/detectors/transformation_noise_detector.py @@ -0,0 +1,23 @@ +from giskard_vision.core.dataloaders.wrappers import NoisyDataLoader + +from ...core.detectors.decorator import maybe_detector +from .perturbation import PerturbationBaseDetector, Robustness + + +@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) +class TransformationNoiseDetectorLandmark(PerturbationBaseDetector): + """ + Detector that evaluates models performance on noisy images + """ + + issue_group = Robustness + + def __init__(self, sigma=0.1): + self.sigma = sigma + + def get_dataloaders(self, dataset): + dl = NoisyDataLoader(dataset, self.sigma) + + dls = [dl] + + return dls diff --git a/giskard_vision/core/tests/base.py b/giskard_vision/core/tests/base.py index e5d5c1b8..25008094 100644 --- a/giskard_vision/core/tests/base.py +++ b/giskard_vision/core/tests/base.py @@ -314,7 +314,7 @@ def run( prediction_time=prediction_time, prediction_fail_rate=prediction_fail_rate, metric_name=self.metric.name, - model_name=model.name, + model_name=model.name if hasattr(model, "name") else model.__class__.__name__, dataloader_name=dataloader.name, dataloader_ref_name=dataloader_ref.name, indexes_examples=indexes, diff --git a/giskard_vision/object_detection/detectors/metadata_detector.py b/giskard_vision/object_detection/detectors/metadata_detector.py index 1d4f02eb..74844b98 100644 --- a/giskard_vision/object_detection/detectors/metadata_detector.py +++ b/giskard_vision/object_detection/detectors/metadata_detector.py @@ -15,7 +15,7 @@ SurrogateRelativeTopLeftY, SurrogateStdIntensity, ) -from giskard_vision.object_detection.tests.performance import IoU +from giskard_vision.object_detection.tests.performance import IoUMean from ...core.detectors.decorator import maybe_detector @@ -38,7 +38,7 @@ class MetaDataScanDetectorObjectDetection(MetaDataScanDetector): SurrogateRelativeTopLeftY, SurrogateNormalizedPerimeter, ] - metric = IoU + metric = IoUMean type_task = "regression" metric_type = "absolute" metric_direction = "better_higher" diff --git a/giskard_vision/object_detection/tests/performance.py b/giskard_vision/object_detection/tests/performance.py index 933f6c67..35c63f60 100644 --- a/giskard_vision/object_detection/tests/performance.py +++ b/giskard_vision/object_detection/tests/performance.py @@ -1,48 +1,59 @@ from dataclasses import dataclass +import numpy as np + from ..types import Types from .base import Metric @dataclass -class IoU(Metric): +class IoUMean(Metric): """Intersection over Union distance between a prediction and a ground truth""" - name = "IoU" + name = "IoUMean" description = "Intersection over Union" @staticmethod def definition(prediction_result: Types.prediction_result, ground_truth: Types.label): - # if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"): # return 0 - gt_box = prediction_result.prediction.item().get("boxes") - pred_box = ground_truth.item().get("boxes") + ious = [] + + for i in range(len(prediction_result.prediction)): + if isinstance(prediction_result.prediction[i], dict): + gt_box = prediction_result.prediction[i].get("boxes") + else: + ious.append(0) + continue + + pred_box = ground_truth[i].get("boxes") + + x1_min, y1_min, x1_max, y1_max = gt_box + x2_min, y2_min, x2_max, y2_max = pred_box - x1_min, y1_min, x1_max, y1_max = gt_box - x2_min, y2_min, x2_max, y2_max = pred_box + # Calculate the coordinates of the intersection rectangle + x_inter_min = max(x1_min, x2_min) + y_inter_min = max(y1_min, y2_min) + x_inter_max = min(x1_max, x2_max) + y_inter_max = min(y1_max, y2_max) - # Calculate the coordinates of the intersection rectangle - x_inter_min = max(x1_min, x2_min) - y_inter_min = max(y1_min, y2_min) - x_inter_max = min(x1_max, x2_max) - y_inter_max = min(y1_max, y2_max) + # Compute the area of the intersection rectangle + if x_inter_max < x_inter_min or y_inter_max < y_inter_min: + inter_area = 0 + else: + inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min) - # Compute the area of the intersection rectangle - if x_inter_max < x_inter_min or y_inter_max < y_inter_min: - inter_area = 0 - else: - inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min) + # Compute the area of both the prediction and ground-truth rectangles + box1_area = (x1_max - x1_min) * (y1_max - y1_min) + box2_area = (x2_max - x2_min) * (y2_max - y2_min) - # Compute the area of both the prediction and ground-truth rectangles - box1_area = (x1_max - x1_min) * (y1_max - y1_min) - box2_area = (x2_max - x2_min) * (y2_max - y2_min) + # Compute the union area + union_area = box1_area + box2_area - inter_area - # Compute the union area - union_area = box1_area + box2_area - inter_area + # Compute the IoU + iou = inter_area / union_area - # Compute the IoU - iou = inter_area / union_area + ious.append(iou) - return iou + return np.mean(ious) From e547d4d9bb4922fd9ff0508fecdf0c2da341475c Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Mon, 12 Aug 2024 15:25:51 +0200 Subject: [PATCH 10/17] updating detectors --- .../core/detectors/transformation_noise_detector.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/giskard_vision/core/detectors/transformation_noise_detector.py b/giskard_vision/core/detectors/transformation_noise_detector.py index e5c03652..0ce63b13 100644 --- a/giskard_vision/core/detectors/transformation_noise_detector.py +++ b/giskard_vision/core/detectors/transformation_noise_detector.py @@ -1,7 +1,7 @@ from giskard_vision.core.dataloaders.wrappers import NoisyDataLoader from ...core.detectors.decorator import maybe_detector -from .perturbation import PerturbationBaseDetector, Robustness +from .perturbation import PerturbationBaseDetector @maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) @@ -10,8 +10,6 @@ class TransformationNoiseDetectorLandmark(PerturbationBaseDetector): Detector that evaluates models performance on noisy images """ - issue_group = Robustness - def __init__(self, sigma=0.1): self.sigma = sigma From a44399da505bb916c52031e7b61ee1141810ae9a Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Tue, 13 Aug 2024 12:40:13 +0200 Subject: [PATCH 11/17] refactoring detectors --- giskard_vision/core/dataloaders/base.py | 10 --- giskard_vision/core/dataloaders/meta.py | 2 +- giskard_vision/core/detectors/base.py | 51 ++++++++--- .../core/detectors/metadata_scan_detector.py | 43 +--------- giskard_vision/core/detectors/metrics.py | 9 -- giskard_vision/core/detectors/perturbation.py | 86 +++++++++---------- giskard_vision/core/detectors/specs.py | 13 +++ .../transformation_color_detector.py | 4 +- .../transformation_noise_detector.py | 2 +- giskard_vision/core/issues.py | 21 +++++ .../dataloaders/loaders.py | 2 +- .../detectors/metadata_detector.py | 11 +-- .../image_classification/detectors/specs.py | 11 +++ .../landmark_detection/dataloaders/loaders.py | 11 +-- .../landmark_detection/detectors/base.py | 25 +----- .../detectors/cropping_detector.py | 6 +- .../detectors/metadata_detector.py | 7 +- .../landmark_detection/detectors/specs.py | 8 ++ .../transformation_blurring_detector.py | 24 ------ .../transformation_color_detector.py | 20 ----- .../transformation_resize_detector.py | 3 +- .../landmark_detection/models/base.py | 2 +- .../detectors/metadata_detector.py | 10 +-- .../object_detection/detectors/specs.py | 11 +++ 24 files changed, 165 insertions(+), 227 deletions(-) delete mode 100644 giskard_vision/core/detectors/metrics.py create mode 100644 giskard_vision/core/detectors/specs.py create mode 100644 giskard_vision/core/issues.py create mode 100644 giskard_vision/image_classification/detectors/specs.py create mode 100644 giskard_vision/landmark_detection/detectors/specs.py delete mode 100644 giskard_vision/landmark_detection/detectors/transformation_blurring_detector.py delete mode 100644 giskard_vision/landmark_detection/detectors/transformation_color_detector.py create mode 100644 giskard_vision/object_detection/detectors/specs.py diff --git a/giskard_vision/core/dataloaders/base.py b/giskard_vision/core/dataloaders/base.py index f0a3d4a9..2219dc08 100644 --- a/giskard_vision/core/dataloaders/base.py +++ b/giskard_vision/core/dataloaders/base.py @@ -5,19 +5,9 @@ import numpy as np from giskard_vision.core.dataloaders.meta import MetaData -from giskard_vision.core.detectors.base import IssueGroup from ..types import TypesBase -EthicalIssueMeta = IssueGroup( - "Ethical", - description="The data are filtered by metadata like age, facial hair, or gender to detect ethical biases.", -) -PerformanceIssueMeta = IssueGroup( - "Performance", - description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.", -) - class DataIteratorBase(ABC): """Abstract class serving as a base template for DataLoaderBase and DataLoaderWrapper classes. diff --git a/giskard_vision/core/dataloaders/meta.py b/giskard_vision/core/dataloaders/meta.py index f7d79ff9..19dff893 100644 --- a/giskard_vision/core/dataloaders/meta.py +++ b/giskard_vision/core/dataloaders/meta.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional -from giskard_vision.core.detectors.base import IssueGroup +from giskard_vision.core.issues import IssueGroup class MetaData: diff --git a/giskard_vision/core/detectors/base.py b/giskard_vision/core/detectors/base.py index d6197753..1ca41ca3 100644 --- a/giskard_vision/core/detectors/base.py +++ b/giskard_vision/core/detectors/base.py @@ -2,13 +2,10 @@ from dataclasses import dataclass from typing import Any, List, Optional, Sequence, Tuple +from giskard_vision.core.issues import IssueGroup from giskard_vision.utils.errors import GiskardImportError - -@dataclass(frozen=True) -class IssueGroup: - name: str - description: str +from .specs import DetectorSpecsBase @dataclass @@ -51,7 +48,7 @@ def get_meta_required(self) -> dict: } -class DetectorVisionBase: +class DetectorVisionBase(DetectorSpecsBase): """ Abstract class for Vision Detectors @@ -67,12 +64,6 @@ class DetectorVisionBase: evaluation results for the scan. """ - issue_group: IssueGroup - warning_messages: dict - issue_level_threshold: float = 0.2 - deviation_threshold: float = 0.05 - num_images: int = 0 - def run( self, model: Any, @@ -139,6 +130,42 @@ def get_issues( return issues + def get_scan_result( + self, metric_value, metric_reference_value, metric_name, filename_examples, name, size_data, issue_group + ) -> ScanResult: + try: + from giskard.scanner.issues import IssueLevel + except (ImportError, ModuleNotFoundError) as e: + raise GiskardImportError(["giskard"]) from e + + relative_delta = metric_value - metric_reference_value + if self.metric_type == "relative": + relative_delta /= metric_reference_value + + issue_level = IssueLevel.MINOR + if self.metric_direction == "better_lower": + if relative_delta > self.issue_level_threshold + self.deviation_threshold: + issue_level = IssueLevel.MAJOR + elif relative_delta > self.issue_level_threshold: + issue_level = IssueLevel.MEDIUM + elif self.metric_direction == "better_higher": + if relative_delta < -(self.issue_level_threshold + self.deviation_threshold): + issue_level = IssueLevel.MAJOR + elif relative_delta < -self.issue_level_threshold: + issue_level = IssueLevel.MEDIUM + + return ScanResult( + name=name, + metric_name=metric_name, + metric_value=metric_value, + metric_reference_value=metric_reference_value, + issue_level=issue_level, + slice_size=size_data, + filename_examples=filename_examples, + relative_delta=relative_delta, + issue_group=issue_group, + ) + @abstractmethod def get_results(self, model: Any, dataset: Any) -> List[ScanResult]: """Returns a list of ScanResult diff --git a/giskard_vision/core/detectors/metadata_scan_detector.py b/giskard_vision/core/detectors/metadata_scan_detector.py index 56cc7782..f657c3bb 100644 --- a/giskard_vision/core/detectors/metadata_scan_detector.py +++ b/giskard_vision/core/detectors/metadata_scan_detector.py @@ -4,11 +4,8 @@ import numpy as np import pandas as pd -from giskard_vision.core.detectors.base import ( - DetectorVisionBase, - IssueGroup, - ScanResult, -) +from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult +from giskard_vision.core.issues import IssueGroup from giskard_vision.core.tests.base import MetricBase from giskard_vision.utils.errors import GiskardImportError @@ -263,39 +260,3 @@ def get_df_for_scan(self, model: Any, dataset: Any, list_metadata: Sequence[str] pass return pd.DataFrame(df) - - def get_scan_result( - self, metric_value, metric_reference_value, metric_name, filename_examples, name, size_data, issue_group - ) -> ScanResult: - try: - from giskard.scanner.issues import IssueLevel - except (ImportError, ModuleNotFoundError) as e: - raise GiskardImportError(["giskard"]) from e - - relative_delta = metric_value - metric_reference_value - if self.metric_type == "relative": - relative_delta /= metric_reference_value - - issue_level = IssueLevel.MINOR - if self.metric_direction == "better_lower": - if relative_delta > self.issue_level_threshold + self.deviation_threshold: - issue_level = IssueLevel.MAJOR - elif relative_delta > self.issue_level_threshold: - issue_level = IssueLevel.MEDIUM - elif self.metric_direction == "better_higher": - if relative_delta < -(self.issue_level_threshold + self.deviation_threshold): - issue_level = IssueLevel.MAJOR - elif relative_delta < -self.issue_level_threshold: - issue_level = IssueLevel.MEDIUM - - return ScanResult( - name=name, - metric_name=metric_name, - metric_value=metric_value, - metric_reference_value=metric_reference_value, - issue_level=issue_level, - slice_size=size_data, - filename_examples=filename_examples, - relative_delta=relative_delta, - issue_group=issue_group, - ) diff --git a/giskard_vision/core/detectors/metrics.py b/giskard_vision/core/detectors/metrics.py deleted file mode 100644 index 744ee1f7..00000000 --- a/giskard_vision/core/detectors/metrics.py +++ /dev/null @@ -1,9 +0,0 @@ -from giskard_vision.image_classification.tests.performance import Accuracy -from giskard_vision.landmark_detection.tests.performance import NMEMean -from giskard_vision.object_detection.tests.performance import IoUMean - -detector_metrics = { - "image_classification": Accuracy, - "landmark": NMEMean, - "object_detection": IoUMean, -} diff --git a/giskard_vision/core/detectors/perturbation.py b/giskard_vision/core/detectors/perturbation.py index 61a767f8..e6a33811 100644 --- a/giskard_vision/core/detectors/perturbation.py +++ b/giskard_vision/core/detectors/perturbation.py @@ -1,25 +1,15 @@ import os from abc import abstractmethod +from importlib import import_module from pathlib import Path -from typing import Any, Sequence +from typing import Any, Sequence, Tuple import cv2 from giskard_vision.core.dataloaders.wrappers import FilteredDataLoader -from giskard_vision.core.detectors.base import ( - DetectorVisionBase, - IssueGroup, - ScanResult, -) +from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult +from giskard_vision.core.issues import Robustness from giskard_vision.core.tests.base import TestDiffBase -from giskard_vision.utils.errors import GiskardImportError - -from .metrics import detector_metrics - -Robustness = IssueGroup( - "Robustness", - description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", -) class PerturbationBaseDetector(DetectorVisionBase): @@ -40,6 +30,28 @@ class PerturbationBaseDetector(DetectorVisionBase): issue_group = Robustness + def run( + self, + model: Any, + dataset: Any, + features: Any | None = None, + issue_levels: Tuple[Any] = None, + embed: bool = True, + num_images: int = 0, + ) -> Sequence[Any]: + module = import_module(f"giskard_vision.{model.model_type}.detectors.specs") + DetectorSpecs = getattr(module, "DetectorSpecs") + + if DetectorSpecs: + # Only set attributes that are not part of Python's special attributes (those starting with __) + for attr_name, attr_value in vars(DetectorSpecs).items(): + if not attr_name.startswith("__") and hasattr(self, attr_name): + setattr(self, attr_name, attr_value) + else: + raise ValueError(f"No detector specifications found for model type: {model.model_type}") + + return super().run(model, dataset, features, issue_levels, embed, num_images) + @abstractmethod def get_dataloaders(self, dataset: Any) -> Sequence[Any]: ... @@ -48,7 +60,7 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: results = [] for dl in dataloaders: - test_result = TestDiffBase(metric=detector_metrics[model.model_type], threshold=1).run( + test_result = TestDiffBase(metric=self.metric, threshold=1).run( model=model, dataloader=dl, dataloader_ref=dataset, @@ -63,40 +75,22 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: if isinstance(dl, FilteredDataLoader): filename_example_dataloader_ref = str(Path() / "examples_images" / f"{dataset.name}_{index_worst}.png") - cv2.imwrite( - filename_example_dataloader_ref, cv2.resize(dataset[index_worst][0][0], (0, 0), fx=0.3, fy=0.3) - ) + cv2.imwrite(filename_example_dataloader_ref, dataset[index_worst][0][0]) filename_examples.append(filename_example_dataloader_ref) filename_example_dataloader = str(Path() / "examples_images" / f"{dl.name}_{index_worst}.png") - cv2.imwrite(filename_example_dataloader, cv2.resize(dl[index_worst][0][0], (0, 0), fx=0.3, fy=0.3)) + cv2.imwrite(filename_example_dataloader, dl[index_worst][0][0]) filename_examples.append(filename_example_dataloader) - results.append(self.get_scan_result(test_result, filename_examples, dl.name, len(dl))) + results.append( + self.get_scan_result( + test_result.metric_value_test, + test_result.metric_value_test, + test_result.metric_name, + filename_examples, + dl.name, + len(dl), + issue_group=self.issue_group, + ) + ) return results - - def get_scan_result(self, test_result, filename_examples, name, size_data) -> ScanResult: - try: - from giskard.scanner.issues import IssueLevel - except (ImportError, ModuleNotFoundError) as e: - raise GiskardImportError(["giskard"]) from e - - relative_delta = (test_result.metric_value_test - test_result.metric_value_ref) / test_result.metric_value_ref - - if relative_delta > self.issue_level_threshold + self.deviation_threshold: - issue_level = IssueLevel.MAJOR - elif relative_delta > self.issue_level_threshold: - issue_level = IssueLevel.MEDIUM - else: - issue_level = IssueLevel.MINOR - - return ScanResult( - name=name, - metric_name=test_result.metric_name, - metric_value=test_result.metric_value_test, - metric_reference_value=test_result.metric_value_ref, - issue_level=issue_level, - slice_size=size_data, - filename_examples=filename_examples, - relative_delta=relative_delta, - ) diff --git a/giskard_vision/core/detectors/specs.py b/giskard_vision/core/detectors/specs.py new file mode 100644 index 00000000..3bf01c69 --- /dev/null +++ b/giskard_vision/core/detectors/specs.py @@ -0,0 +1,13 @@ +from giskard_vision.core.issues import IssueGroup +from giskard_vision.image_classification.tests.performance import MetricBase + + +class DetectorSpecsBase: + issue_group: IssueGroup + warning_messages: dict + metric: MetricBase = None + metric_type: str = None + metric_direction: str = None + deviation_threshold: float = 0.10 + issue_level_threshold: float = 0.05 + num_images: int = 0 diff --git a/giskard_vision/core/detectors/transformation_color_detector.py b/giskard_vision/core/detectors/transformation_color_detector.py index 70488ef0..11213bb2 100644 --- a/giskard_vision/core/detectors/transformation_color_detector.py +++ b/giskard_vision/core/detectors/transformation_color_detector.py @@ -1,7 +1,7 @@ from giskard_vision.core.dataloaders.wrappers import ColoredDataLoader from ...core.detectors.decorator import maybe_detector -from .perturbation import PerturbationBaseDetector, Robustness +from .perturbation import PerturbationBaseDetector @maybe_detector("coloring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) @@ -10,8 +10,6 @@ class TransformationColorDetectorLandmark(PerturbationBaseDetector): Detector that evaluates models performance depending on images in grayscale """ - issue_group = Robustness - def get_dataloaders(self, dataset): dl = ColoredDataLoader(dataset) diff --git a/giskard_vision/core/detectors/transformation_noise_detector.py b/giskard_vision/core/detectors/transformation_noise_detector.py index 0ce63b13..8949a139 100644 --- a/giskard_vision/core/detectors/transformation_noise_detector.py +++ b/giskard_vision/core/detectors/transformation_noise_detector.py @@ -4,7 +4,7 @@ from .perturbation import PerturbationBaseDetector -@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) +@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection", "noise"]) class TransformationNoiseDetectorLandmark(PerturbationBaseDetector): """ Detector that evaluates models performance on noisy images diff --git a/giskard_vision/core/issues.py b/giskard_vision/core/issues.py new file mode 100644 index 00000000..951ff3a0 --- /dev/null +++ b/giskard_vision/core/issues.py @@ -0,0 +1,21 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class IssueGroup: + name: str + description: str + + +EthicalIssueMeta = IssueGroup( + "Ethical", + description="The data are filtered by metadata like age, facial hair, or gender to detect ethical biases.", +) +PerformanceIssueMeta = IssueGroup( + "Performance", + description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.", +) +Robustness = IssueGroup( + "Robustness", + description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", +) diff --git a/giskard_vision/image_classification/dataloaders/loaders.py b/giskard_vision/image_classification/dataloaders/loaders.py index ba22aa3c..6057c9c7 100644 --- a/giskard_vision/image_classification/dataloaders/loaders.py +++ b/giskard_vision/image_classification/dataloaders/loaders.py @@ -2,11 +2,11 @@ import numpy as np -from giskard_vision.core.dataloaders.base import EthicalIssueMeta, PerformanceIssueMeta from giskard_vision.core.dataloaders.hf import HFDataLoader from giskard_vision.core.dataloaders.meta import MetaData from giskard_vision.core.dataloaders.tfds import DataLoaderTensorFlowDatasets from giskard_vision.core.dataloaders.utils import flatten_dict +from giskard_vision.core.issues import EthicalIssueMeta, PerformanceIssueMeta from giskard_vision.image_classification.types import Types diff --git a/giskard_vision/image_classification/detectors/metadata_detector.py b/giskard_vision/image_classification/detectors/metadata_detector.py index 42a876da..4e567b24 100644 --- a/giskard_vision/image_classification/detectors/metadata_detector.py +++ b/giskard_vision/image_classification/detectors/metadata_detector.py @@ -1,14 +1,9 @@ from giskard_vision.core.detectors.metadata_scan_detector import MetaDataScanDetector -from giskard_vision.image_classification.tests.performance import Accuracy from ...core.detectors.decorator import maybe_detector +from .specs import DetectorSpecs @maybe_detector("metadata_classification", tags=["vision", "image_classification", "metadata"]) -class MetaDataScanDetectorClassification(MetaDataScanDetector): - metric = Accuracy - type_task = "classification" - metric_type = "absolute" - metric_direction = "better_higher" - deviation_threshold = 0.10 - issue_level_threshold = 0.05 +class MetaDataScanDetectorClassification(DetectorSpecs, MetaDataScanDetector): + pass diff --git a/giskard_vision/image_classification/detectors/specs.py b/giskard_vision/image_classification/detectors/specs.py new file mode 100644 index 00000000..109ba5a1 --- /dev/null +++ b/giskard_vision/image_classification/detectors/specs.py @@ -0,0 +1,11 @@ +from giskard_vision.core.detectors.specs import DetectorSpecsBase +from giskard_vision.image_classification.tests.performance import Accuracy, MetricBase + + +class DetectorSpecs(DetectorSpecsBase): + metric: MetricBase = Accuracy + type_task: str = "classification" + metric_type: str = "absolute" + metric_direction: str = "better_higher" + deviation_threshold: float = 0.10 + issue_level_threshold: float = 0.05 diff --git a/giskard_vision/landmark_detection/dataloaders/loaders.py b/giskard_vision/landmark_detection/dataloaders/loaders.py index c73e335f..8cb7de77 100644 --- a/giskard_vision/landmark_detection/dataloaders/loaders.py +++ b/giskard_vision/landmark_detection/dataloaders/loaders.py @@ -8,20 +8,11 @@ from giskard_vision.core.dataloaders.meta import MetaData from giskard_vision.core.dataloaders.tfds import DataLoaderTensorFlowDatasets from giskard_vision.core.dataloaders.utils import flatten_dict -from giskard_vision.core.detectors.base import IssueGroup +from giskard_vision.core.issues import EthicalIssueMeta, PerformanceIssueMeta from giskard_vision.landmark_detection.types import Types from .base import DataLoaderBase -EthicalIssueMeta = IssueGroup( - "Ethical", - description="The data are filtered by metadata like age, facial hair, or gender to detect ethical biases.", -) -PerformanceIssueMeta = IssueGroup( - "Performance", - description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.", -) - class DataLoader300W(DataLoaderBase): """Data loader for the 300W dataset. Ref: https://ibug.doc.ic.ac.uk/resources/300-W/ diff --git a/giskard_vision/landmark_detection/detectors/base.py b/giskard_vision/landmark_detection/detectors/base.py index a1b1b8d7..d94d2560 100644 --- a/giskard_vision/landmark_detection/detectors/base.py +++ b/giskard_vision/landmark_detection/detectors/base.py @@ -6,34 +6,11 @@ import cv2 from giskard_vision.core.dataloaders.wrappers import FilteredDataLoader -from giskard_vision.core.detectors.base import ( - DetectorVisionBase, - IssueGroup, - ScanResult, -) +from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult from giskard_vision.landmark_detection.tests.base import TestDiff from giskard_vision.landmark_detection.tests.performance import NMEMean from giskard_vision.utils.errors import GiskardImportError -Cropping = IssueGroup( - "Cropping", description="Cropping involves evaluating the landmark detection model on specific face areas." -) - -Ethical = IssueGroup( - "Ethical", - description="The data are filtered by ethnicity to detect ethical biases in the landmark detection model.", -) - -Pose = IssueGroup( - "Head Pose", - description="The data are filtered by head pose to detect biases in the landmark detection model.", -) - -Robustness = IssueGroup( - "Robustness", - description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", -) - class LandmarkDetectionBaseDetector(DetectorVisionBase): """ diff --git a/giskard_vision/landmark_detection/detectors/cropping_detector.py b/giskard_vision/landmark_detection/detectors/cropping_detector.py index 96a710e7..959be975 100644 --- a/giskard_vision/landmark_detection/detectors/cropping_detector.py +++ b/giskard_vision/landmark_detection/detectors/cropping_detector.py @@ -1,7 +1,9 @@ +from giskard_vision.core.scanner.issues import Robustness + from ...core.detectors.decorator import maybe_detector from ..dataloaders.wrappers import CroppedDataLoader from ..marks.facial_parts import FacialParts -from .base import Cropping, LandmarkDetectionBaseDetector +from .base import LandmarkDetectionBaseDetector @maybe_detector("cropping_landmark", tags=["vision", "face", "landmark", "transformed", "cropped"]) @@ -10,7 +12,7 @@ class CroppingDetectorLandmark(LandmarkDetectionBaseDetector): Detector that evaluates models performance relative to a facial part """ - issue_group = Cropping + issue_group = Robustness def get_dataloaders(self, dataset): facial_parts = [elt.value for elt in FacialParts] diff --git a/giskard_vision/landmark_detection/detectors/metadata_detector.py b/giskard_vision/landmark_detection/detectors/metadata_detector.py index 327ca570..6e182239 100644 --- a/giskard_vision/landmark_detection/detectors/metadata_detector.py +++ b/giskard_vision/landmark_detection/detectors/metadata_detector.py @@ -3,14 +3,11 @@ SurrogateNME, SurrogateVolumeConvexHull, ) -from giskard_vision.landmark_detection.tests.performance import NMEMean from ...core.detectors.decorator import maybe_detector +from .specs import DetectorSpecs @maybe_detector("metadata_landmark", tags=["vision", "face", "landmark", "metadata"]) -class MetaDataScanDetectorLandmark(MetaDataScanDetector): +class MetaDataScanDetectorLandmark(DetectorSpecs, MetaDataScanDetector): surrogates = [SurrogateVolumeConvexHull, SurrogateNME] - metric = NMEMean - type_task = "regression" - metric_type = "relative" diff --git a/giskard_vision/landmark_detection/detectors/specs.py b/giskard_vision/landmark_detection/detectors/specs.py new file mode 100644 index 00000000..9e8afa80 --- /dev/null +++ b/giskard_vision/landmark_detection/detectors/specs.py @@ -0,0 +1,8 @@ +from giskard_vision.core.detectors.specs import DetectorSpecsBase +from giskard_vision.landmark_detection.tests.performance import NMEMean + + +class DetectorSpecs(DetectorSpecsBase): + metric = NMEMean + type_task = "regression" + metric_type = "relative" diff --git a/giskard_vision/landmark_detection/detectors/transformation_blurring_detector.py b/giskard_vision/landmark_detection/detectors/transformation_blurring_detector.py deleted file mode 100644 index 7e811089..00000000 --- a/giskard_vision/landmark_detection/detectors/transformation_blurring_detector.py +++ /dev/null @@ -1,24 +0,0 @@ -from giskard_vision.core.dataloaders.wrappers import BlurredDataLoader - -from ...core.detectors.decorator import maybe_detector -from .base import LandmarkDetectionBaseDetector, Robustness - - -@maybe_detector("blurring_landmark", tags=["vision", "face", "landmark", "transformed", "blurred"]) -class TransformationBlurringDetectorLandmark(LandmarkDetectionBaseDetector): - """ - Detector that evaluates models performance on blurred images - """ - - issue_group = Robustness - - def __init__(self, kernel_size=(11, 11), sigma=(3, 3)): - self.kernel_size = kernel_size - self.sigma = sigma - - def get_dataloaders(self, dataset): - dl = BlurredDataLoader(dataset, self.kernel_size, self.sigma) - - dls = [dl] - - return dls diff --git a/giskard_vision/landmark_detection/detectors/transformation_color_detector.py b/giskard_vision/landmark_detection/detectors/transformation_color_detector.py deleted file mode 100644 index f4559df5..00000000 --- a/giskard_vision/landmark_detection/detectors/transformation_color_detector.py +++ /dev/null @@ -1,20 +0,0 @@ -from giskard_vision.core.dataloaders.wrappers import ColoredDataLoader - -from ...core.detectors.decorator import maybe_detector -from .base import LandmarkDetectionBaseDetector, Robustness - - -@maybe_detector("color_landmark", tags=["vision", "face", "landmark", "filtered", "colored"]) -class TransformationColorDetectorLandmark(LandmarkDetectionBaseDetector): - """ - Detector that evaluates models performance depending on images in grayscale - """ - - issue_group = Robustness - - def get_dataloaders(self, dataset): - dl = ColoredDataLoader(dataset) - - dls = [dl] - - return dls diff --git a/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py b/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py index 68950fe1..f017df22 100644 --- a/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py +++ b/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py @@ -1,7 +1,8 @@ +from giskard_vision.core.issues import Robustness from giskard_vision.landmark_detection.dataloaders.wrappers import ResizedDataLoader from ...core.detectors.decorator import maybe_detector -from .base import LandmarkDetectionBaseDetector, Robustness +from .base import LandmarkDetectionBaseDetector @maybe_detector("resize_landmark", tags=["vision", "face", "landmark", "transformed", "resized"]) diff --git a/giskard_vision/landmark_detection/models/base.py b/giskard_vision/landmark_detection/models/base.py index 95047c20..738427e3 100644 --- a/giskard_vision/landmark_detection/models/base.py +++ b/giskard_vision/landmark_detection/models/base.py @@ -13,7 +13,7 @@ class FaceLandmarksModelBase(ModelBase): """Abstract class that serves as a template for all landmark model predictions""" - model_type = "landmark" + model_type = "landmark_detection" prediction_result_cls = Types.prediction_result def __init__(self, n_landmarks: int, n_dimensions: int, name: Optional[str] = None) -> None: diff --git a/giskard_vision/object_detection/detectors/metadata_detector.py b/giskard_vision/object_detection/detectors/metadata_detector.py index 74844b98..b1b740f2 100644 --- a/giskard_vision/object_detection/detectors/metadata_detector.py +++ b/giskard_vision/object_detection/detectors/metadata_detector.py @@ -15,13 +15,13 @@ SurrogateRelativeTopLeftY, SurrogateStdIntensity, ) -from giskard_vision.object_detection.tests.performance import IoUMean from ...core.detectors.decorator import maybe_detector +from .specs import DetectorSpecs @maybe_detector("metadata_object_detection", tags=["vision", "object_detection", "metadata"]) -class MetaDataScanDetectorObjectDetection(MetaDataScanDetector): +class MetaDataScanDetectorObjectDetection(DetectorSpecs, MetaDataScanDetector): surrogates = [ SurrogateCenterMassX, SurrogateCenterMassY, @@ -38,9 +38,3 @@ class MetaDataScanDetectorObjectDetection(MetaDataScanDetector): SurrogateRelativeTopLeftY, SurrogateNormalizedPerimeter, ] - metric = IoUMean - type_task = "regression" - metric_type = "absolute" - metric_direction = "better_higher" - deviation_threshold = 0.10 - issue_level_threshold = 0.05 diff --git a/giskard_vision/object_detection/detectors/specs.py b/giskard_vision/object_detection/detectors/specs.py new file mode 100644 index 00000000..2f13483a --- /dev/null +++ b/giskard_vision/object_detection/detectors/specs.py @@ -0,0 +1,11 @@ +from giskard_vision.core.detectors.specs import DetectorSpecsBase +from giskard_vision.object_detection.tests.performance import IoUMean + + +class DetectorSpecs(DetectorSpecsBase): + metric = IoUMean + type_task = "regression" + metric_type = "absolute" + metric_direction = "better_higher" + deviation_threshold = 0.10 + issue_level_threshold = 0.05 From fe26272e1b6ea3b131ef4751ab50dfc05f6b9379 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Tue, 13 Aug 2024 12:52:05 +0200 Subject: [PATCH 12/17] small updates --- giskard_vision/landmark_detection/detectors/__init__.py | 4 ---- .../landmark_detection/detectors/cropping_detector.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/giskard_vision/landmark_detection/detectors/__init__.py b/giskard_vision/landmark_detection/detectors/__init__.py index e9726a55..d90f809e 100644 --- a/giskard_vision/landmark_detection/detectors/__init__.py +++ b/giskard_vision/landmark_detection/detectors/__init__.py @@ -1,13 +1,9 @@ from .cropping_detector import CroppingDetectorLandmark from .metadata_detector import MetaDataScanDetectorLandmark -from .transformation_blurring_detector import TransformationBlurringDetectorLandmark -from .transformation_color_detector import TransformationColorDetectorLandmark from .transformation_resize_detector import TransformationResizeDetectorLandmark __all__ = [ "CroppingDetectorLandmark", - "TransformationBlurringDetectorLandmark", - "TransformationColorDetectorLandmark", "TransformationResizeDetectorLandmark", "MetaDataScanDetectorLandmark", ] diff --git a/giskard_vision/landmark_detection/detectors/cropping_detector.py b/giskard_vision/landmark_detection/detectors/cropping_detector.py index 959be975..8689374f 100644 --- a/giskard_vision/landmark_detection/detectors/cropping_detector.py +++ b/giskard_vision/landmark_detection/detectors/cropping_detector.py @@ -1,4 +1,4 @@ -from giskard_vision.core.scanner.issues import Robustness +from giskard_vision.core.issues import Robustness from ...core.detectors.decorator import maybe_detector from ..dataloaders.wrappers import CroppedDataLoader From c359c9c1586440f60ec1c0d902f6eaf5758078aa Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Tue, 13 Aug 2024 13:04:40 +0200 Subject: [PATCH 13/17] refactored spec setting --- giskard_vision/core/detectors/perturbation.py | 19 +++++-------------- .../detectors/test_detectors.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/giskard_vision/core/detectors/perturbation.py b/giskard_vision/core/detectors/perturbation.py index e6a33811..cb5c0c26 100644 --- a/giskard_vision/core/detectors/perturbation.py +++ b/giskard_vision/core/detectors/perturbation.py @@ -2,7 +2,7 @@ from abc import abstractmethod from importlib import import_module from pathlib import Path -from typing import Any, Sequence, Tuple +from typing import Any, Sequence import cv2 @@ -30,16 +30,8 @@ class PerturbationBaseDetector(DetectorVisionBase): issue_group = Robustness - def run( - self, - model: Any, - dataset: Any, - features: Any | None = None, - issue_levels: Tuple[Any] = None, - embed: bool = True, - num_images: int = 0, - ) -> Sequence[Any]: - module = import_module(f"giskard_vision.{model.model_type}.detectors.specs") + def set_specs_from_model_type(self, model_type): + module = import_module(f"giskard_vision.{model_type}.detectors.specs") DetectorSpecs = getattr(module, "DetectorSpecs") if DetectorSpecs: @@ -48,14 +40,13 @@ def run( if not attr_name.startswith("__") and hasattr(self, attr_name): setattr(self, attr_name, attr_value) else: - raise ValueError(f"No detector specifications found for model type: {model.model_type}") - - return super().run(model, dataset, features, issue_levels, embed, num_images) + raise ValueError(f"No detector specifications found for model type: {model_type}") @abstractmethod def get_dataloaders(self, dataset: Any) -> Sequence[Any]: ... def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: + self.set_specs_from_model_type(model.model_type) dataloaders = self.get_dataloaders(dataset) results = [] diff --git a/tests/landmark_detection/detectors/test_detectors.py b/tests/landmark_detection/detectors/test_detectors.py index f812e740..7170b7a0 100644 --- a/tests/landmark_detection/detectors/test_detectors.py +++ b/tests/landmark_detection/detectors/test_detectors.py @@ -1,11 +1,18 @@ from giskard.scanner.issues import Issue, IssueLevel from pytest import mark +from giskard_vision.core.detectors.transformation_blurring_detector import ( + TransformationBlurringDetectorLandmark, +) +from giskard_vision.core.detectors.transformation_color_detector import ( + TransformationColorDetectorLandmark, +) +from giskard_vision.core.detectors.transformation_noise_detector import ( + TransformationNoiseDetectorLandmark, +) from giskard_vision.landmark_detection.detectors import ( CroppingDetectorLandmark, MetaDataScanDetectorLandmark, - TransformationBlurringDetectorLandmark, - TransformationColorDetectorLandmark, TransformationResizeDetectorLandmark, ) from giskard_vision.landmark_detection.detectors.base import ScanResult @@ -17,6 +24,7 @@ CroppingDetectorLandmark, TransformationBlurringDetectorLandmark, TransformationColorDetectorLandmark, + TransformationNoiseDetectorLandmark, TransformationResizeDetectorLandmark, ], ) From 6dca401df611d8683a0e0a6aa90139e89989412e Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Tue, 13 Aug 2024 13:12:44 +0200 Subject: [PATCH 14/17] fixed import in object_detection dataloader --- giskard_vision/object_detection/dataloaders/loaders.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/giskard_vision/object_detection/dataloaders/loaders.py b/giskard_vision/object_detection/dataloaders/loaders.py index b2e91239..32d70a8c 100644 --- a/giskard_vision/object_detection/dataloaders/loaders.py +++ b/giskard_vision/object_detection/dataloaders/loaders.py @@ -8,9 +8,10 @@ import pandas as pd from numpy import ndarray -from giskard_vision.core.dataloaders.base import DataIteratorBase, PerformanceIssueMeta +from giskard_vision.core.dataloaders.base import DataIteratorBase from giskard_vision.core.dataloaders.hf import HFDataLoader from giskard_vision.core.dataloaders.meta import MetaData +from giskard_vision.core.issues import PerformanceIssueMeta from giskard_vision.landmark_detection.dataloaders.loaders import ( DataLoader300W, DataLoaderFFHQ, From 99d98ddd8d859c246c8ae009d1db919192f21d1e Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Tue, 13 Aug 2024 13:14:54 +0200 Subject: [PATCH 15/17] renaming pert detectors --- .../detectors/transformation_blurring_detector.py | 2 +- .../core/detectors/transformation_color_detector.py | 2 +- .../core/detectors/transformation_noise_detector.py | 2 +- tests/landmark_detection/detectors/test_detectors.py | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/giskard_vision/core/detectors/transformation_blurring_detector.py b/giskard_vision/core/detectors/transformation_blurring_detector.py index be1b912c..05f21e1c 100644 --- a/giskard_vision/core/detectors/transformation_blurring_detector.py +++ b/giskard_vision/core/detectors/transformation_blurring_detector.py @@ -5,7 +5,7 @@ @maybe_detector("blurring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) -class TransformationBlurringDetectorLandmark(PerturbationBaseDetector): +class TransformationBlurringDetector(PerturbationBaseDetector): """ Detector that evaluates models performance on blurred images """ diff --git a/giskard_vision/core/detectors/transformation_color_detector.py b/giskard_vision/core/detectors/transformation_color_detector.py index 11213bb2..6b23e9c3 100644 --- a/giskard_vision/core/detectors/transformation_color_detector.py +++ b/giskard_vision/core/detectors/transformation_color_detector.py @@ -5,7 +5,7 @@ @maybe_detector("coloring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) -class TransformationColorDetectorLandmark(PerturbationBaseDetector): +class TransformationColorDetector(PerturbationBaseDetector): """ Detector that evaluates models performance depending on images in grayscale """ diff --git a/giskard_vision/core/detectors/transformation_noise_detector.py b/giskard_vision/core/detectors/transformation_noise_detector.py index 8949a139..11f8f2ba 100644 --- a/giskard_vision/core/detectors/transformation_noise_detector.py +++ b/giskard_vision/core/detectors/transformation_noise_detector.py @@ -5,7 +5,7 @@ @maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection", "noise"]) -class TransformationNoiseDetectorLandmark(PerturbationBaseDetector): +class TransformationNoiseDetector(PerturbationBaseDetector): """ Detector that evaluates models performance on noisy images """ diff --git a/tests/landmark_detection/detectors/test_detectors.py b/tests/landmark_detection/detectors/test_detectors.py index 7170b7a0..1e893809 100644 --- a/tests/landmark_detection/detectors/test_detectors.py +++ b/tests/landmark_detection/detectors/test_detectors.py @@ -2,13 +2,13 @@ from pytest import mark from giskard_vision.core.detectors.transformation_blurring_detector import ( - TransformationBlurringDetectorLandmark, + TransformationBlurringDetector, ) from giskard_vision.core.detectors.transformation_color_detector import ( - TransformationColorDetectorLandmark, + TransformationColorDetector, ) from giskard_vision.core.detectors.transformation_noise_detector import ( - TransformationNoiseDetectorLandmark, + TransformationNoiseDetector, ) from giskard_vision.landmark_detection.detectors import ( CroppingDetectorLandmark, @@ -22,9 +22,9 @@ "detector", [ CroppingDetectorLandmark, - TransformationBlurringDetectorLandmark, - TransformationColorDetectorLandmark, - TransformationNoiseDetectorLandmark, + TransformationBlurringDetector, + TransformationColorDetector, + TransformationNoiseDetector, TransformationResizeDetectorLandmark, ], ) From 182731c185debe78a9f70ff0f687b268f1eee903 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Tue, 13 Aug 2024 13:31:55 +0200 Subject: [PATCH 16/17] fixed import --- giskard_vision/core/dataloaders/base.py | 1 + giskard_vision/core/dataloaders/hf.py | 3 ++- giskard_vision/core/detectors/metadata_scan_detector.py | 3 +-- giskard_vision/core/issues.py | 4 ++++ giskard_vision/landmark_detection/dataloaders/loaders.py | 1 - 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/giskard_vision/core/dataloaders/base.py b/giskard_vision/core/dataloaders/base.py index 35f6971f..73f9dfaf 100644 --- a/giskard_vision/core/dataloaders/base.py +++ b/giskard_vision/core/dataloaders/base.py @@ -12,6 +12,7 @@ get_image_channel_number, get_image_size, ) +from giskard_vision.core.issues import AttributesIssueMeta from ..types import TypesBase diff --git a/giskard_vision/core/dataloaders/hf.py b/giskard_vision/core/dataloaders/hf.py index 10100319..60dfc274 100644 --- a/giskard_vision/core/dataloaders/hf.py +++ b/giskard_vision/core/dataloaders/hf.py @@ -7,8 +7,9 @@ from PIL.Image import Image as PILImage -from giskard_vision.core.dataloaders.base import AttributesIssueMeta, DataIteratorBase +from giskard_vision.core.dataloaders.base import DataIteratorBase from giskard_vision.core.dataloaders.meta import MetaData, get_pil_image_depth +from giskard_vision.core.issues import AttributesIssueMeta from giskard_vision.utils.errors import GiskardError, GiskardImportError diff --git a/giskard_vision/core/detectors/metadata_scan_detector.py b/giskard_vision/core/detectors/metadata_scan_detector.py index 7443c652..83fcab7e 100644 --- a/giskard_vision/core/detectors/metadata_scan_detector.py +++ b/giskard_vision/core/detectors/metadata_scan_detector.py @@ -5,8 +5,7 @@ import pandas as pd from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult -from giskard_vision.core.issues import IssueGroup, PerformanceIssueMeta - +from giskard_vision.core.issues import PerformanceIssueMeta from giskard_vision.core.tests.base import MetricBase from giskard_vision.utils.errors import GiskardImportError diff --git a/giskard_vision/core/issues.py b/giskard_vision/core/issues.py index 951ff3a0..9e4cebc7 100644 --- a/giskard_vision/core/issues.py +++ b/giskard_vision/core/issues.py @@ -15,6 +15,10 @@ class IssueGroup: "Performance", description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.", ) +AttributesIssueMeta = IssueGroup( + "Attributes", + description="The data are filtered by the image attributes like width, height, or brightness value to detect issues.", +) Robustness = IssueGroup( "Robustness", description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", diff --git a/giskard_vision/landmark_detection/dataloaders/loaders.py b/giskard_vision/landmark_detection/dataloaders/loaders.py index 1ce1d089..24060a44 100644 --- a/giskard_vision/landmark_detection/dataloaders/loaders.py +++ b/giskard_vision/landmark_detection/dataloaders/loaders.py @@ -9,7 +9,6 @@ from giskard_vision.core.dataloaders.tfds import DataLoaderTensorFlowDatasets from giskard_vision.core.dataloaders.utils import flatten_dict from giskard_vision.core.issues import EthicalIssueMeta, PerformanceIssueMeta - from giskard_vision.landmark_detection.types import Types from .base import DataLoaderBase From 6ba199451c2a72abd9d7d9d088271611508ad4e8 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Tue, 13 Aug 2024 13:33:18 +0200 Subject: [PATCH 17/17] fixing get_scan_results args --- giskard_vision/core/detectors/perturbation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giskard_vision/core/detectors/perturbation.py b/giskard_vision/core/detectors/perturbation.py index cb5c0c26..91edbbe7 100644 --- a/giskard_vision/core/detectors/perturbation.py +++ b/giskard_vision/core/detectors/perturbation.py @@ -75,7 +75,7 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: results.append( self.get_scan_result( test_result.metric_value_test, - test_result.metric_value_test, + test_result.metric_value_ref, test_result.metric_name, filename_examples, dl.name,