import numpy as np
from evaluator.evaluate import Evaluate
[docs]
class Plausibility(Evaluate):
def __init__(self, model, processor, target_layers, targets, bbox_func, explainability_method="cam", cam_method='gradcamelementwise', discard_ratio=0.9, head_fusion="max"):
super(Plausibility, self).__init__(model, processor, target_layers, targets, explainability_method=explainability_method, cam_method=cam_method, discard_ratio=discard_ratio, head_fusion=head_fusion)
self.bbox_func = bbox_func
def _get_pred_box(self, heatmap):
pred_box = self.bbox_func(heatmap)
return pred_box
[docs]
def calculate_iou(self, box1, box2):
"""
Calculates the Intersection over Union (IoU) of two bounding boxes.
Args:
box1 (tuple or list): A tuple/list of four integers (x1, y1, x2, y2) representing the
coordinates of the first bounding box.
(x1, y1) is the top-left corner, and (x2, y2) is the bottom-right corner.
box2 (tuple or list): A tuple/list of four integers (x1_p, y1_p, x2_p, y2_p) representing the
coordinates of the second bounding box.
(x1_p, y1_p) is the top-left corner, and (x2_p, y2_p) is the bottom-right corner.
Returns:
tuple: A tuple containing:
- iou (float): The Intersection over Union (IoU) value, a float between 0 and 1.
- box1_area (int): The area of the first bounding box.
- box2_area (int): The area of the second bounding box.
"""
x1, y1, x2, y2 = box1
x1_p, y1_p, x2_p, y2_p = box2
xi1 = max(x1, x1_p)
yi1 = max(y1, y1_p)
xi2 = min(x2, x2_p)
yi2 = min(y2, y2_p)
inter_width = max(0, xi2 - xi1)
inter_height = max(0, yi2 - yi1)
inter_area = inter_width * inter_height
box1_area = (x2 - x1) * (y2 - y1)
box2_area = (x2_p - x1_p) * (y2_p - y1_p)
union_area = box1_area + box2_area - inter_area
if union_area == 0:
return 0.0
iou = inter_area / union_area
return iou, box1_area, box2_area
[docs]
def threshold_heatmap(self, heatmap, threshold):
"""
Normalizes a numpy array heatmap and retains values above a specified threshold.
Args:
heatmap (np.ndarray): A 2D numpy array representing the heatmap.
threshold (float): A value between 0 and 1. Only normalized values
greater than this threshold will be retained.
Returns:
np.ndarray: A numpy array of the same shape as the input heatmap,
where values below or equal to the threshold are set to 0,
and the remaining values are the normalized original values.
"""
# Normalize the heatmap to the range [0, 1]
min_val = np.min(heatmap)
max_val = np.max(heatmap)
normalized_heatmap = (heatmap - min_val) / (max_val - min_val + 1e-8) # Adding a small epsilon to avoid division by zero
# Apply the threshold
thresholded_heatmap = np.where(normalized_heatmap > threshold, normalized_heatmap, 0)
return thresholded_heatmap
[docs]
def positive_attribution_sum(self, heatmap, ground_truth=None):
"""
Sums the number of non-zero heatmap values within a ground truth bounding box or for the entire heatmap.
Args:
heatmap (np.ndarray): A 2D numpy array representing the heatmap.
ground_truth (tuple or list): A tuple or list of four integers
(x_min, y_min, x_max, y_max) defining
the coordinates of the ground truth
bounding box.
Returns:
int: The number of heatmap values within the ground truth bounding box
that are strictly greater than 0.
"""
if ground_truth:
x_min, y_min, x_max, y_max = ground_truth
# Ensure the bounding box coordinates are within the heatmap boundaries
rows, cols = heatmap.shape
x_min = max(0, x_min)
y_min = max(0, y_min)
x_max = min(cols, x_max)
y_max = min(rows, y_max)
# Extract the region of interest (ROI) from the heatmap
roi = heatmap[y_min:y_max, x_min:x_max]
else:
roi = heatmap
# Count the number of values in the ROI that are greater than 0
sum = np.sum(roi)
detection_count = np.sum(roi > 0)
return sum, detection_count
[docs]
def relevance_mass_accuracy(self, heatmap, ground_truth):
pos_attributions_gt, _ = self.positive_attribution_sum(heatmap, ground_truth)
total_pos_attributions, _ = self.positive_attribution_sum(heatmap)
if total_pos_attributions == 0:
return 0
return pos_attributions_gt / total_pos_attributions
[docs]
def relevance_rank_accuracy(self, heatmap, ground_truth, threshold=0.9):
heatmap = self.threshold_heatmap(heatmap, threshold)
_, pos_attributions_gt = self.positive_attribution_sum(heatmap, ground_truth)
x_min, y_min, x_max, y_max = ground_truth
ground_truth_size = (x_max - x_min) * (y_max - y_min)
return pos_attributions_gt / ground_truth_size
[docs]
def calculate_plausibility_overlap(self, heatmap, ground_truth, cam_image):
pred_box = self._get_pred_box(heatmap)
iou = self.calculate_iou(ground_truth, pred_box)[0]
cam_image_with_boxes = self.draw_boxes(cam_image, [pred_box, ground_truth])
return {
"iou": iou,
"cam_image_with_boxes": cam_image_with_boxes
}
[docs]
def calculate_plausibility_remove_iou(self, image, ground_truth):
input_tensor, image_arr, grayscale_cam, cam_image, label = self._generate_heatmaps(image)
# Heatmap for perturbed image
perturbed_image = self._remove_roi(image, ground_truth)
perturbed_input_tensor, perturbed_image, perturbed_grayscale_cam, perturbed_cam_image, perturbed_label = self._generate_heatmaps(perturbed_image)
pred_box = self._get_pred_box(perturbed_grayscale_cam)
iou = self.calculate_iou(ground_truth, pred_box)[0]
cam_image_with_boxes = self.draw_boxes(perturbed_cam_image, [pred_box, ground_truth])
return {
"iou": iou,
"cam_image_with_boxes": cam_image_with_boxes,
"label": perturbed_label
}
def _get_cam_image_with_bbox(self, cam_image, ground_truth, pred_box):
heatmap_with_bbox = self.draw_boxes(np.array(cam_image), [pred_box, ground_truth])
return heatmap_with_bbox
[docs]
def plausibility(self, image, ground_truth, threshold=0.9):
if ground_truth is None:
return None
input_tensor, image_arr, grayscale_cam, cam_image, label = self._generate_heatmaps(image)
grayscale_cam, cam_image = grayscale_cam[0], cam_image[0]
plausibility_overlap = self.calculate_plausibility_overlap(grayscale_cam, ground_truth, cam_image)
rel_attr_loc = self.relevance_mass_accuracy(grayscale_cam, ground_truth)
rel_rank_acc = self.relevance_rank_accuracy(grayscale_cam, ground_truth)
iou = plausibility_overlap["iou"]
return {
"iou": iou,
"rel_attr_loc": rel_attr_loc,
"rel_rank_acc": rel_rank_acc,
"cam_image": plausibility_overlap["cam_image_with_boxes"]
}
[docs]
def calculate_average_batch_plausibility(self, image_list, ground_truth_list, threshold=0.9):
num_images = len(image_list)
max_overlap_index = 0
max_iou = 0
iou_list = np.zeros((num_images))
rel_attr_list = np.zeros((num_images))
rel_rank_list = np.zeros((num_images))
# accuracy = 0
# accuracy_remove = 0
if not len(image_list) == len(ground_truth_list):
raise AssertionError("Each image should have a ground truth. The images list and ground truth list should have the same shape.")
for i in range(num_images):
image = image_list[i]
ground_truth = ground_truth_list[i]
# if i % 5 == 0:
# print(f"Processing image sample for plausibility: {i+1}/{num_images}")
plausibility = self.plausibility(image, ground_truth, threshold=0.9)
iou_list[i] = plausibility["iou"]
rel_attr_list[i] = plausibility["rel_attr_loc"]
rel_rank_list[i] = plausibility["rel_rank_acc"]
iou = plausibility["iou"]
if iou > max_iou:
max_iou = iou
max_overlap_index = i
return {
"iou_list": iou_list,
"rel_attr_list": rel_attr_list,
"rel_rank_list": rel_rank_list,
"max_iou_index": max_overlap_index,
"max_iou": max_iou,
}