【DL】CAM | 与嵌入的概念相比，图像中有什么相似或不同之处？| 热力图可视化 | python | Adapting pixel attribution methods for embeddin

本文将采用像素属性方法嵌入模型输出（Adapting pixel attribution methods for embedding outputs from models）的实践。话不多说，先看看效果吧！！！

目录

1 安装pytorch-gradcam

2 实践

① 代码

② 效果图

“与嵌入的概念相比，图像中有什么相似或不同之处？”

为了实现这一点，将创建一个目标函数，通过引用嵌入来计算模型输出的相似性与不同。本示例将只使用resnet50 Global Average Pooling层的输出，并删除全连接部分。

# A model wrapper that gets a resnet model and returns the features before the fully connected layer.

class ResnetFeatureExtractor(torch.nn.Module):

    def __init__(self, model):

        super(ResnetFeatureExtractor, self).__init__()

        self.model = model

        self.feature_extractor = torch.nn.Sequential(*list(self.model.children())[:-1])

                

    def __call__(self, x):

        return self.feature_extractor(x)[:, :, 0, 0]

◆目标函数

对于分类网络，这通常是模型类别输出之一，例如,ClassifierOutputTarget(281)。然而，这里的目标是与目标嵌入的相似性（或不相似性），所以将为这个用例实现它。

① 相似

class SimilarityToConceptTarget:

    def __init__(self, features):

        self.features = features

    

    def __call__(self, model_output):

        cos = torch.nn.CosineSimilarity(dim=0)

        return cos(model_output, self.features)

② 不同

class DifferenceFromConceptTarget:

    def __init__(self, features):

        self.features = features

    

    def __call__(self, model_output):

        cos = torch.nn.CosineSimilarity(dim=0)

        return 1 - cos(model_output, self.features)

◆目标层

希望使用模型中具有2D空间输出的哪一层。这通常是网络中最后一个卷积层的输出。

target_layers = [resnet.layer4[-1]]

1 安装pytorch-gradcam

pip install grad-cam==1.4.8 -i https://pypi.tuna.tsinghua.edu.cn/simple

2 实践

① 代码

import warnings
warnings.filterwarnings('ignore')
warnings.simplefilter('ignore')
from torchvision.models.segmentation import deeplabv3_resnet50
import torch
import numpy as np
import cv2
import torchvision
from pytorch_grad_cam.utils.image import show_cam_on_image, preprocess_image
from pytorch_grad_cam import GradCAM
import matplotlib.pyplot as plt
import os


# 这部分根据自己的实际路径去对应
os.environ['QT_QPA_PLATFORM_PLUGIN_PATH'] = '/home/test/anaconda3/envs/yolov11/lib/python3.9/site-packages/PyQt5/Qt5/plugins'

# A model wrapper that gets a resnet model and returns the features before the fully connected layer.
class ResnetFeatureExtractor(torch.nn.Module):
    def __init__(self, model):
        super(ResnetFeatureExtractor, self).__init__()
        self.model = model
        self.feature_extractor = torch.nn.Sequential(*list(self.model.children())[:-1])

    def __call__(self, x):
        return self.feature_extractor(x)[:, :, 0, 0]


def dealImg(imgpath):
    img = cv2.imread(imgpath)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = cv2.resize(img, (512, 512))
    rgb_img_float = np.float32(img) / 255
    input_tensor = preprocess_image(rgb_img_float, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    return img, rgb_img_float, input_tensor


class SimilarityToConceptTarget:
    def __init__(self, features):
        self.features = features

    def __call__(self, model_output):
        cos = torch.nn.CosineSimilarity(dim=0)
        return cos(model_output, self.features)


class DifferenceFromConceptTarget:
    def __init__(self, features):
        self.features = features

    def __call__(self, model_output):
        cos = torch.nn.CosineSimilarity(dim=0)
        return 1 - cos(model_output, self.features)


def RUN(carimg_path, cloudimg_path, testimg_path):
    car_img, car_img_float, car_tensor = dealImg(carimg_path)
    cloud_img, cloud_img_float, cloud_tensor = dealImg(cloudimg_path)
    img, img_float, tensor = dealImg(testimg_path)
    resnet = torchvision.models.resnet50(pretrained=True)
    resnet.eval()
    model = ResnetFeatureExtractor(resnet)

    car_concept_features = model(car_tensor)[0, :]
    cloud_concept_features = model(cloud_tensor)[0, :]

    target_layers = [resnet.layer4[-1]]
    # 相似性
    car_targets = [SimilarityToConceptTarget(car_concept_features)]
    cloud_targets = [SimilarityToConceptTarget(cloud_concept_features)]
    # Where is the car in the image
    with GradCAM(model=model, target_layers=target_layers, use_cuda=False) as cam:
        car_grayscale_cam = cam(input_tensor=tensor, targets=car_targets)[0, :]
    car_cam_image = show_cam_on_image(img_float, car_grayscale_cam, use_rgb=True)
    # Where is the cloud in the image
    with GradCAM(model=model, target_layers=target_layers, use_cuda=False) as cam:
        cloud_grayscale_cam = cam(input_tensor=tensor, targets=cloud_targets)[0, :]
    cloud_cam_image = show_cam_on_image(img_float, cloud_grayscale_cam, use_rgb=True)

    # 非相似
    not_car_targets = [DifferenceFromConceptTarget(car_concept_features)]
    not_cloud_targets = [DifferenceFromConceptTarget(cloud_concept_features)]
    with GradCAM(model=model, target_layers=target_layers, use_cuda=False) as cam:
        not_cloud_grayscale_cam = cam(input_tensor=tensor, targets=not_cloud_targets)[0, :]
    cam_image_notcloud = show_cam_on_image(img_float, not_cloud_grayscale_cam, use_rgb=True)
    # Where is the car not in the image?
    with GradCAM(model=model, target_layers=target_layers,use_cuda=False) as cam:
        not_car_grayscale_cam = cam(input_tensor=tensor,targets=not_car_targets)[0, :]
    cam_image_notcar = show_cam_on_image(img_float, not_car_grayscale_cam, use_rgb=True)

    # display
    result1 = np.hstack((cloud_img, cloud_cam_image, car_img, car_cam_image))
    result2 = np.hstack((cloud_img, cam_image_notcloud, car_img, cam_image_notcar))
    titles = ["SimilarityToConceptTarget", "DifferenceFromConceptTarget"]
    imgs = [result1, result2]
    for i in range(2):
        plt.subplot(2, 1, i + 1), plt.imshow(imgs[i])
        plt.title(titles[i])
    plt.savefig("result.jpg")
    plt.show()


if __name__ == '__main__':
    RUN("car.jpg", "clouds.jpg", "test.jpg")
    pass

② 效果图

至此，本文分享的内容就结束啦。