EMIFF
本文提出了一种新的基于摄像机的三维检测框架，增强型多尺度图像特征融合（EMIFF）。虽然EMIFF的输入是2D图像，但是它的neck层的结构设计应该普适于点云的3D目标检测，同时其中的MFC等模块可以简单地被替换成更先进的其他组件。
为了充分利用车辆和基础设施的整体视角，本文提出了多尺度交叉注意MCA（包含了MFC和MFS）和相机感知通道掩蔽CCM模块，以在尺度、空间和通道（MFC尺度级增强、MFS空间级增强、CCM通道级增强）级别增强基础设施和车辆特征，从而纠正相机异步引入的姿态误差。我们还引入了一个特征压缩FC模块，该模块具有信道和空间压缩块，以提高传输效率。
MFC
MFC模块首先应用于多尺度特征。由于姿态误差会导致2D平面上投影位置和地面真实位置之间的位移，我们对每个比例特征应用DCN，以允许每个像素获得其周围的空间信息。然后，通过UpConv块将不同尺度的特征上采样到相同的尺寸。
class double_conv(nn.Module):
def __init__(self, in_ch, out_ch):
super(double_conv, self).__init__()
self.conv = nn.Sequential(
nn.Conv2d(in_ch, out_ch, 3, stride=1, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(),
nn.Conv2d(out_ch, out_ch, 3, stride=1, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU()
)
def forward(self, x):
x = self.conv(x)
return x
class DCN_Up_Conv_List(nn.Module):
def __init__(self, neck_dcn, channels):
super(DCN_Up_Conv_List, self).__init__()
self.upconv0 = nn.Sequential(
double_conv(channels,channels),
)
self.upconv1 = nn.Sequential(
nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
double_conv(channels,channels),
)
self.upconv2 = nn.Sequential(
nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
double_conv(channels,channels),
nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
double_conv(channels,channels),
)
self.upconv3 = nn.Sequential(
nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
double_conv(channels,channels),
nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
double_conv(channels,channels),
nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
double_conv(channels,channels),
)
self.dcn0 = build_neck(neck_dcn)
self.dcn1 = build_neck(neck_dcn)
self.dcn2 = build_neck(neck_dcn)
self.dcn3 = build_neck(neck_dcn)
def forward(self, x):
assert x.__len__() == 4
x0 = self.dcn0(x[0])
x0 = self.upconv0(x0)
x1 = self.dcn1(x[1])
x1 = self.upconv1(x1)
x2 = self.dcn2(x[2])
x2 = self.upconv2(x2)
x3 = self.dcn3(x[3])
x3 = self.upconv3(x3)
return [x0,x1,x2,x3]
MFS
MFS应用MeanPooling操作获得不同尺度的基础设施特征的表示，而不同尺度的车辆特征首先通过mean操作融合，然后通过MeanPooling进行细化。为了寻找不同尺度下车辆特征和基础设施特征之间的相关性，交叉注意应用于基础设施表示作为关键，车辆表示作为查询，生成每个尺度m的注意权重ω m inf。我们计算特征^fM inf和权重ω m inf之间的乘积。MCA的最终输出是增强的基础设施图像特征finf和车辆图像特征fveh。
def attention(query, key, mask=None, dropout=None):
# from IPython import embed
# embed()
"Compute 'Scaled Dot Product Attention'"
d_k = query.size(-1)
scores = torch.matmul(query, key.transpose(-2, -1)) \
/ math.sqrt(d_k)
if mask is not None:
scores = scores.masked_fill(mask == 0, -1e9)
p_attn = F.softmax(scores, dim = -1)
if dropout is not None:
p_attn = dropout(p_attn)
return p_attn
def extract_img_feat(self, img, img_metas):
"""Extract features from images."""
bs = img.shape[0]
img_v = img[:,0,...]
img_i = img[:,1,...]
x_v = self.backbone_v(img_v)
x_v = self.neck_v(x_v)
x_v = self.dcn_up_conv_v(list(x_v))
x_v_tensor = torch.stack(x_v).permute(1,0,2,3,4)
x_v_out = torch.mean(x_v_tensor,dim=1)
x_i = self.backbone_i(img_i)
x_i = self.neck_i(x_i)
# from IPython import embed
# embed(header='compress')
# Add compression encoder-decoder
x_i = self.inf_compressor(x_i)
x_i = self.dcn_up_conv_i(list(x_i))
x_i_tensor = torch.stack(x_i).permute(1,0,2,3,4)
# query.shape[B,C]
# key.shape[B,N_levels,C]
query = torch.mean(x_v_out,dim=(-2,-1))[:,None,:]
key = torch.mean(x_i_tensor,dim=(-2,-1))
weights_i = attention(query,key).squeeze(1)
# print('attention_weights',weights_i)
x_i_out = (weights_i[:,:,None,None,None] * x_i_tensor).sum(dim=1)
return tuple((x_v_out, x_i_out))
CCM
CCM将学习一个通道掩码来衡量通道之间的重要性。由于不同的通道表示不同距离的目标信息，这些信息与相机参数密切相关，因此将相机参数作为先验来增强图像特征是直观的。首先，将摄像机的内、外特性拉伸成一维并进行连接。然后，使用MLP将它们放大到特征的维数C，以生成通道掩模Mveh/inf。最后，Mveh/inf用于在通道方向上重新加权图像特征fveh/inf，并获得结果f’veh/inf。
class CCMNet(nn.Module):
def __init__(self, in_channels, mid_channels, context_channels, reduction_ratio=1):
super(CCMNet, self).__init__()
self.reduce_conv = nn.Sequential(
nn.Conv2d(in_channels,
mid_channels,
kernel_size=3,
stride=1,
padding=1),
nn.BatchNorm2d(mid_channels),
nn.ReLU(inplace=True),
)
self.context_conv = nn.Conv2d(mid_channels,
context_channels,
kernel_size=1,
stride=1,
padding=0)
self.bn = nn.BatchNorm1d(16)
self.context_mlp = Mlp(16, mid_channels, mid_channels)
self.context_se = SE_Inception_Layer(mid_channels,reduction_ratio=reduction_ratio) # NOTE: add camera-aware
# self.context_se = CASELayer(mid_channels,reduction_ratio=8) # NOTE: add camera-aware
def ida_mat_cal(self,img_meta):
img_scale_factor = (img_meta['scale_factor'][:2]
if 'scale_factor' in img_meta.keys() else 1)
img_shape = img_meta['img_shape'][:2]
orig_h, orig_w = img_shape
ida_rot = torch.eye(2)
ida_tran = torch.zeros(2)
ida_rot *= img_scale_factor
# ida_tran -= torch.Tensor(crop[:2])
if 'flip' in img_meta.keys() and img_meta['flip']:
A = torch.Tensor([[-1, 0], [0, 1]])
b = torch.Tensor([orig_w, 0])
ida_rot = A.matmul(ida_rot)
ida_tran = A.matmul(ida_tran) + b
ida_mat = ida_rot.new_zeros(4, 4)
ida_mat[3, 3] = 1
ida_mat[2, 2] = 1
ida_mat[:2, :2] = ida_rot
ida_mat[:2, 3] = ida_tran
return ida_mat
def forward(self, x_v, x_i, img_metas):
# x [bs,num_cams,C,H,W]
bs, C, H, W = x_v.shape
num_cams = 2
x = torch.stack((x_v,x_i),dim=1).reshape(-1, C, H, W)
extrinsic_v_list = list()
extrinsic_i_list = list()
intrinsic_v_list = list()
intrinsic_i_list = list()
for img_meta in img_metas:
extrinsic_v = torch.Tensor(img_meta['lidar2img']['extrinsic'][0])
extrinsic_i = torch.Tensor(img_meta['lidar2img']['extrinsic'][1])
intrinsic_v = torch.Tensor(img_meta['lidar2img']['intrinsic'][0])
intrinsic_i = torch.Tensor(img_meta['lidar2img']['intrinsic'][1])
# from IPython import embed
# embed(header='ida')
ida_mat = self.ida_mat_cal(img_meta)
intrinsic_v = ida_mat @ intrinsic_v
intrinsic_i = ida_mat @ intrinsic_i
extrinsic_v_list.append(extrinsic_v)
extrinsic_i_list.append(extrinsic_i)
intrinsic_v_list.append(intrinsic_v)
intrinsic_i_list.append(intrinsic_i)
extrinsic_v = torch.stack(extrinsic_v_list)
extrinsic_i = torch.stack(extrinsic_i_list)
intrinsic_v = torch.stack(intrinsic_v_list)
intrinsic_i = torch.stack(intrinsic_i_list)
extrinsic = torch.stack((extrinsic_v,extrinsic_i),dim=1)
intrinsic = torch.stack((intrinsic_v,intrinsic_i),dim=1)
in_mlp = torch.stack(
(
intrinsic[..., 0, 0],
intrinsic[..., 1, 1],
intrinsic[..., 0, 2],
intrinsic[ ..., 1, 2],
),
dim=-1
)
# from IPython import embed
# embed(header='DCMNet')
ex_mlp = extrinsic[...,:3,:].view(bs,num_cams,-1)
mlp_input = torch.cat((in_mlp,ex_mlp),dim=-1)
mlp_input = mlp_input.reshape(-1,mlp_input.shape[-1]).to(x.device)
mlp_input = self.bn(mlp_input)
x = self.reduce_conv(x)
# context_se = self.context_mlp(mlp_input)[..., None, None]
context_se = self.context_mlp(mlp_input)
context = self.context_se(x, context_se)
context = self.context_conv(context)
context = context.reshape(bs,num_cams,C,H,W)
x_v_out = context[:,0,...]
x_i_out = context[:,1,...]
# from IPython import embed
# embed(header='DCMNet end')
return tuple((x_v_out, x_i_out))