PeizeSun
commented
3 years ago Hi~
- For 37.2 vs 35.7, we will check it again.
-
Below is the code of multi-head attention in dynamic head. `class RCNNHead(nn.Module):
def init(self, cfg, d_model, num_classes, dim_feedforward=2048, nhead=8, dropout=0.1, activation="relu", scale_clamp: float = _DEFAULT_SCALE_CLAMP, bbox_weights=(2.0, 2.0, 1.0, 1.0)): super().init()
self.d_model = d_model self.attetion = cfg.MODEL.SparseRCNN.ATTENTION # head. self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) if self.attetion: self.inst_interact = nn.MultiheadAttention(d_model, nhead, dropout=dropout) else: self.inst_interact = DynamicConv(cfg) self.linear1 = nn.Linear(d_model, dim_feedforward) self.dropout = nn.Dropout(dropout) self.linear2 = nn.Linear(dim_feedforward, d_model) self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) self.norm3 = nn.LayerNorm(d_model) self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout) self.dropout3 = nn.Dropout(dropout) self.activation = _get_activation_fn(activation) # cls. num_cls = cfg.MODEL.SparseRCNN.NUM_CLS cls_module = list() for _ in range(num_cls): cls_module.append(nn.Linear(d_model, d_model, False)) cls_module.append(nn.LayerNorm(d_model)) cls_module.append(nn.ReLU(inplace=True)) self.cls_module = nn.ModuleList(cls_module) # reg. num_reg = cfg.MODEL.SparseRCNN.NUM_REG reg_module = list() for _ in range(num_reg): reg_module.append(nn.Linear(d_model, d_model, False)) reg_module.append(nn.LayerNorm(d_model)) reg_module.append(nn.ReLU(inplace=True)) self.reg_module = nn.ModuleList(reg_module) # pred. self.use_focal = cfg.MODEL.SparseRCNN.USE_FOCAL if self.use_focal: self.class_logits = nn.Linear(d_model, num_classes) else: self.class_logits = nn.Linear(d_model, num_classes + 1) self.bboxes_delta = nn.Linear(d_model, 4) self.scale_clamp = scale_clamp self.bbox_weights = bbox_weightsdef forward(self, features, bboxes, pro_features, pooler): """ :param bboxes: (N, nr_boxes, 4) :param pro_features: (N, nr_boxes, d_model) """
N, nr_boxes = bboxes.shape[:2] # roi_feature. proposal_boxes = list() for b in range(N): proposal_boxes.append(Boxes(bboxes[b])) roi_features = pooler(features, proposal_boxes) roi_features = roi_features.view(N * nr_boxes, self.d_model, -1).permute(2, 0, 1) # self_att. pro_features = pro_features.view(N, nr_boxes, self.d_model).permute(1, 0, 2) pro_features2 = self.self_attn(pro_features, pro_features, value=pro_features)[0] pro_features = pro_features + self.dropout1(pro_features2) pro_features = self.norm1(pro_features) # inst_interact. pro_features = pro_features.view(nr_boxes, N, self.d_model).permute(1, 0, 2).reshape(1, N * nr_boxes, self.d_model) if self.attetion: pro_features2 = self.inst_interact(query=pro_features, key=roi_features, value=roi_features)[0] else: pro_features2 = self.inst_interact(pro_features, roi_features) pro_features = pro_features + self.dropout2(pro_features2) obj_features = self.norm2(pro_features) # obj_feature. obj_features2 = self.linear2(self.dropout(self.activation(self.linear1(obj_features)))) obj_features = obj_features + self.dropout3(obj_features2) obj_features = self.norm3(obj_features) fc_feature = obj_features.transpose(0, 1).reshape(N * nr_boxes, -1) cls_feature = fc_feature.clone() reg_feature = fc_feature.clone() for cls_layer in self.cls_module: cls_feature = cls_layer(cls_feature) for reg_layer in self.reg_module: reg_feature = reg_layer(reg_feature) class_logits = self.class_logits(cls_feature) bboxes_deltas = self.bboxes_delta(reg_feature) pred_bboxes = self.apply_deltas(bboxes_deltas, bboxes.view(-1, 4)) return class_logits.view(N, nr_boxes, -1), pred_bboxes.view(N, nr_boxes, -1), obj_features` - In my opinion, it is unpractical to have no any hand-crafted component in the pipeline, for example, choosing visualization threshold value, the depth of the network, the kernel size of convolution...
HYUNJS
zzzzzz0407
Hi :) Thank you for sharing code of Sparse RCNN. I really like your novel ideas in this paper.
I would like to ask regarding the instance-interaction module in dynamic head.
According to the Table 4 in your paper, 37.2 AP is achieved without instance interaction module. However, in the Table 8, using multi-head attention as instance interaction would rather decrease the AP to 35.7 (-1.8). What is the reason for such performance degradation? Would it be due to small number of epochs for training self-attention enough? If yes, then may I ask have you trained with longer training schedule (I would appreciate if you show the training graphs)?
Also, may I ask how did you implement multi-head attention in dynamic head? I guess you may reduce the dimension of flattened roi featuers (7x7xC -> C) in order to apply attention with object features (since object features dimension is C). Which method did you use in the ablation study?
Furthermore, I would like to ask regarding the inference time usage of SparseRCNN.