search

facebookresearch / detectron2

Detectron2 is a platform for object detection, segmentation and other visual recognition tasks.
https://detectron2.readthedocs.io/en/latest/
Apache License 2.0
30.1k stars 7.42k forks source link

Custom model training error. #2417

Closed meet-VC closed 3 years ago

meet-VC commented 3 years ago

I am trying to do custom training on a dataset, which I label using labelme and is in COCO format. 

#We are importing our own Trainer Module here to use the COCO validation evaluation during training. Otherwise no validation eval occurs.

from detectron2.engine import DefaultTrainer
from detectron2.evaluation import COCOEvaluator

class CocoTrainer(DefaultTrainer):

  @classmethod
  def build_evaluator(cls, cfg, dataset_name, output_folder=None):

    if output_folder is None:
        os.makedirs("coco_eval", exist_ok=True)
        output_folder = "coco_eval"

    return COCOEvaluator(dataset_name, cfg, False, output_folder)
import json 
with open('/content/Bbox/metadata_config.json') as f:
  data = json.load(f)

number_classes=(data['number_of_class'])
name_classes=(data['classes'])
print(number_classes,name_classes)

Output: 3 ['Airplane', 'Runway', 'Building']

from detectron2.config import get_cfg
import os
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml"))
cfg.DATASETS.TRAIN = ("custom_train",)
cfg.DATASETS.TEST = ("custom_val",)
cfg.DATALOADER.NUM_WORKERS = 4
#cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("/content/drive/MyDrive/IIT/model_final_68b088.pkl")  # Let training initialize from model zoo
cfg.MODEL.WEIGHTS = "/content/drive/MyDrive/IIT/model_final_68b088.pkl"
cfg.SOLVER.IMS_PER_BATCH = 4
cfg.SOLVER.BASE_LR = 0.001
cfg.SOLVER.WARMUP_ITERS = 100
cfg.SOLVER.MAX_ITER = 150 #adjust up if val mAP is still rising, adjust down if overfit
cfg.SOLVER.STEPS = (100, 150)
cfg.SOLVER.GAMMA = 0.05
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 64
cfg.MODEL.ROI_HEADS.NUM_CLASSES = number_classes
cfg.TEST.EVAL_PERIOD = 50
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
trainer = CocoTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()

Error Log

[12/25 13:12:58 d2.engine.defaults]: Model:
GeneralizedRCNN(
  (backbone): FPN(
    (fpn_lateral2): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral3): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral4): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral5): Conv2d(2048, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output5): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (top_block): LastLevelMaxPool()
    (bottom_up): ResNet(
      (stem): BasicStem(
        (conv1): Conv2d(
          3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False
          (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)
        )
      )
      (res2): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv1): Conv2d(
            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv2): Conv2d(
            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv3): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv2): Conv2d(
            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv3): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv2): Conv2d(
            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv3): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
        )
      )
      (res3): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv1): Conv2d(
            256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
        (3): BottleneckBlock(
          (conv1): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
      )
      (res4): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv1): Conv2d(
            512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (3): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (4): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (5): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (6): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (7): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (8): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (9): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (10): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (11): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (12): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (13): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (14): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (15): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (16): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (17): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (18): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (19): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (20): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (21): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (22): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
      )
      (res5): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv1): Conv2d(
            1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv2): Conv2d(
            2048, 2048, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv3): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv2): Conv2d(
            2048, 2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv3): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv2): Conv2d(
            2048, 2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv3): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
        )
      )
    )
  )
  (proposal_generator): RPN(
    (anchor_generator): DefaultAnchorGenerator(
      (cell_anchors): BufferList()
    )
    (rpn_head): StandardRPNHead(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (objectness_logits): Conv2d(256, 3, kernel_size=(1, 1), stride=(1, 1))
      (anchor_deltas): Conv2d(256, 12, kernel_size=(1, 1), stride=(1, 1))
    )
  )
  (roi_heads): StandardROIHeads(
    (box_pooler): ROIPooler(
      (level_poolers): ModuleList(
        (0): ROIAlign(output_size=(7, 7), spatial_scale=0.25, sampling_ratio=0, aligned=True)
        (1): ROIAlign(output_size=(7, 7), spatial_scale=0.125, sampling_ratio=0, aligned=True)
        (2): ROIAlign(output_size=(7, 7), spatial_scale=0.0625, sampling_ratio=0, aligned=True)
        (3): ROIAlign(output_size=(7, 7), spatial_scale=0.03125, sampling_ratio=0, aligned=True)
      )
    )
    (box_head): FastRCNNConvFCHead(
      (fc1): Linear(in_features=12544, out_features=1024, bias=True)
      (fc2): Linear(in_features=1024, out_features=1024, bias=True)
    )
    (box_predictor): FastRCNNOutputLayers(
      (cls_score): Linear(in_features=1024, out_features=4, bias=True)
      (bbox_pred): Linear(in_features=1024, out_features=12, bias=True)
    )
  )
)
[12/25 13:12:58 d2.data.datasets.coco]: Loaded 39 images in COCO format from /content/Bbox/train/annotations.json
[12/25 13:12:58 d2.data.build]: Removed 0 images with no usable annotations. 39 images left.
[12/25 13:12:58 d2.data.common]: Serializing 39 elements to byte tensors and concatenating them all ...
[12/25 13:12:58 d2.data.common]: Serialized dataset takes 0.02 MiB
[12/25 13:12:58 d2.data.detection_utils]: TransformGens used in training: [ResizeShortestEdge(short_edge_length=(640, 672, 704, 736, 768, 800), max_size=1333, sample_style='choice'), RandomFlip()]
[12/25 13:12:58 d2.data.build]: Using training sampler TrainingSampler
Skip loading parameter 'roi_heads.box_predictor.cls_score.weight' to the model due to incompatible shapes: (81, 1024) in the checkpoint but (4, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.cls_score.bias' to the model due to incompatible shapes: (81,) in the checkpoint but (4,) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.weight' to the model due to incompatible shapes: (320, 1024) in the checkpoint but (12, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.bias' to the model due to incompatible shapes: (320,) in the checkpoint but (12,) in the model! You might want to double check if this is expected.
[12/25 13:12:59 d2.engine.train_loop]: Starting training from iteration 0
[12/25 13:13:37 d2.utils.events]:  eta: 0:04:19  iter: 19  total_loss: 2.323  loss_cls: 1.156  loss_box_reg: 0.595  loss_rpn_cls: 0.458  loss_rpn_loc: 0.112  time: 1.9469  data_time: 0.0330  lr: 0.000191  max_mem: 8422M
[12/25 13:14:17 d2.utils.events]:  eta: 0:03:40  iter: 39  total_loss: 1.757  loss_cls: 0.764  loss_box_reg: 0.723  loss_rpn_cls: 0.132  loss_rpn_loc: 0.095  time: 1.9598  data_time: 0.0126  lr: 0.000391  max_mem: 8422M
[12/25 13:14:37 d2.data.datasets.coco]: Loaded 3 images in COCO format from /content/Bbox/validation/annotations.json
[12/25 13:14:37 d2.data.common]: Serializing 3 elements to byte tensors and concatenating them all ...
[12/25 13:14:37 d2.data.common]: Serialized dataset takes 0.00 MiB
[12/25 13:14:37 d2.evaluation.evaluator]: Start inference on 3 images
[12/25 13:14:38 d2.evaluation.evaluator]: Total inference time: 0:00:00.298119 (0.298119 s / img per device, on 1 devices)
[12/25 13:14:38 d2.evaluation.evaluator]: Total inference pure compute time: 0:00:00 (0.188229 s / img per device, on 1 devices)
[12/25 13:14:38 d2.evaluation.coco_evaluation]: Preparing results for COCO format ...
ERROR [12/25 13:14:38 d2.engine.train_loop]: Exception during training:
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py", line 133, in train
    self.after_step()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py", line 154, in after_step
    h.after_step()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py", line 350, in after_step
    self._do_eval()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py", line 324, in _do_eval
    results = self._func()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py", line 352, in test_and_save_results
    self._last_eval_results = self.test(self.cfg, self.model)
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py", line 518, in test
    results_i = inference_on_dataset(model, data_loader, evaluator)
  File "/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/evaluator.py", line 176, in inference_on_dataset
    results = evaluator.evaluate()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py", line 143, in evaluate
    self._eval_predictions(set(self._tasks), predictions)
  File "/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py", line 165, in _eval_predictions
    category_id
AssertionError: A prediction has category_id=1, which is not available in the dataset.
[12/25 13:14:38 d2.engine.hooks]: Overall training speed: 47 iterations in 0:01:34 (2.0086 s / it)
[12/25 13:14:38 d2.engine.hooks]: Total training time: 0:01:35 (0:00:01 on hooks)
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-15-699fd785ed90> in <module>()
      2 trainer = CocoTrainer(cfg)
      3 trainer.resume_or_load(resume=False)
----> 4 trainer.train()

9 frames
/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py in train(self)
    399             OrderedDict of results, if evaluation is enabled. Otherwise None.
    400         """
--> 401         super().train(self.start_iter, self.max_iter)
    402         if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process():
    403             assert hasattr(

/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py in train(self, start_iter, max_iter)
    131                     self.before_step()
    132                     self.run_step()
--> 133                     self.after_step()
    134             except Exception:
    135                 logger.exception("Exception during training:")

/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py in after_step(self)
    152     def after_step(self):
    153         for h in self._hooks:
--> 154             h.after_step()
    155         # this guarantees, that in each hook's after_step, storage.iter == trainer.iter
    156         self.storage.step()

/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py in after_step(self)
    348         is_final = next_iter == self.trainer.max_iter
    349         if is_final or (self._period > 0 and next_iter % self._period == 0):
--> 350             self._do_eval()
    351 
    352     def after_train(self):

/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py in _do_eval(self)
    322 
    323     def _do_eval(self):
--> 324         results = self._func()
    325 
    326         if results:

/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py in test_and_save_results()
    350 
    351         def test_and_save_results():
--> 352             self._last_eval_results = self.test(self.cfg, self.model)
    353             return self._last_eval_results
    354 

/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py in test(cls, cfg, model, evaluators)
    516                     results[dataset_name] = {}
    517                     continue
--> 518             results_i = inference_on_dataset(model, data_loader, evaluator)
    519             results[dataset_name] = results_i
    520             if comm.is_main_process():

/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/evaluator.py in inference_on_dataset(model, data_loader, evaluator)
    174     )
    175 
--> 176     results = evaluator.evaluate()
    177     # An evaluator may return None when not in main process.
    178     # Replace it by an empty dict instead to make it easier for downstream code to handle

/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py in evaluate(self)
    141             self._eval_box_proposals(predictions)
    142         if "instances" in predictions[0]:
--> 143             self._eval_predictions(set(self._tasks), predictions)
    144         # Copy so the caller can do whatever with results
    145         return copy.deepcopy(self._results)

/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py in _eval_predictions(self, tasks, predictions)
    163                     category_id in reverse_id_mapping
    164                 ), "A prediction has category_id={}, which is not available in the dataset.".format(
--> 165                     category_id
    166                 )
    167                 result["category_id"] = reverse_id_mapping[category_id]

AssertionError: A prediction has category_id=1, which is not available in the dataset.
shub2k commented 3 years ago

i have created a custom dataset but i am getting this error while training

TypeError Traceback (most recent call last)

in 1 os.makedirs(cfg.OUTPUT_DIR, exist_ok=True) ----> 2 trainer = NFL_Trainer(cfg) 3 trainer.resume_or_load(resume=False) 4 trainer.train() /opt/conda/lib/python3.7/site-packages/detectron2/engine/defaults.py in __init__(self, cfg) 282 model = self.build_model(cfg) 283 optimizer = self.build_optimizer(cfg, model) --> 284 data_loader = self.build_train_loader(cfg) 285 286 # For training, wrap with DDP. But don't need this for inference. /opt/conda/lib/python3.7/site-packages/detectron2/engine/defaults.py in build_train_loader(cls, cfg) 471 Overwrite it if you'd like a different data loader. 472 """ --> 473 return build_detection_train_loader(cfg) 474 475 @classmethod /opt/conda/lib/python3.7/site-packages/detectron2/config/config.py in wrapped(*args, **kwargs) 199 def wrapped(*args, **kwargs): 200 if _called_with_cfg(*args, **kwargs): --> 201 explicit_args = _get_args_from_config(from_config, *args, **kwargs) 202 return orig_func(**explicit_args) 203 else: /opt/conda/lib/python3.7/site-packages/detectron2/config/config.py in _get_args_from_config(from_config_func, *args, **kwargs) 234 if name not in supported_arg_names: 235 extra_kwargs[name] = kwargs.pop(name) --> 236 ret = from_config_func(*args, **kwargs) 237 # forward the other arguments to __init__ 238 ret.update(extra_kwargs) /opt/conda/lib/python3.7/site-packages/detectron2/data/build.py in _train_loader_from_config(cfg, mapper, dataset, sampler) 307 if cfg.MODEL.KEYPOINT_ON 308 else 0, --> 309 proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, 310 ) 311 /opt/conda/lib/python3.7/site-packages/detectron2/data/build.py in get_detection_dataset_dicts(dataset_names, filter_empty, min_keypoints, proposal_files) 244 class_names = MetadataCatalog.get(dataset_names[0]).thing_classes 245 check_metadata_consistency("thing_classes", dataset_names) --> 246 print_instances_class_histogram(dataset_dicts, class_names) 247 except AttributeError: # class names are not available for this dataset 248 pass /opt/conda/lib/python3.7/site-packages/detectron2/data/build.py in print_instances_class_histogram(dataset_dicts, class_names) 178 179 data = list( --> 180 itertools.chain(*[[short_name(class_names[i]), int(v)] for i, v in enumerate(histogram)]) 181 ) 182 total_num_instances = sum(data[1::2]) /opt/conda/lib/python3.7/site-packages/detectron2/data/build.py in (.0) 178 179 data = list( --> 180 itertools.chain(*[[short_name(class_names[i]), int(v)] for i, v in enumerate(histogram)]) 181 ) 182 total_num_instances = sum(data[1::2]) /opt/conda/lib/python3.7/site-packages/detectron2/data/build.py in short_name(x) 173 def short_name(x): 174 # make long class names shorter. useful for lvis --> 175 if len(x) > 13: 176 return x[:11] + ".." 177 return x TypeError: object of type 'int' has no len()
shub2k commented 3 years ago

I am trying to do custom training on a dataset, which I label using labelme and is in COCO format.

#We are importing our own Trainer Module here to use the COCO validation evaluation during training. Otherwise no validation eval occurs.

from detectron2.engine import DefaultTrainer
from detectron2.evaluation import COCOEvaluator

class CocoTrainer(DefaultTrainer):

  @classmethod
  def build_evaluator(cls, cfg, dataset_name, output_folder=None):

    if output_folder is None:
        os.makedirs("coco_eval", exist_ok=True)
        output_folder = "coco_eval"

    return COCOEvaluator(dataset_name, cfg, False, output_folder)
import json 
with open('/content/Bbox/metadata_config.json') as f:
  data = json.load(f)

number_classes=(data['number_of_class'])
name_classes=(data['classes'])
print(number_classes,name_classes)

Output: 3 ['Airplane', 'Runway', 'Building']

from detectron2.config import get_cfg
import os
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml"))
cfg.DATASETS.TRAIN = ("custom_train",)
cfg.DATASETS.TEST = ("custom_val",)
cfg.DATALOADER.NUM_WORKERS = 4
#cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("/content/drive/MyDrive/IIT/model_final_68b088.pkl")  # Let training initialize from model zoo
cfg.MODEL.WEIGHTS = "/content/drive/MyDrive/IIT/model_final_68b088.pkl"
cfg.SOLVER.IMS_PER_BATCH = 4
cfg.SOLVER.BASE_LR = 0.001
cfg.SOLVER.WARMUP_ITERS = 100
cfg.SOLVER.MAX_ITER = 150 #adjust up if val mAP is still rising, adjust down if overfit
cfg.SOLVER.STEPS = (100, 150)
cfg.SOLVER.GAMMA = 0.05
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 64
cfg.MODEL.ROI_HEADS.NUM_CLASSES = number_classes
cfg.TEST.EVAL_PERIOD = 50
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
trainer = CocoTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()

Error Log

[12/25 13:12:58 d2.engine.defaults]: Model:
GeneralizedRCNN(
  (backbone): FPN(
    (fpn_lateral2): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral3): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral4): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (fpn_lateral5): Conv2d(2048, 256, kernel_size=(1, 1), stride=(1, 1))
    (fpn_output5): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (top_block): LastLevelMaxPool()
    (bottom_up): ResNet(
      (stem): BasicStem(
        (conv1): Conv2d(
          3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False
          (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)
        )
      )
      (res2): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv1): Conv2d(
            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv2): Conv2d(
            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv3): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv2): Conv2d(
            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv3): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv2): Conv2d(
            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
          (conv3): Conv2d(
            256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)
          )
        )
      )
      (res3): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv1): Conv2d(
            256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
        (3): BottleneckBlock(
          (conv1): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv2): Conv2d(
            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
          (conv3): Conv2d(
            512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)
          )
        )
      )
      (res4): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv1): Conv2d(
            512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (3): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (4): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (5): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (6): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (7): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (8): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (9): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (10): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (11): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (12): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (13): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (14): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (15): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (16): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (17): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (18): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (19): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (20): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (21): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
        (22): BottleneckBlock(
          (conv1): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv2): Conv2d(
            1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
          (conv3): Conv2d(
            1024, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)
          )
        )
      )
      (res5): Sequential(
        (0): BottleneckBlock(
          (shortcut): Conv2d(
            1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv1): Conv2d(
            1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv2): Conv2d(
            2048, 2048, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv3): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
        )
        (1): BottleneckBlock(
          (conv1): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv2): Conv2d(
            2048, 2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv3): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
        )
        (2): BottleneckBlock(
          (conv1): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv2): Conv2d(
            2048, 2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
          (conv3): Conv2d(
            2048, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False
            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)
          )
        )
      )
    )
  )
  (proposal_generator): RPN(
    (anchor_generator): DefaultAnchorGenerator(
      (cell_anchors): BufferList()
    )
    (rpn_head): StandardRPNHead(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (objectness_logits): Conv2d(256, 3, kernel_size=(1, 1), stride=(1, 1))
      (anchor_deltas): Conv2d(256, 12, kernel_size=(1, 1), stride=(1, 1))
    )
  )
  (roi_heads): StandardROIHeads(
    (box_pooler): ROIPooler(
      (level_poolers): ModuleList(
        (0): ROIAlign(output_size=(7, 7), spatial_scale=0.25, sampling_ratio=0, aligned=True)
        (1): ROIAlign(output_size=(7, 7), spatial_scale=0.125, sampling_ratio=0, aligned=True)
        (2): ROIAlign(output_size=(7, 7), spatial_scale=0.0625, sampling_ratio=0, aligned=True)
        (3): ROIAlign(output_size=(7, 7), spatial_scale=0.03125, sampling_ratio=0, aligned=True)
      )
    )
    (box_head): FastRCNNConvFCHead(
      (fc1): Linear(in_features=12544, out_features=1024, bias=True)
      (fc2): Linear(in_features=1024, out_features=1024, bias=True)
    )
    (box_predictor): FastRCNNOutputLayers(
      (cls_score): Linear(in_features=1024, out_features=4, bias=True)
      (bbox_pred): Linear(in_features=1024, out_features=12, bias=True)
    )
  )
)
[12/25 13:12:58 d2.data.datasets.coco]: Loaded 39 images in COCO format from /content/Bbox/train/annotations.json
[12/25 13:12:58 d2.data.build]: Removed 0 images with no usable annotations. 39 images left.
[12/25 13:12:58 d2.data.common]: Serializing 39 elements to byte tensors and concatenating them all ...
[12/25 13:12:58 d2.data.common]: Serialized dataset takes 0.02 MiB
[12/25 13:12:58 d2.data.detection_utils]: TransformGens used in training: [ResizeShortestEdge(short_edge_length=(640, 672, 704, 736, 768, 800), max_size=1333, sample_style='choice'), RandomFlip()]
[12/25 13:12:58 d2.data.build]: Using training sampler TrainingSampler
Skip loading parameter 'roi_heads.box_predictor.cls_score.weight' to the model due to incompatible shapes: (81, 1024) in the checkpoint but (4, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.cls_score.bias' to the model due to incompatible shapes: (81,) in the checkpoint but (4,) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.weight' to the model due to incompatible shapes: (320, 1024) in the checkpoint but (12, 1024) in the model! You might want to double check if this is expected.
Skip loading parameter 'roi_heads.box_predictor.bbox_pred.bias' to the model due to incompatible shapes: (320,) in the checkpoint but (12,) in the model! You might want to double check if this is expected.
[12/25 13:12:59 d2.engine.train_loop]: Starting training from iteration 0
[12/25 13:13:37 d2.utils.events]:  eta: 0:04:19  iter: 19  total_loss: 2.323  loss_cls: 1.156  loss_box_reg: 0.595  loss_rpn_cls: 0.458  loss_rpn_loc: 0.112  time: 1.9469  data_time: 0.0330  lr: 0.000191  max_mem: 8422M
[12/25 13:14:17 d2.utils.events]:  eta: 0:03:40  iter: 39  total_loss: 1.757  loss_cls: 0.764  loss_box_reg: 0.723  loss_rpn_cls: 0.132  loss_rpn_loc: 0.095  time: 1.9598  data_time: 0.0126  lr: 0.000391  max_mem: 8422M
[12/25 13:14:37 d2.data.datasets.coco]: Loaded 3 images in COCO format from /content/Bbox/validation/annotations.json
[12/25 13:14:37 d2.data.common]: Serializing 3 elements to byte tensors and concatenating them all ...
[12/25 13:14:37 d2.data.common]: Serialized dataset takes 0.00 MiB
[12/25 13:14:37 d2.evaluation.evaluator]: Start inference on 3 images
[12/25 13:14:38 d2.evaluation.evaluator]: Total inference time: 0:00:00.298119 (0.298119 s / img per device, on 1 devices)
[12/25 13:14:38 d2.evaluation.evaluator]: Total inference pure compute time: 0:00:00 (0.188229 s / img per device, on 1 devices)
[12/25 13:14:38 d2.evaluation.coco_evaluation]: Preparing results for COCO format ...
ERROR [12/25 13:14:38 d2.engine.train_loop]: Exception during training:
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py", line 133, in train
    self.after_step()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py", line 154, in after_step
    h.after_step()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py", line 350, in after_step
    self._do_eval()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py", line 324, in _do_eval
    results = self._func()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py", line 352, in test_and_save_results
    self._last_eval_results = self.test(self.cfg, self.model)
  File "/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py", line 518, in test
    results_i = inference_on_dataset(model, data_loader, evaluator)
  File "/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/evaluator.py", line 176, in inference_on_dataset
    results = evaluator.evaluate()
  File "/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py", line 143, in evaluate
    self._eval_predictions(set(self._tasks), predictions)
  File "/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py", line 165, in _eval_predictions
    category_id
AssertionError: A prediction has category_id=1, which is not available in the dataset.
[12/25 13:14:38 d2.engine.hooks]: Overall training speed: 47 iterations in 0:01:34 (2.0086 s / it)
[12/25 13:14:38 d2.engine.hooks]: Total training time: 0:01:35 (0:00:01 on hooks)
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-15-699fd785ed90> in <module>()
      2 trainer = CocoTrainer(cfg)
      3 trainer.resume_or_load(resume=False)
----> 4 trainer.train()

9 frames
/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py in train(self)
    399             OrderedDict of results, if evaluation is enabled. Otherwise None.
    400         """
--> 401         super().train(self.start_iter, self.max_iter)
    402         if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process():
    403             assert hasattr(

/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py in train(self, start_iter, max_iter)
    131                     self.before_step()
    132                     self.run_step()
--> 133                     self.after_step()
    134             except Exception:
    135                 logger.exception("Exception during training:")

/usr/local/lib/python3.6/dist-packages/detectron2/engine/train_loop.py in after_step(self)
    152     def after_step(self):
    153         for h in self._hooks:
--> 154             h.after_step()
    155         # this guarantees, that in each hook's after_step, storage.iter == trainer.iter
    156         self.storage.step()

/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py in after_step(self)
    348         is_final = next_iter == self.trainer.max_iter
    349         if is_final or (self._period > 0 and next_iter % self._period == 0):
--> 350             self._do_eval()
    351 
    352     def after_train(self):

/usr/local/lib/python3.6/dist-packages/detectron2/engine/hooks.py in _do_eval(self)
    322 
    323     def _do_eval(self):
--> 324         results = self._func()
    325 
    326         if results:

/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py in test_and_save_results()
    350 
    351         def test_and_save_results():
--> 352             self._last_eval_results = self.test(self.cfg, self.model)
    353             return self._last_eval_results
    354 

/usr/local/lib/python3.6/dist-packages/detectron2/engine/defaults.py in test(cls, cfg, model, evaluators)
    516                     results[dataset_name] = {}
    517                     continue
--> 518             results_i = inference_on_dataset(model, data_loader, evaluator)
    519             results[dataset_name] = results_i
    520             if comm.is_main_process():

/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/evaluator.py in inference_on_dataset(model, data_loader, evaluator)
    174     )
    175 
--> 176     results = evaluator.evaluate()
    177     # An evaluator may return None when not in main process.
    178     # Replace it by an empty dict instead to make it easier for downstream code to handle

/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py in evaluate(self)
    141             self._eval_box_proposals(predictions)
    142         if "instances" in predictions[0]:
--> 143             self._eval_predictions(set(self._tasks), predictions)
    144         # Copy so the caller can do whatever with results
    145         return copy.deepcopy(self._results)

/usr/local/lib/python3.6/dist-packages/detectron2/evaluation/coco_evaluation.py in _eval_predictions(self, tasks, predictions)
    163                     category_id in reverse_id_mapping
    164                 ), "A prediction has category_id={}, which is not available in the dataset.".format(
--> 165                     category_id
    166                 )
    167                 result["category_id"] = reverse_id_mapping[category_id]

AssertionError: A prediction has category_id=1, which is not available in the dataset.

i guess the category id aka the class index you have mentioned is not there

ppwwyyxx commented 3 years ago

AssertionError: A prediction has category_id=1, which is not available in the dataset.

The test dataset doesn't define the correct number of classes.