question about custom dataset train

bruce2k commented 5 years ago

hi , this is a great work!

after read the paper and try the inference , i want to train the model use my own dataset with 'keypoint_rcnn_R-50-FPN_s1x.yaml'. because my dataset have 13 categories and i notice the 'background' will add to class , so i set the MODEL.NUM_CLASSES: 14 in the '*.yaml' file; but when i train the model , i found the train log is strange, at beginning the accuacy keep increase and loss keep decrease, but after train about 2900 iters the "accuracy_cls" suddenly become very small, and loss become very big , even continue train , the situation no change, i use single GPU 1080Ti. it's cause by single GPU? don't know why, what should I do?

json_stats: {"accuracy_cls": "0.666667", "eta": "6:43:21", "iter": 29160, "loss": "5.467151", "loss_bbox": "0.010930", "loss_cls": "1.028070", "loss_kps": "4.319079", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239996"} json_stats: {"accuracy_cls": "0.666667", "eta": "6:43:16", "iter": 29180, "loss": "5.673819", "loss_bbox": "0.015378", "loss_cls": "0.981587", "loss_kps": "4.502552", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240001"} json_stats: {"accuracy_cls": "0.500000", "eta": "6:43:12", "iter": 29200, "loss": "5.811498", "loss_bbox": "0.028074", "loss_cls": "1.491318", "loss_kps": "4.309727", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240007"} json_stats: {"accuracy_cls": "0.775000", "eta": "6:43:08", "iter": 29220, "loss": "5.127981", "loss_bbox": "0.017984", "loss_cls": "0.899908", "loss_kps": "4.192500", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240012"} json_stats: {"accuracy_cls": "0.633333", "eta": "6:43:04", "iter": 29240, "loss": "5.661786", "loss_bbox": "0.011890", "loss_cls": "0.835383", "loss_kps": "4.459731", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240017"} json_stats: {"accuracy_cls": "0.666667", "eta": "6:42:59", "iter": 29260, "loss": "5.341789", "loss_bbox": "0.020640", "loss_cls": "1.075727", "loss_kps": "4.053029", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240020"} json_stats: {"accuracy_cls": "0.708333", "eta": "6:42:54", "iter": 29280, "loss": "5.621279", "loss_bbox": "0.007555", "loss_cls": "1.199983", "loss_kps": "4.206930", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240016"} json_stats: {"accuracy_cls": "0.500000", "eta": "6:42:49", "iter": 29300, "loss": "6.222597", "loss_bbox": "0.009620", "loss_cls": "1.344310", "loss_kps": "4.376087", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240017"} json_stats: {"accuracy_cls": "0.500000", "eta": "6:42:44", "iter": 29320, "loss": "5.720005", "loss_bbox": "0.012881", "loss_cls": "1.318774", "loss_kps": "4.252826", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240015"} json_stats: {"accuracy_cls": "0.500000", "eta": "6:42:40", "iter": 29340, "loss": "6.000343", "loss_bbox": "0.021975", "loss_cls": "1.289776", "loss_kps": "4.473332", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240016"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:42:35", "iter": 29360, "loss": "8.278537", "loss_bbox": "0.198123", "loss_cls": "2.201621", "loss_kps": "4.681865", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240019"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:42:30", "iter": 29380, "loss": "8.063788", "loss_bbox": "0.001438", "loss_cls": "2.124703", "loss_kps": "5.813482", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240022"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:42:26", "iter": 29400, "loss": "7.717090", "loss_bbox": "0.000990", "loss_cls": "2.099766", "loss_kps": "5.642518", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240022"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:42:21", "iter": 29420, "loss": "8.316284", "loss_bbox": "0.000652", "loss_cls": "2.320769", "loss_kps": "5.571927", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240020"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:42:16", "iter": 29440, "loss": "7.269794", "loss_bbox": "0.001455", "loss_cls": "2.132237", "loss_kps": "5.073633", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240023"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:42:11", "iter": 29460, "loss": "11.304214", "loss_bbox": "0.000598", "loss_cls": "3.072437", "loss_kps": "8.049737", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240024"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:42:07", "iter": 29480, "loss": "11.185940", "loss_bbox": "0.001746", "loss_cls": "3.135216", "loss_kps": "8.050022", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240025"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:42:02", "iter": 29500, "loss": "10.535622", "loss_bbox": "0.000504", "loss_cls": "2.486256", "loss_kps": "8.049674", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240029"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:41:58", "iter": 29520, "loss": "10.200245", "loss_bbox": "0.001737", "loss_cls": "2.147464", "loss_kps": "8.049827", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240033"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:41:53", "iter": 29540, "loss": "10.369551", "loss_bbox": "0.000180", "loss_cls": "2.319638", "loss_kps": "8.049517", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240031"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:41:48", "iter": 29560, "loss": "10.335373", "loss_bbox": "0.000089", "loss_cls": "2.285769", "loss_kps": "8.050155", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240031"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:41:43", "iter": 29580, "loss": "10.154091", "loss_bbox": "0.000133", "loss_cls": "2.104799", "loss_kps": "8.050002", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240027"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:41:39", "iter": 29600, "loss": "10.092386", "loss_bbox": "0.000033", "loss_cls": "2.042093", "loss_kps": "8.049589", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240030"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:41:34", "iter": 29620, "loss": "10.066527", "loss_bbox": "0.000797", "loss_cls": "2.015740", "loss_kps": "8.049904", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240028"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:41:29", "iter": 29640, "loss": "10.203655", "loss_bbox": "0.000074", "loss_cls": "2.152822", "loss_kps": "8.050062", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240030"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:41:24", "iter": 29660, "loss": "10.293951", "loss_bbox": "0.000029", "loss_cls": "2.243261", "loss_kps": "8.049760", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240034"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:41:19", "iter": 29680, "loss": "10.480244", "loss_bbox": "0.000026", "loss_cls": "2.430091", "loss_kps": "8.050114", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240030"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:41:14", "iter": 29700, "loss": "10.279327", "loss_bbox": "0.000007", "loss_cls": "2.229466", "loss_kps": "8.050054", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240025"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:41:09", "iter": 29720, "loss": "10.350375", "loss_bbox": "0.000010", "loss_cls": "2.300287", "loss_kps": "8.050275", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240024"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:41:04", "iter": 29740, "loss": "10.407750", "loss_bbox": "0.000010", "loss_cls": "2.357896", "loss_kps": "8.050261", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240019"} json_stats: {"accuracy_cls": "0.208333", "eta": "6:40:59", "iter": 29760, "loss": "10.281849", "loss_bbox": "0.000174", "loss_cls": "2.231058", "loss_kps": "8.050293", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240018"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:40:53", "iter": 29780, "loss": "10.202653", "loss_bbox": "0.000015", "loss_cls": "2.153254", "loss_kps": "8.050015", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240012"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:40:47", "iter": 29800, "loss": "10.338449", "loss_bbox": "0.000018", "loss_cls": "2.289509", "loss_kps": "8.049931", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239997"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:40:42", "iter": 29820, "loss": "10.374899", "loss_bbox": "0.000014", "loss_cls": "2.324546", "loss_kps": "8.050224", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239993"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:40:37", "iter": 29840, "loss": "10.198301", "loss_bbox": "0.000005", "loss_cls": "2.148395", "loss_kps": "8.049785", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239994"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:40:33", "iter": 29860, "loss": "10.274776", "loss_bbox": "0.000002", "loss_cls": "2.224470", "loss_kps": "8.050076", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239996"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:40:28", "iter": 29880, "loss": "10.312390", "loss_bbox": "0.000077", "loss_cls": "2.262911", "loss_kps": "8.050130", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239997"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:40:23", "iter": 29900, "loss": "10.220694", "loss_bbox": "0.000273", "loss_cls": "2.171055", "loss_kps": "8.049952", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239998"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:40:19", "iter": 29920, "loss": "10.283627", "loss_bbox": "0.000026", "loss_cls": "2.232925", "loss_kps": "8.050063", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240004"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:40:15", "iter": 29940, "loss": "10.371342", "loss_bbox": "0.000034", "loss_cls": "2.322415", "loss_kps": "8.050118", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240010"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:40:11", "iter": 29960, "loss": "10.295013", "loss_bbox": "0.000292", "loss_cls": "2.245483", "loss_kps": "8.049655", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240014"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:40:06", "iter": 29980, "loss": "10.322361", "loss_bbox": "0.000014", "loss_cls": "2.271739", "loss_kps": "8.049766", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240017"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:40:02", "iter": 30000, "loss": "10.202982", "loss_bbox": "0.000007", "loss_cls": "2.155206", "loss_kps": "8.049845", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240022"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:39:56", "iter": 30020, "loss": "10.188197", "loss_bbox": "0.000009", "loss_cls": "2.138806", "loss_kps": "8.049730", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240012"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:39:50", "iter": 30040, "loss": "10.401936", "loss_bbox": "0.000014", "loss_cls": "2.351543", "loss_kps": "8.050201", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240006"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:39:45", "iter": 30060, "loss": "10.199454", "loss_bbox": "0.000065", "loss_cls": "2.149361", "loss_kps": "8.049711", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.240003"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:39:40", "iter": 30080, "loss": "10.327039", "loss_bbox": "0.000387", "loss_cls": "2.275589", "loss_kps": "8.049937", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239998"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:39:35", "iter": 30100, "loss": "10.192726", "loss_bbox": "0.000208", "loss_cls": "2.142447", "loss_kps": "8.050004", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239995"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:39:30", "iter": 30120, "loss": "10.292066", "loss_bbox": "0.000068", "loss_cls": "2.241122", "loss_kps": "8.050055", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239995"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:39:25", "iter": 30140, "loss": "10.329316", "loss_bbox": "0.000279", "loss_cls": "2.279729", "loss_kps": "8.049994", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239992"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:39:20", "iter": 30160, "loss": "10.289998", "loss_bbox": "0.000018", "loss_cls": "2.242330", "loss_kps": "8.049366", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239992"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:39:15", "iter": 30180, "loss": "10.197249", "loss_bbox": "0.000001", "loss_cls": "2.146952", "loss_kps": "8.049759", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239988"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:39:10", "iter": 30200, "loss": "10.238089", "loss_bbox": "0.000009", "loss_cls": "2.187582", "loss_kps": "8.049773", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239982"} json_stats: {"accuracy_cls": "0.291667", "eta": "6:39:05", "iter": 30220, "loss": "10.222706", "loss_bbox": "0.000066", "loss_cls": "2.172647", "loss_kps": "8.049679", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239980"} json_stats: {"accuracy_cls": "0.291667", "eta": "6:39:00", "iter": 30240, "loss": "10.304263", "loss_bbox": "0.000010", "loss_cls": "2.254199", "loss_kps": "8.049764", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239978"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:38:54", "iter": 30260, "loss": "10.405146", "loss_bbox": "0.000005", "loss_cls": "2.355292", "loss_kps": "8.050019", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239969"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:38:48", "iter": 30280, "loss": "10.180382", "loss_bbox": "0.000057", "loss_cls": "2.130430", "loss_kps": "8.049191", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239960"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:38:42", "iter": 30300, "loss": "10.054037", "loss_bbox": "0.000461", "loss_cls": "2.002733", "loss_kps": "8.050159", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239946"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:38:37", "iter": 30320, "loss": "10.257288", "loss_bbox": "0.000046", "loss_cls": "2.207901", "loss_kps": "8.049732", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239943"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:38:32", "iter": 30340, "loss": "10.110934", "loss_bbox": "0.000007", "loss_cls": "2.060928", "loss_kps": "8.049686", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239939"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:38:26", "iter": 30360, "loss": "10.374855", "loss_bbox": "0.000158", "loss_cls": "2.322812", "loss_kps": "8.049711", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239934"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:38:22", "iter": 30380, "loss": "10.259105", "loss_bbox": "0.000021", "loss_cls": "2.208861", "loss_kps": "8.049861", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239933"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:38:17", "iter": 30400, "loss": "10.279961", "loss_bbox": "0.000017", "loss_cls": "2.229804", "loss_kps": "8.050116", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239939"} json_stats: {"accuracy_cls": "0.333333", "eta": "6:38:13", "iter": 30420, "loss": "10.040842", "loss_bbox": "0.000021", "loss_cls": "1.991944", "loss_kps": "8.049424", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239945"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:38:09", "iter": 30440, "loss": "10.338309", "loss_bbox": "0.000957", "loss_cls": "2.288189", "loss_kps": "8.049911", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239953"} json_stats: {"accuracy_cls": "0.125000", "eta": "6:38:05", "iter": 30460, "loss": "10.300609", "loss_bbox": "0.000026", "loss_cls": "2.251068", "loss_kps": "8.049866", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239958"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:38:00", "iter": 30480, "loss": "10.374983", "loss_bbox": "0.000160", "loss_cls": "2.324005", "loss_kps": "8.050022", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239960"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:37:55", "iter": 30500, "loss": "10.291791", "loss_bbox": "0.000137", "loss_cls": "2.240138", "loss_kps": "8.049871", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239958"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:37:51", "iter": 30520, "loss": "10.180387", "loss_bbox": "0.000096", "loss_cls": "2.131058", "loss_kps": "8.049663", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239961"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:37:46", "iter": 30540, "loss": "10.261712", "loss_bbox": "0.000032", "loss_cls": "2.211586", "loss_kps": "8.049699", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239962"} json_stats: {"accuracy_cls": "0.100000", "eta": "6:37:40", "iter": 30560, "loss": "10.177417", "loss_bbox": "0.000021", "loss_cls": "2.127708", "loss_kps": "8.049428", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239952"} json_stats: {"accuracy_cls": "0.250000", "eta": "6:37:36", "iter": 30580, "loss": "10.177744", "loss_bbox": "0.000249", "loss_cls": "2.127720", "loss_kps": "8.049449", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239953"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:37:31", "iter": 30600, "loss": "10.475524", "loss_bbox": "0.000021", "loss_cls": "2.425317", "loss_kps": "8.049826", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239957"} json_stats: {"accuracy_cls": "0.225000", "eta": "6:37:26", "iter": 30620, "loss": "10.258576", "loss_bbox": "0.000781", "loss_cls": "2.208184", "loss_kps": "8.049946", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239954"} json_stats: {"accuracy_cls": "0.000000", "eta": "6:37:22", "iter": 30640, "loss": "10.212185", "loss_bbox": "0.000045", "loss_cls": "2.164384", "loss_kps": "8.049806", "lr": "0.002500", "mb_qsize": 64, "mem": 6340, "time": "0.239956"}