search

ldkong1205 / PointCloud-C

Benchmarking and Analyzing Point Cloud Perception Robustness under Corruptions
https://pointcloud-c.github.io/home
164 stars 22 forks source link

Can't reproduce RPC + WolfMix results with the provided weights. #11

Open Haalum opened 2 years ago

Haalum commented 2 years ago

Hi, The RPC + WolfMix results with your pre-trained model differ from the paper's results. Any idea why? Thanks.

python PCT/main.py --exp_name=test --num_points=1024 --use_sgd=True --eval_corrupt=True --model_path pretrained_models/RPC_WOLFMix_final.t7 --test_batch_size 8 --model RPC

Namespace(batch_size=32, beta=0.0, dataset='modelnet40', dropout=0.5, epochs=250, eval=False, eval_corrupt=True, exp_name='test', jitter=False, knn=False, lr=0.0001, model='RPC', model_path='pretrained_models/RPC_WOLFMix_final.t7', momentum=0.9, no_cuda=False, nsample=512, num_points=1024, pw=False, rddrop=False, rdscale=False, rot=False, rsmix_prob=0.5, seed=1, shift=False, shuffle=False, test_batch_size=8, use_sgd=True, w_R_range=10, w_S_range=3, w_T_range=0.25, w_num_anchor=4, w_sample_type='fps', w_sigma=0.5)
Using GPU : 0 from 1 devices

{'acc': 0.923419773095624, 'avg_per_class_acc': 0.8932906976744185, 'corruption': 'clean'}
{'OA': 0.923, 'corruption': 'clean', 'level': 'Overall'}
{'acc': 0.9116693679092382, 'avg_per_class_acc': 0.8796686046511628, 'corruption': 'scale', 'level': 0}
{'acc': 0.9136952998379254, 'avg_per_class_acc': 0.8816279069767441, 'corruption': 'scale', 'level': 1}
{'acc': 0.906807131280389, 'avg_per_class_acc': 0.8727558139534886, 'corruption': 'scale', 'level': 2}
{'acc': 0.9116693679092382, 'avg_per_class_acc': 0.8772965116279069, 'corruption': 'scale', 'level': 3}
{'acc': 0.9120745542949756, 'avg_per_class_acc': 0.87625, 'corruption': 'scale', 'level': 4}
{'CE': 0.947, 'OA': 0.911, 'RCE': 0.6, 'corruption': 'scale', 'level': 'Overall'}
{'acc': 0.9173419773095624, 'avg_per_class_acc': 0.8749941860465116, 'corruption': 'jitter', 'level': 0}
{'acc': 0.8865478119935171, 'avg_per_class_acc': 0.8260523255813954, 'corruption': 'jitter', 'level': 1}
{'acc': 0.8103727714748784, 'avg_per_class_acc': 0.7123197674418604, 'corruption': 'jitter', 'level': 2}
{'acc': 0.6511345218800648, 'avg_per_class_acc': 0.524656976744186, 'corruption': 'jitter', 'level': 3}
{'acc': 0.4266612641815235, 'avg_per_class_acc': 0.32084883720930235, 'corruption': 'jitter', 'level': 4}
{'CE': 0.829, 'OA': 0.738, 'RCE': 0.764, 'corruption': 'jitter', 'level': 'Overall'}
{'acc': 0.9238249594813615, 'avg_per_class_acc': 0.8925813953488373, 'corruption': 'rotate', 'level': 0}
{'acc': 0.9217990275526742, 'avg_per_class_acc': 0.8923313953488371, 'corruption': 'rotate', 'level': 1}
{'acc': 0.923419773095624, 'avg_per_class_acc': 0.8891627906976745, 'corruption': 'rotate', 'level': 2}
{'acc': 0.8987034035656402, 'avg_per_class_acc': 0.8598837209302325, 'corruption': 'rotate', 'level': 3}
{'acc': 0.8456239870340356, 'avg_per_class_acc': 0.8082790697674419, 'corruption': 'rotate', 'level': 4}
{'CE': 0.451, 'OA': 0.903, 'RCE': 0.142, 'corruption': 'rotate', 'level': 'Overall'}
{'acc': 0.9153160453808752, 'avg_per_class_acc': 0.8827558139534883, 'corruption': 'dropout_global', 'level': 0}
{'acc': 0.9124797406807131, 'avg_per_class_acc': 0.8762558139534884, 'corruption': 'dropout_global', 'level': 1}
{'acc': 0.9055915721231766, 'avg_per_class_acc': 0.8673430232558139, 'corruption': 'dropout_global', 'level': 2}
{'acc': 0.8804700162074555, 'avg_per_class_acc': 0.834029069767442, 'corruption': 'dropout_global', 'level': 3}
{'acc': 0.7467585089141004, 'avg_per_class_acc': 0.7035290697674418, 'corruption': 'dropout_global', 'level': 4}
{'CE': 0.516, 'OA': 0.872, 'RCE': 0.293, 'corruption': 'dropout_global', 'level': 'Overall'}
{'acc': 0.9076175040518638, 'avg_per_class_acc': 0.8779244186046512, 'corruption': 'dropout_local', 'level': 0}
{'acc': 0.8772285251215559, 'avg_per_class_acc': 0.8394709302325583, 'corruption': 'dropout_local', 'level': 1}
{'acc': 0.8286061588330632, 'avg_per_class_acc': 0.7695232558139534, 'corruption': 'dropout_local', 'level': 2}
{'acc': 0.7726904376012966, 'avg_per_class_acc': 0.7092906976744185, 'corruption': 'dropout_local', 'level': 3}
{'acc': 0.6717990275526742, 'avg_per_class_acc': 0.6149418604651162, 'corruption': 'dropout_local', 'level': 4}
{'CE': 0.908, 'OA': 0.812, 'RCE': 0.835, 'corruption': 'dropout_local', 'level': 'Overall'}
{'acc': 0.8905996758508914, 'avg_per_class_acc': 0.8544593023255814, 'corruption': 'add_global', 'level': 0}
{'acc': 0.8472447325769854, 'avg_per_class_acc': 0.7881860465116277, 'corruption': 'add_global', 'level': 1}
{'acc': 0.8107779578606159, 'avg_per_class_acc': 0.7417383720930232, 'corruption': 'add_global', 'level': 2}
{'acc': 0.7548622366288493, 'avg_per_class_acc': 0.6675813953488372, 'corruption': 'add_global', 'level': 3}
{'acc': 0.6977309562398704, 'avg_per_class_acc': 0.6104883720930232, 'corruption': 'add_global', 'level': 4}
{'CE': 0.678, 'OA': 0.8, 'RCE': 0.557, 'corruption': 'add_global', 'level': 'Overall'}
{'acc': 0.8614262560777958, 'avg_per_class_acc': 0.803970930232558, 'corruption': 'add_local', 'level': 0}
{'acc': 0.8399513776337115, 'avg_per_class_acc': 0.776389534883721, 'corruption': 'add_local', 'level': 1}
{'acc': 0.813614262560778, 'avg_per_class_acc': 0.7350639534883721, 'corruption': 'add_local', 'level': 2}
{'acc': 0.7876823338735819, 'avg_per_class_acc': 0.7037848837209302, 'corruption': 'add_local', 'level': 3}
{'acc': 0.7759319286871961, 'avg_per_class_acc': 0.6841220930232559, 'corruption': 'add_local', 'level': 4}
{'CE': 0.669, 'OA': 0.816, 'RCE': 0.532, 'corruption': 'add_local', 'level': 'Overall'}
{'RmCE': 0.532, 'mCE': 0.714, 'mOA': 0.836}