mlcommons / training

Reference implementations of MLPerf™ training benchmarks
https://mlcommons.org/en/groups/training
Apache License 2.0
1.57k stars 549 forks source link

Segmentation preprocess skips from Case 210 until Case 299 #618

Closed mahmoodn closed 1 year ago

mahmoodn commented 1 year ago

Hi Is there any suggestion for the following error in 3d-unet preprocess?

$ docker run --ipc=host -it --rm --runtime=nvidia \
-v /disk1/scratch/data/KiTS19/kits19/data/:/raw_data 
-v /disk1/scratch/preprocessed_data/:/data \
-v /disk1/scratch/results/:/results unet3d:latest /bin/bash
root@7c3f2d9f30da:/workspace/unet3d# python3 preprocess_dataset.py \
--data_dir /raw_data \
--results_dir /results/
Preprocessing /raw_data
Saving case_00000 shape (1, 190, 392, 392) mean [-2.] std [0.85]
Saving case_00001 shape (1, 188, 340, 340) mean [-1.82] std [0.94]
...
Saving case_00208 shape (1, 278, 377, 377) mean [-1.88] std [0.99]
Saving case_00209 shape (1, 318, 333, 333) mean [-1.65] std [1.11]
Case 210. Skipped.
Case 211. Skipped.
...
Case 299. Skipped.
Mean value: -1.850000023841858, std: 0.9800000190734863, d: 256.0, h: 333.0, w: 333.0
Traceback (most recent call last):
  File "preprocess_dataset.py", line 147, in <module>
    verify_dataset(args.results_dir)
  File "preprocess_dataset.py", line 127, in verify_dataset
    assert len(source) == len(os.listdir(results_dir))
AssertionError

The inference has no problem. The 209 and 210 folder are shown below:

root@7c3f2d9f30da:/workspace/unet3d# ls /data/KiTS19/
calibration/ etc/         inference/   reference/   
# ls /raw_data/
LICENSE     case_00027  case_00055  case_00083  case_00111  case_00139  case_00167  case_00195  case_00223  case_00251  case_00279
case_00000  case_00028  case_00056  case_00084  case_00112  case_00140  case_00168  case_00196  case_00224  case_00252  case_00280
case_00001  case_00029  case_00057  case_00085  case_00113  case_00141  case_00169  case_00197  case_00225  case_00253  case_00281
case_00002  case_00030  case_00058  case_00086  case_00114  case_00142  case_00170  case_00198  case_00226  case_00254  case_00282
case_00003  case_00031  case_00059  case_00087  case_00115  case_00143  case_00171  case_00199  case_00227  case_00255  case_00283
case_00004  case_00032  case_00060  case_00088  case_00116  case_00144  case_00172  case_00200  case_00228  case_00256  case_00284
case_00005  case_00033  case_00061  case_00089  case_00117  case_00145  case_00173  case_00201  case_00229  case_00257  case_00285
case_00006  case_00034  case_00062  case_00090  case_00118  case_00146  case_00174  case_00202  case_00230  case_00258  case_00286
case_00007  case_00035  case_00063  case_00091  case_00119  case_00147  case_00175  case_00203  case_00231  case_00259  case_00287
case_00008  case_00036  case_00064  case_00092  case_00120  case_00148  case_00176  case_00204  case_00232  case_00260  case_00288
case_00009  case_00037  case_00065  case_00093  case_00121  case_00149  case_00177  case_00205  case_00233  case_00261  case_00289
case_00010  case_00038  case_00066  case_00094  case_00122  case_00150  case_00178  case_00206  case_00234  case_00262  case_00290
case_00011  case_00039  case_00067  case_00095  case_00123  case_00151  case_00179  case_00207  case_00235  case_00263  case_00291
case_00012  case_00040  case_00068  case_00096  case_00124  case_00152  case_00180  case_00208  case_00236  case_00264  case_00292
case_00013  case_00041  case_00069  case_00097  case_00125  case_00153  case_00181  case_00209  case_00237  case_00265  case_00293
case_00014  case_00042  case_00070  case_00098  case_00126  case_00154  case_00182  case_00210  case_00238  case_00266  case_00294
case_00015  case_00043  case_00071  case_00099  case_00127  case_00155  case_00183  case_00211  case_00239  case_00267  case_00295
case_00016  case_00044  case_00072  case_00100  case_00128  case_00156  case_00184  case_00212  case_00240  case_00268  case_00296
case_00017  case_00045  case_00073  case_00101  case_00129  case_00157  case_00185  case_00213  case_00241  case_00269  case_00297
case_00018  case_00046  case_00074  case_00102  case_00130  case_00158  case_00186  case_00214  case_00242  case_00270  case_00298
case_00019  case_00047  case_00075  case_00103  case_00131  case_00159  case_00187  case_00215  case_00243  case_00271  case_00299
case_00020  case_00048  case_00076  case_00104  case_00132  case_00160  case_00188  case_00216  case_00244  case_00272  kits.json
case_00021  case_00049  case_00077  case_00105  case_00133  case_00161  case_00189  case_00217  case_00245  case_00273
case_00022  case_00050  case_00078  case_00106  case_00134  case_00162  case_00190  case_00218  case_00246  case_00274
case_00023  case_00051  case_00079  case_00107  case_00135  case_00163  case_00191  case_00219  case_00247  case_00275
case_00024  case_00052  case_00080  case_00108  case_00136  case_00164  case_00192  case_00220  case_00248  case_00276
case_00025  case_00053  case_00081  case_00109  case_00137  case_00165  case_00193  case_00221  case_00249  case_00277
case_00026  case_00054  case_00082  case_00110  case_00138  case_00166  case_00194  case_00222  case_00250  case_00278
root@7c3f2d9f30da:/workspace/unet3d# ls /raw_data/case_00210/
imaging.nii.gz
root@7c3f2d9f30da:/workspace/unet3d# ls /raw_data/case_00209/
imaging.nii.gz  segmentation.nii.gz