ljwztc
commented
6 months ago You can use get_loader_without_gt in dataloader.py
Closed msds21024 closed 1 month ago
ljwztc
commented
6 months ago You can use get_loader_without_gt in dataloader.py
msds21024
commented
5 months ago Hi, thank you for your kind response, I used get_loader_without_gt in dataloader.py, but in test.py the validation function is working with those labels.
def validation(model, ValLoader, val_transforms, args): save_dir = '/content/drive/MyDrive/clip/out/' + args.log_name + f'/testhealthp{args.epoch}' print(save_dir) if not os.path.isdir(save_dir): print('im here now') os.makedirs(save_dir) os.makedirs(os.path.join(save_dir, 'predict')) model.eval() dice_list = {} for key in TEMPLATE.keys(): dice_list[key] = np.zeros((2, NUM_CLASS)) # 1st row for dice, 2nd row for count
print('ValLoader' , ValLoader)
for batch in ValLoader:
print("Keys in batch:", batch.keys())
break # Only print keys for the first batch
for index, batch in enumerate(tqdm(ValLoader)):
# print('%d processd' % (index))
image, label, name = batch["image"].cuda(), batch["post_label"], batch["name"]
print(image.shape)
# print(label.shape)
with torch.no_grad():
# with torch.autocast(device_type="cuda", dtype=torch.float16):
pred = sliding_window_inference(image, (args.roi_x, args.roi_y, args.roi_z), 1, model, overlap=0.5, mode='gaussian')
pred_sigmoid = F.sigmoid(pred)
#pred_hard = threshold_organ(pred_sigmoid, organ=args.threshold_organ, threshold=args.threshold)
pred_hard = threshold_organ(pred_sigmoid)
pred_hard = pred_hard.cpu()
torch.cuda.empty_cache()
B = pred_hard.shape[0]
for b in range(B):
content = 'case%s| '%(name[b])
template_key = get_key(name[b])
organ_list = TEMPLATE[template_key]
pred_hard_post = organ_post_process(pred_hard.numpy(), organ_list, args.log_name+'/'+name[0].split('/')[0]+'/'+name[0].split('/')[-1],args)
pred_hard_post = torch.tensor(pred_hard_post)
for organ in organ_list:
if torch.sum(label[b,organ-1,:,:,:].cuda()) != 0:
dice_organ, recall, precision = dice_score(pred_hard_post[b,organ-1,:,:,:].cuda(), label[b,organ-1,:,:,:].cuda())
dice_list[template_key][0][organ-1] += dice_organ.item()
dice_list[template_key][1][organ-1] += 1
content += '%s: %.4f, '%(ORGAN_NAME[organ-1], dice_organ.item())
print('%s: dice %.4f, recall %.4f, precision %.4f.'%(ORGAN_NAME[organ-1], dice_organ.item(), recall.item(), precision.item()))
print(content)
if args.store_result:
pred_sigmoid_store = (pred_sigmoid.cpu().numpy() * 255).astype(np.uint8)
label_store = (label.numpy()).astype(np.uint8)
np.savez_compressed(save_dir + '/predict/' + name[0].split('/')[0] + name[0].split('/')[-1],
pred=pred_sigmoid_store, label=label_store)
### testing phase for this function
one_channel_label_v1, one_channel_label_v2 = merge_label(pred_hard_post, name)
batch['one_channel_label_v1'] = one_channel_label_v1.cpu()
batch['one_channel_label_v2'] = one_channel_label_v2.cpu()
_, split_label = merge_label(batch["post_label"], name)
batch['split_label'] = split_label.cpu()
# print(batch['label'].shape, batch['one_channel_label'].shape)
# print(torch.unique(batch['label']), torch.unique(batch['one_channel_label']))
visualize_label(batch, save_dir + '/output/' + name[0].split('/')[0] , val_transforms)
## load data
# data = np.load('/out/epoch_80/predict/****.npz')
# pred, label = data['pred'], data['label']
torch.cuda.empty_cache()
ave_organ_dice = np.zeros((2, NUM_CLASS))
with open('out/'+args.log_name+f'/test_{args.epoch}.txt', 'w') as f:
for key in TEMPLATE.keys():
organ_list = TEMPLATE[key]
content = 'Task%s| '%(key)
for organ in organ_list:
dice = dice_list[key][0][organ-1] / dice_list[key][1][organ-1]
content += '%s: %.4f, '%(ORGAN_NAME[organ-1], dice)
ave_organ_dice[0][organ-1] += dice_list[key][0][organ-1]
ave_organ_dice[1][organ-1] += dice_list[key][1][organ-1]
print(content)
f.write(content)
f.write('\n')
content = 'Average | '
for i in range(NUM_CLASS):
content += '%s: %.4f, '%(ORGAN_NAME[i], ave_organ_dice[0][i] / ave_organ_dice[1][i])
print(content)
f.write(content)
f.write('\n')
print(np.mean(ave_organ_dice[0] / ave_organ_dice[1]))
f.write('%s: %.4f, '%('average', np.mean(ave_organ_dice[0] / ave_organ_dice[1])))
f.write('\n')
# np.save(save_dir + '/result.npy', dice_list)
# load
# dice_list = np.load(/out/epoch_xxx/result.npy, allow_pickle=True)How do i change that?
Also, can you please explain is it necessary to have dataset with post_labels? I am having a hard time understanding it. I'd be grateful for your response. Thanks in advance!
msds21024
commented
5 months ago And if i manually try to create post_label folder containing post_label000.h5 files after converting label.nii.gz files to .h5 format i face the following error,
Use pretrained weights Total number of parameters: 19416417 train len 1445 val len 106 test len 10 testy Loading dataset: 0% 0/6 [00:18<?, ?it/s] Traceback (most recent call last): File "/usr/local/lib/python3.10/dist-packages/monai/transforms/transform.py", line 141, in apply_transform return _apply_transform(transform, data, unpack_items, lazy, overrides, log_stats) File "/usr/local/lib/python3.10/dist-packages/monai/transforms/transform.py", line 98, in _apply_transform return transform(data, lazy=lazy) if isinstance(transform, LazyTrait) else transform(data) File "/content/drive/MyDrive/clip/dataset/dataloader.py", line 174, in call data = hf['post_label'][()] File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper File "/usr/local/lib/python3.10/dist-packages/h5py/_hl/group.py", line 357, in getitem oid = h5o.open(self.id, self._e(name), lapl=self._lapl) File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper File "h5py/h5o.pyx", line 190, in h5py.h5o.open KeyError: "Unable to open object (object 'post_label' doesn't exist)"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/content/drive/MyDrive/clip/test.py", line 230, in
Kindly, guide me what am I missing?? I'd be grateful
ljwztc
commented
5 months ago you can replace the get_loader with get_loader_without_gt.
duania
commented
5 months ago I am trying to train the model , I face same problems. before and after converting to h5.
Could you please let us know how to generate the data for post_labelsTr folders and how these labels are different from the original labels so we could generate them. I was not able to find the generation of these labels or how they might differ from the original data in the scripts. I would appreciate your help. Thank you.
hi, i am trying to perform inference using unet on just kits19 dataset using test.py, i created the dataset directory as required satisfying following test.txt file
but when i run the following command
! CUDA_VISIBLE_DEVICES=0 python -W ignore test.py --resume ./out/unet.pth --data_root_path /content/drive/MyDrive/clip/data/kits19/ --store_result --cache_dataset --cache_rate 0.6
i get the follwing error
Traceback (most recent call last): File "/usr/local/lib/python3.10/dist-packages/monai/transforms/transform.py", line 141, in apply_transform return _apply_transform(transform, data, unpack_items, lazy, overrides, log_stats) File "/usr/local/lib/python3.10/dist-packages/monai/transforms/transform.py", line 98, in _apply_transform return transform(data, lazy=lazy) if isinstance(transform, LazyTrait) else transform(data) File "/content/drive/MyDrive/clip/dataset/dataloader.py", line 173, in call with h5py.File(post_label_pth, 'r') as hf: File "/usr/local/lib/python3.10/dist-packages/h5py/_hl/files.py", line 567, in init fid = make_fid(name, mode, userblock_size, fapl, fcpl, swmr=swmr) File "/usr/local/lib/python3.10/dist-packages/h5py/_hl/files.py", line 231, in make_fid fid = h5f.open(name, flags, fapl=fapl) File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper File "h5py/h5f.pyx", line 106, in h5py.h5f.open FileNotFoundError: [Errno 2] Unable to open file (unable to open file: name = '/content/drive/MyDrive/clip/data/kits19/05_KiTS/post_label/post_label0010.h5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)
The above exception was the direct cause of the following exception:
Traceback (most recent call last): File "/content/drive/MyDrive/clip/test.py", line 222, in
main()
File "/content/drive/MyDrive/clip/test.py", line 217, in main
test_loader, val_transforms = get_loader(args)
File "/content/drive/MyDrive/clip/dataset/dataloader.py", line 388, in get_loader
test_dataset = CacheDataset(data=data_dicts_test, transform=val_transforms, cache_rate=args.cache_rate)
File "/usr/local/lib/python3.10/dist-packages/monai/data/dataset.py", line 821, in init
self.set_data(data)
File "/usr/local/lib/python3.10/dist-packages/monai/data/dataset.py", line 848, in set_data
self._cache = self._fill_cache(indices)
File "/usr/local/lib/python3.10/dist-packages/monai/data/dataset.py", line 877, in _fill_cache
return list(tqdm(p.imap(self._load_cache_item, indices), total=len(indices), desc="Loading dataset"))
File "/usr/local/lib/python3.10/dist-packages/tqdm/std.py", line 1182, in iter
for obj in iterable:
File "/usr/lib/python3.10/multiprocessing/pool.py", line 873, in next
raise value
File "/usr/lib/python3.10/multiprocessing/pool.py", line 125, in worker
result = (True, func(*args, **kwds))
File "/usr/local/lib/python3.10/dist-packages/monai/data/dataset.py", line 890, in _load_cache_item
item = self.transform(item, end=first_random, threading=True)
File "/usr/local/lib/python3.10/dist-packages/monai/transforms/compose.py", line 322, in call
result = execute_compose(
File "/usr/local/lib/python3.10/dist-packages/monai/transforms/compose.py", line 111, in execute_compose
data = apply_transform(
File "/usr/local/lib/python3.10/dist-packages/monai/transforms/transform.py", line 171, in apply_transform
raise RuntimeError(f"applying transform {transform}") from e
RuntimeError: applying transform <dataset.dataloader.LoadImageh5d object at 0x7e66b1b03070>
I am just trying to perform inference on a small subset of kits19 dataset, how can i modify label_transfer.py to get the post_label files for testing only??