search

Project-MONAI / MONAI

AI Toolkit for Healthcare Imaging
https://monai.io/
Apache License 2.0
5.86k stars 1.09k forks source link

CacheDataset with .nrrd files throws error -> itk and swig #3014

Closed jpcenteno80 closed 2 years ago

jpcenteno80 commented 3 years ago

Describe the bug When working with .nrrd files, CacheDataset throws an error during instantiation. However, if working in a jupyter notebook, executing the CacheDataset instantiation cell again runs with no problems. Using monai.data.Dataset and monai.data.PersistentDataset work as expected.

To Reproduce

import SimpleITK as sitk, numpy as np
from monai.transforms import Compose, LoadImaged, AddChanneld, Spacingd
from monai.data import CacheDataset

# create test data
for i in range(10):
    img = sitk.Image(64, 64, 32, sitk.sitkInt16)
    seg = sitk.Image(64, 64, 32, sitk.sitkUInt8)

   # tested with default voxel size (1,1,1) and these random values
    x_px = np.round(np.random.uniform(.5), 1)
    z_px = np.round(np.random.uniform(.5), 1)
    spacing = (x_px, x_px, z_px)

    img.SetSpacing(spacing)
    seg.SetSpacing(spacing)    

    sitk.WriteImage(img ,f'tmp/img{i}.nrrd')
    sitk.WriteImage(seg ,f'tmp/seg{i}.nrrd')

datalist = [{'image': f'tmp/img{i}.nrrd', 'label': f'tmp/seg{i}.nrrd'} for i in range(10)]

keys = ['image', 'label']
transforms = Compose([LoadImaged(keys),
                      AddChanneld(keys),
                      Spacingd(keys, pixdim=(2,-1,-1))])

ds_cache = CacheDataset(datalist, transforms)

# error
Loading dataset:  10%|████████████████▌                                                                                                                                                     | 1/10 [00:00<00:00,  9.38it/s]
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/transforms/transform.py in apply_transform(transform, data, map_items, unpack_items)
     83             return [_apply_transform(transform, item, unpack_items) for item in data]
---> 84         return _apply_transform(transform, data, unpack_items)
     85     except Exception as e:

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/transforms/transform.py in _apply_transform(transform, parameters, unpack_parameters)
     51 
---> 52     return transform(parameters)
     53 

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/transforms/io/dictionary.py in __call__(self, data, reader)
    112         for key, meta_key, meta_key_postfix in self.key_iterator(d, self.meta_keys, self.meta_key_postfix):
--> 113             data = self._loader(d[key], reader)
    114             if self._loader.image_only:

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/transforms/io/array.py in __call__(self, filename, reader)
    162 
--> 163         img = reader.read(filename)
    164         img_array, meta_data = reader.get_data(img)

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/data/image_reader.py in read(self, data, **kwargs)
    195 
--> 196             img_.append(itk.imread(name, **kwargs_))
    197         return img_ if len(filenames) > 1 else img_[0]

~/miniconda3/envs/deployment/lib/python3.9/site-packages/itkExtras.py in imread(filename, pixel_type, fallback_only)
    627     else:
--> 628         TemplateReaderType = itk.ImageFileReader
    629         io_filename=filename

~/miniconda3/envs/deployment/lib/python3.9/site-packages/itkLazy.py in __getattribute__(self, attr)
     51             namespace = {}
---> 52             itkBase.LoadModule(module, namespace)
     53             self.loaded_lazy_modules.add(module)

~/miniconda3/envs/deployment/lib/python3.9/site-packages/itkBase.py in LoadModule(name, namespace)
     60             swig = namespace.setdefault('swig', {})
---> 61             swig.update(this_module.swig)
     62 

AttributeError: module 'itk.ITKIOImageBasePython' has no attribute 'swig'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
/tmp/ipykernel_27656/673064276.py in <module>
----> 1 ds_cache = CacheDataset(datalist, transforms)

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/data/dataset.py in __init__(self, data, transform, cache_num, cache_rate, num_workers, progress)
    577         if self.num_workers is not None:
    578             self.num_workers = max(int(self.num_workers), 1)
--> 579         self._cache: List = self._fill_cache()
    580 
    581     def set_data(self, data: Sequence):

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/data/dataset.py in _fill_cache(self)
    598         with ThreadPool(self.num_workers) as p:
    599             if self.progress and has_tqdm:
--> 600                 return list(
    601                     tqdm(
    602                         p.imap(self._load_cache_item, range(self.cache_num)),

~/miniconda3/envs/deployment/lib/python3.9/site-packages/tqdm/std.py in __iter__(self)
   1183 
   1184         try:
-> 1185             for obj in iterable:
   1186                 yield obj
   1187                 # Update and possibly print the progressbar.

~/miniconda3/envs/deployment/lib/python3.9/multiprocessing/pool.py in next(self, timeout)
    868         if success:
    869             return value
--> 870         raise value
    871 
    872     __next__ = next                    # XXX

~/miniconda3/envs/deployment/lib/python3.9/multiprocessing/pool.py in worker(inqueue, outqueue, initializer, initargs, maxtasks, wrap_exception)
    123         job, i, func, args, kwds = task
    124         try:
--> 125             result = (True, func(*args, **kwds))
    126         except Exception as e:
    127             if wrap_exception and func is not _helper_reraises_exception:

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/data/dataset.py in _load_cache_item(self, idx)
    618                 break
    619             _xform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform
--> 620             item = apply_transform(_xform, item)
    621         return item
    622 

~/miniconda3/envs/deployment/lib/python3.9/site-packages/monai/transforms/transform.py in apply_transform(transform, data, map_items, unpack_items)
    106             else:
    107                 _log_stats(data=data)
--> 108         raise RuntimeError(f"applying transform {transform}") from e
    109 
    110 

RuntimeError: applying transform <monai.transforms.io.dictionary.LoadImaged object at 0x7f762c05b4c0>

Expected behavior Should not throw error. Training code not in jupyter but as script, so not allowed to execute problem cell twice.

Screenshots If applicable, add screenshots to help explain your problem.

Environment

MONAI version: 0.6.0
Numpy version: 1.20.1
Pytorch version: 1.9.0+cu102
MONAI flags: HAS_EXT = False, USE_COMPILED = False
MONAI rev id: 0ad9e73639e30f4f1af5a1f4a45da9cb09930179

Optional dependencies:
Pytorch Ignite version: 0.4.5
Nibabel version: 3.2.1
scikit-image version: 0.18.3
Pillow version: 8.3.1
Tensorboard version: 2.6.0
gdown version: 3.13.0
TorchVision version: 0.10.0+cu102
ITK version: 5.1.2
tqdm version: 4.62.2
lmdb version: 1.2.1
psutil version: 5.8.0
pandas version: 1.3.1
einops version: 0.3.2

For details about installing the optional dependencies, please visit:
    https://docs.monai.io/en/latest/installation.html#installing-the-recommended-dependencies

Additional context Add any other context about the problem here.

Nic-Ma commented 3 years ago

I think the error mesage is: AttributeError: module 'itk.ITKIOImageBasePython' has no attribute 'swig'. @thewtex Could you please help share any comments here?

Thanks in advance.

wyli commented 2 years ago

I think this has been addressed in the recent versions: https://github.com/InsightSoftwareConsortium/ITK/blob/a89145bccda6a36f42cfdd45d3a6b27234ff54fe/Wrapping/Generators/Python/itk/support/base.py#L71-L72 I'm closing this ticket for now. Please feel free if it's still an issue after updating itk.

DAC-ACD commented 1 year ago

@wyli Hello! Have the same error. Using Monai-Label in 3D Slicer: monai 1.0.1 pypi_0 pypi monailabel 0.5.2 pypi_0 pypi When I trigger training process in Slicer I have this error written in Anaconda Powershell Promt:

[2022-12-03 02:43:13,947] [13744] [MainThread] [INFO] (monailabel.tasks.train.basic_train:608) - 0 - Load Path C:\Users\HYPERPC\radiology\model\pretrained_deepedit_dynunet.pt Loading dataset: 0%| | 0/1 [00:00<?, ?it/s] Loading dataset: 0%| | 0/1 [00:11<?, ?it/s] Traceback (most recent call last): File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\transform.py", line 91, in apply_transform return _apply_transform(transform, data, unpack_items) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\transform.py", line 55, in _apply_transform return transform(parameters) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\io\dictionary.py", line 154, in call data = self._loader(d[key], reader) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\io\array.py", line 271, in call meta_data = switch_endianness(meta_data, "<") File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\io\array.py", line 84, in switch_endianness data = {k: switch_endianness(v, new) for k, v in data.items()} File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\io\array.py", line 84, in data = {k: switch_endianness(v, new) for k, v in data.items()} File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\io\array.py", line 86, in switch_endianness raise RuntimeError(f"Unknown type: {type(data).name}") RuntimeError: Unknown type: itkMatrixF44 The above exception was the direct cause of the following exception: Traceback (most recent call last): File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\runpy.py", line 197, in _run_module_as_main return _run_code(code, main_globals, None, File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\runpy.py", line 87, in _run_code exec(code, run_globals) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\interfaces\utils\app.py", line 132, in run_main() File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\interfaces\utils\app.py", line 117, in run_main result = a.train(request) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\interfaces\app.py", line 421, in train result = task(request, self.datastore()) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\tasks\train\basic_train.py", line 396, in call res = self.train(0, world_size, req, datalist) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\tasks\train\basic_train.py", line 454, in train context.trainer = self._create_trainer(context) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\tasks\train\basic_train.py", line 592, in _create_trainer train_data_loader=self.train_data_loader(context), File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\tasks\train\basic_train.py", line 227, in train_data_loader dataset, datalist = self._dataset(context, context.train_datalist) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monailabel\tasks\train\basic_train.py", line 202, in _dataset else SmartCacheDataset(datalist, transforms, replace_rate) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\data\dataset.py", line 979, in init super().init(data, transform, cache_num, cache_rate, num_init_workers, progress, copy_cache, as_contiguous) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\data\dataset.py", line 793, in init self.set_data(data) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\data\dataset.py", line 1013, in set_data super().set_data(data) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\data\dataset.py", line 818, in set_data self._cache = _compute_cache() File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\data\dataset.py", line 807, in _compute_cache return self._fill_cache() File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\data\dataset.py", line 827, in _fill_cache return list( File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\tqdm\std.py", line 1195, in iter for obj in iterable: File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\multiprocessing\pool.py", line 870, in next raise value File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\multiprocessing\pool.py", line 125, in worker result = (True, func(*args, **kwds)) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\data\dataset.py", line 847, in _load_cache_item item = apply_transform(_xform, item) File "C:\Users\HYPERPC\anaconda3\envs\ColonCancer\lib\site-packages\monai\transforms\transform.py", line 118, in apply_transform raise RuntimeError(f"applying transform {transform}") from e RuntimeError: applying transform <monai.transforms.io.dictionary.LoadImaged object at 0x0000010B12384640> [2022-12-03 02:43:26,061] [11536] [ThreadPoolExecutor-2_0] [INFO] (monailabel.utils.async_tasks.utils:77) - Return code: 1

So, as written here first problem "Unknown type: itkMatrixF44" cause transform problem. As You recommended, I updated some packages (last version): swig 4.1.0 itk 5.3.0 pypi_0 pypi itk-core 5.3.0 pypi_0 pypi itk-filtering 5.3.0 pypi_0 pypi itk-io 5.3.0 pypi_0 pypi itk-numerics 5.3.0 pypi_0 pypi itk-registration 5.3.0 pypi_0 pypi itk-segmentation 5.3.0 pypi_0 pypi simpleitk 2.2.0 pypi_0 pypi

Then had absolutely same error. Additional info: all recommended dependencies were built from git, according to Your documentation https://docs.monai.io/en/latest/installation.html#installing-the-recommended-dependencies

P.s. I'm so sorry for uncomfortable error typing, I've just registered in GitHub to ask you.

wyli commented 1 year ago

Sure, it's this issue according to your log https://github.com/Project-MONAI/MONAI/issues/5603, please try 'pip install monai==1.1.0rc1', this is a pre-release that has a patch for it.

DAC-ACD commented 1 year ago

Thank You so much, it works! Now I'll try to find the way to download seg.nii completed earlier on server to train...