Closed mfkeles closed 1 year ago
talmo
commented
3 years ago Hey Mehmet,
SLEAP should handle this at training time by padding to the size of the largest video.
At what stage do you get this error? When exporting a training package or when training or?
If you don't mind, you could send this specific project file (.slp) to sleap@princeton.edu. We're going to be a little delayed with responses this and next week but we'll check it out as soon as we get a chance. Thanks!
mfkeles
commented
3 years ago Thanks Talmo! It also happens when I try to export it as well as when training.
I have sent the .slp file your way. Appreciate the help!
Best, Mehmet
mfkeles
commented
3 years ago Hi Talmo,
I was wondering if you had a chance to look at this! Thanks so much again!
Best, Mehmet
arie-matsliah
commented
3 years ago Hi @mfkeles Sorry for the late response here we're a bit swamped. Let me try to reproduce the issue and get back to you. I found the file you sent and will let you know soon if I need more info.
Thanks
arie-matsliah
commented
3 years ago Hi @mfkeles Can you please email a self-contained folder (with images included) and precise steps to reproduce this bug? I have tried reproducing from the labels file alone (by using sample images of variable sizes) but could not (both exporting and training worked fine).
Thanks
mfkeles
commented
3 years ago Hey @ariematsliah-princeton thanks for the help again!
I think I found the problem. Most of the labels that I imported from DLC labels were fine but there was a set that had labels outside image for some reason. The individual points were occluded ones so they wouldn't even show up when using DLC. When I took out that set of images, I was able to export it. However, I ran into a different issue. This might be related to the fact that my training dataset is composed of images with different dimensions. I am not sure but I was using google colab to train and I got the following error. Let me know if this is something that I can fix! I would be happy to send you the package. Thanks so much !
```python
2021-05-24 21:00:17.239067: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
INFO:sleap.nn.training:Versions:
SLEAP: 1.1.3
TensorFlow: 2.3.1
Numpy: 1.18.5
Python: 3.7.10
OS: Linux-5.4.109+-x86_64-with-Ubuntu-18.04-bionic
INFO:sleap.nn.training:Training labels file: colab.pkg.slp
INFO:sleap.nn.training:Training profile: single_instance.json
INFO:sleap.nn.training:
INFO:sleap.nn.training:Arguments:
INFO:sleap.nn.training:{
"training_job_path": "single_instance.json",
"labels_path": "colab.pkg.slp",
"video_paths": "",
"val_labels": null,
"test_labels": null,
"tensorboard": false,
"save_viz": false,
"zmq": false,
"run_name": "",
"prefix": "",
"suffix": ""
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Training job:
INFO:sleap.nn.training:{
"data": {
"labels": {
"training_labels": null,
"validation_labels": null,
"validation_fraction": 0.1,
"test_labels": null,
"split_by_inds": false,
"training_inds": null,
"validation_inds": null,
"test_inds": null,
"search_path_hints": [],
"skeletons": []
},
"preprocessing": {
"ensure_rgb": false,
"ensure_grayscale": true,
"imagenet_mode": null,
"input_scaling": 0.75,
"pad_to_stride": null,
"resize_and_pad_to_target": true,
"target_height": null,
"target_width": null
},
"instance_cropping": {
"center_on_part": null,
"crop_size": null,
"crop_size_detection_padding": 16
}
},
"model": {
"backbone": {
"leap": null,
"unet": {
"stem_stride": null,
"max_stride": 64,
"output_stride": 4,
"filters": 32,
"filters_rate": 1.5,
"middle_block": true,
"up_interpolate": true,
"stacks": 1
},
"hourglass": null,
"resnet": null,
"pretrained_encoder": null
},
"heads": {
"single_instance": {
"part_names": null,
"sigma": 5.0,
"output_stride": 4,
"offset_refinement": false
},
"centroid": null,
"centered_instance": null,
"multi_instance": null
}
},
"optimization": {
"preload_data": true,
"augmentation_config": {
"rotate": true,
"rotation_min_angle": -180.0,
"rotation_max_angle": 180.0,
"translate": false,
"translate_min": -5,
"translate_max": 5,
"scale": false,
"scale_min": 0.9,
"scale_max": 1.1,
"uniform_noise": false,
"uniform_noise_min_val": 0.0,
"uniform_noise_max_val": 10.0,
"gaussian_noise": false,
"gaussian_noise_mean": 5.0,
"gaussian_noise_stddev": 1.0,
"contrast": false,
"contrast_min_gamma": 0.5,
"contrast_max_gamma": 2.0,
"brightness": false,
"brightness_min_val": 0.0,
"brightness_max_val": 10.0,
"random_crop": false,
"random_crop_height": 256,
"random_crop_width": 256,
"random_flip": false,
"flip_horizontal": true
},
"online_shuffling": true,
"shuffle_buffer_size": 128,
"prefetch": true,
"batch_size": 4,
"batches_per_epoch": null,
"min_batches_per_epoch": 200,
"val_batches_per_epoch": null,
"min_val_batches_per_epoch": 10,
"epochs": 200,
"optimizer": "adam",
"initial_learning_rate": 0.0001,
"learning_rate_schedule": {
"reduce_on_plateau": true,
"reduction_factor": 0.5,
"plateau_min_delta": 1e-06,
"plateau_patience": 5,
"plateau_cooldown": 3,
"min_learning_rate": 1e-08
},
"hard_keypoint_mining": {
"online_mining": false,
"hard_to_easy_ratio": 2.0,
"min_hard_keypoints": 2,
"max_hard_keypoints": null,
"loss_scale": 5.0
},
"early_stopping": {
"stop_training_on_plateau": true,
"plateau_min_delta": 1e-06,
"plateau_patience": 10
}
},
"outputs": {
"save_outputs": true,
"run_name": "210524_165811",
"run_name_prefix": "",
"run_name_suffix": ".single_instance",
"runs_folder": "",
"tags": [
""
],
"save_visualizations": true,
"delete_viz_images": true,
"zip_outputs": false,
"log_to_csv": true,
"checkpointing": {
"initial_model": false,
"best_model": true,
"every_epoch": false,
"latest_model": false,
"final_model": false
},
"tensorboard": {
"write_logs": false,
"loss_frequency": "epoch",
"architecture_graph": false,
"profile_graph": false,
"visualizations": true
},
"zmq": {
"subscribe_to_controller": false,
"controller_address": "tcp://127.0.0.1:9000",
"controller_polling_timeout": 10,
"publish_updates": false,
"publish_address": "tcp://127.0.0.1:9001"
}
},
"name": "",
"description": "",
"sleap_version": "1.1.3",
"filename": "single_instance.json"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:System:
2021-05-24 21:00:18.660443: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
2021-05-24 21:00:18.674797: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:18.675396: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
pciBusID: 0000:00:04.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.90GiB deviceMemoryBandwidth: 681.88GiB/s
2021-05-24 21:00:18.675447: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-24 21:00:18.677002: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
2021-05-24 21:00:18.678829: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
2021-05-24 21:00:18.679166: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
2021-05-24 21:00:18.681123: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
2021-05-24 21:00:18.682031: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
2021-05-24 21:00:18.686351: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
2021-05-24 21:00:18.686490: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:18.687110: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:18.687665: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
GPUs: 1/1 available
Device: /physical_device:GPU:0
Available: True
Initalized: False
Memory growth: True
INFO:sleap.nn.training:
INFO:sleap.nn.training:Initializing trainer...
INFO:sleap.nn.training:Loading training labels from: colab.pkg.slp
INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1
INFO:sleap.nn.training: Splits: Training = 362 / Validation = 40.
INFO:sleap.nn.training:Setting up for training...
INFO:sleap.nn.training:Setting up pipeline builders...
INFO:sleap.nn.training:Setting up model...
INFO:sleap.nn.training:Building test pipeline...
2021-05-24 21:00:18.897594: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-24 21:00:18.902155: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2000170000 Hz
2021-05-24 21:00:18.902427: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55807278c8c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-24 21:00:18.902462: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
2021-05-24 21:00:18.995227: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:18.996108: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55807278ca80 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2021-05-24 21:00:18.996143: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0
2021-05-24 21:00:18.996384: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:18.996925: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
pciBusID: 0000:00:04.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.90GiB deviceMemoryBandwidth: 681.88GiB/s
2021-05-24 21:00:18.997007: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-24 21:00:18.997054: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
2021-05-24 21:00:18.997081: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
2021-05-24 21:00:18.997108: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
2021-05-24 21:00:18.997132: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
2021-05-24 21:00:18.997155: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
2021-05-24 21:00:18.997180: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
2021-05-24 21:00:18.997280: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:18.997855: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:18.998383: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
2021-05-24 21:00:18.998464: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-24 21:00:19.436736: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
2021-05-24 21:00:19.436792: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263] 0
2021-05-24 21:00:19.436804: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0: N
2021-05-24 21:00:19.437030: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:19.437705: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2021-05-24 21:00:19.438263: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14958 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0)
INFO:sleap.nn.training:Loaded test example. [2.252s]
INFO:sleap.nn.training: Input shape: (704, 1024, 1)
INFO:sleap.nn.training:Created Keras model.
INFO:sleap.nn.training: Backbone: UNet(stacks=1, filters=32, filters_rate=1.5, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=6, middle_block=True, up_blocks=4, up_interpolate=True, block_contraction=False)
INFO:sleap.nn.training: Max stride: 64
INFO:sleap.nn.training: Parameters: 6,786,842
INFO:sleap.nn.training: Heads:
INFO:sleap.nn.training: [0] = SingleInstanceConfmapsHead(part_names=['a6', 'a5', 'atip', 'headtip', 'head_r', 'head_l', 'thor_ant', 'thor_post', 't1_r_tip', 'joint1_r', 'joint1_rmid', 'joint1_rtop', 't1_l_tip', 'joint1_l', 'joint1_lmid', 'joint1_ltop', 't2_r_tip', 'joint2_r', 'joint2_rmid', 'joint2_rtop', 't2_l_tip', 'joint2_l', 'joint2_lmid', 'joint2_ltop', 't3_r_tip', 'joint3_r', 'joint_3rmid', 'joint3_rtop', 't3_l_tip', 'joint3_l', 'joint_3lmid', 'joint3_ltop', 'halt_r', 'halt_l', 'prob'], sigma=5.0, output_stride=4, loss_weight=1.0)
INFO:sleap.nn.training: Outputs:
INFO:sleap.nn.training: [0] = Tensor("SingleInstanceConfmapsHead_0/BiasAdd:0", shape=(None, 176, 256, 35), dtype=float32)
INFO:sleap.nn.training:Setting up data pipelines...
INFO:sleap.nn.training:Training set: n = 362
INFO:sleap.nn.training:Validation set: n = 40
INFO:sleap.nn.training:Setting up optimization...
INFO:sleap.nn.training: Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-06, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08)
INFO:sleap.nn.training: Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-06, plateau_patience=10)
INFO:sleap.nn.training:Setting up outputs...
INFO:sleap.nn.training:Created run path: 210524_165811.single_instance
INFO:sleap.nn.training:Setting up visualization...
Unable to use Qt backend for matplotlib. This probably means Qt is running headless.
INFO:sleap.nn.training:Finished trainer set up. [3.7s]
INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation...
INFO:sleap.nn.training:Finished creating training datasets. [10.8s]
INFO:sleap.nn.training:Starting training loop...
Epoch 1/200
2021-05-24 21:00:35.289608: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
2021-05-24 21:00:36.371513: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0530s vs `on_train_batch_end` time: 0.1285s). Check your callbacks.
Traceback (most recent call last):
File "/usr/local/bin/sleap-train", line 8, in
arie-matsliah
commented
3 years ago Do you mind sharing the exported package or other means to reproduce the same error?
mfkeles
commented
3 years ago mfkeles
commented
3 years ago Hi @ariematsliah-princeton I was wondering if you had a chance to look at this! Thanks so much!
Best, -M
talmo
commented
3 years ago Hey @mfkeles,
@ariematsliah-princeton is out of the office for a few more days, but we'll try to get back to you as soon as possible. Sorry about the delay!!
mfkeles
commented
3 years ago Just checking in to see if there are any updates on this. Thanks,
-mk
arie-matsliah
commented
3 years ago Hi @mfkeles Just wanted to update you that I'm still debugging this issue. Will update once root cause is found. Best
talmo
commented
3 years ago talmo
commented
2 years ago Closing this issue due to inactivity but please feel free to comment again if you're still having issues and we'll reopen it.
cosimogonnelli
commented
2 years ago Hi @talmo,
I got the same error as mentioned here. These are my steps:
Did you guys found any solution to this issue?
Thanks!
MY INFO + TERMINAL OUTPUT
Software versions:
SLEAP: 1.2.6
TensorFlow: 2.8.0
Numpy: 1.21.5
Python: 3.7.5
OS: Darwin-21.1.0-x86_64-i386-64bit
Happy SLEAPing! :)
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/gui/learning/dialog.py", line 684, in export_package
suggested=include_suggestions,
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/gui/commands.py", line 1250, in export_dataset_gui
progress_callback=update_progress,
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/io/dataset.py", line 1971, in save_file
write(filename, labels, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/io/format/main.py", line 162, in write
return disp.write(filename, source_object, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/io/format/dispatch.py", line 78, in write
return adaptor.write(filename, source_object, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/io/format/hdf5.py", line 253, in write
progress_callback=progress_callback,
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/io/dataset.py", line 2258, in save_frame_data_hdf5
frame_numbers=frame_nums,
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/io/video.py", line 1413, in to_hdf5
frame_data = self.get_frames(frame_numbers)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sleap/io/video.py", line 1101, in get_frames
return np.stack([self.get_frame(idx) for idx in idxs], axis=0)
File "<__array_function__ internals>", line 6, in stack
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/numpy/core/shape_base.py", line 426, in stack
raise ValueError('all input arrays must have the same shape')
ValueError: all input arrays must have the same shape
roomrys
commented
2 years ago Hi @cosimogonnelli,
We did not get a fix yet. Can you share your data with lmaree@salk.edu please.
Thanks, Liezl
roomrys
commented
1 year ago I am closing this issue again due to inactivity. If you experience this issue please comment and I will reopen it. Also, we are having trouble recreating the issue, so data for a minimal working example to lmaree@salk.edu would be much appreciated. Thanks!
I have video data with different image sizes that I recorded using different cameras. I was wondering how sleap deals with that. I suspect the input image size is the problem since I get the "ValueError: all input arrays must have the same shape" error.
An example image is below. It's pretty large in size (1300x900) and I have other images that are slightly smaller in size (1100x800).
Thank you! Mehmet