search

talmolab / sleap

A deep learning framework for multi-animal pose tracking.
https://sleap.ai
Other
412 stars 97 forks source link

ValueError with pretrained model training only #1729

Closed jmarkow closed 3 months ago

jmarkow commented 3 months ago

Sleap version 1.3.3. Running sleap-train CLI in an HPC environment (CentOS). Videos are 8 bit grayscale FFV1-encoded AVI files. Training with the Unet backbone works fine, however the pretrained encoder appears to have rounding errors when creating the backbone. Looks similar to issue #826 , but I'm running with the "auto-crop" option enabled AFAIK.

Here's the training json file.

{
    "data": {
        "labels": {
            "training_labels": "../_labels/merged_labels_Basler-267601741000-24383488.slp",
            "validation_labels": null,
            "validation_fraction": 0.2,
            "test_labels": null,
            "split_by_inds": false,
            "training_inds": null,
            "validation_inds": null,
            "test_inds": null,
            "search_path_hints": [
                "../../training_data/Basler-267601746223-24404515",
                "../../training_data/Basler-267601746219-24404505",
                "../../training_data/Basler-2676017611A4-24514980",
                "../../training_data/Basler-26760175421F-24461855",
                "../../training_data/Basler-267601741000-24383488"
            ],
            "skeletons": []
        },
        "preprocessing": {
            "ensure_rgb": false,
            "ensure_grayscale": false,
            "imagenet_mode": null,
            "input_scaling": 1.0,
            "pad_to_stride": 16,
            "resize_and_pad_to_target": true,
            "target_height": 1023,
            "target_width": 1008
        },
        "instance_cropping": {
            "center_on_part": null,
            "crop_size": null,
            "crop_size_detection_padding": 16
        }
    },
    "model": {
        "backbone": {
            "leap": null,
            "unet": null,
            "hourglass": null,
            "resnet": null,
            "pretrained_encoder": {
                "encoder": "densenet121",
                "pretrained": true,
                "decoder_filters": 256,
                "decoder_filters_rate": 1.0,
                "output_stride": 2,
                "decoder_batchnorm": true
            }
        },
        "heads": {
            "single_instance": {
                "part_names": [
                    "nose",
                    "ear_L",
                    "ear_R",
                    "foreleg_L",
                    "foreleg_R",
                    "hindleg_L",
                    "hindleg_R",
                    "back_top",
                    "back_middle",
                    "back_bottom",
                    "tail_base",
                    "tail_middle",
                    "tail_tip"
                ],
                "sigma": 2.5,
                "output_stride": 4,
                "loss_weight": 1.0,
                "offset_refinement": false
            },
            "centroid": null,
            "centered_instance": null,
            "multi_instance": null,
            "multi_class_bottomup": null,
            "multi_class_topdown": null
        },
        "base_checkpoint": null
    },
    "optimization": {
        "preload_data": true,
        "augmentation_config": {
            "rotate": true,
            "rotation_min_angle": -30,
            "rotation_max_angle": 30,
            "translate": true,
            "translate_min": -100,
            "translate_max": 100,
            "scale": true,
            "scale_min": 0.7,
            "scale_max": 1.3,
            "uniform_noise": false,
            "uniform_noise_min_val": 0.0,
            "uniform_noise_max_val": 10.0,
            "gaussian_noise": true,
            "gaussian_noise_mean": 5.0,
            "gaussian_noise_stddev": 5.0,
            "contrast": true,
            "contrast_min_gamma": 0.5,
            "contrast_max_gamma": 2.0,
            "brightness": true,
            "brightness_min_val": -30,
            "brightness_max_val": 30,
            "random_crop": false,
            "random_crop_height": 256,
            "random_crop_width": 256,
            "random_flip": false,
            "flip_horizontal": false
        },
        "online_shuffling": true,
        "shuffle_buffer_size": 128,
        "prefetch": true,
        "batch_size": 4,
        "batches_per_epoch": 200,
        "min_batches_per_epoch": 400,
        "val_batches_per_epoch": 10,
        "min_val_batches_per_epoch": 10,
        "epochs": 100,
        "optimizer": "adam",
        "initial_learning_rate": 0.0001,
        "learning_rate_schedule": {
            "reduce_on_plateau": true,
            "reduction_factor": 0.5,
            "plateau_min_delta": 1e-06,
            "plateau_patience": 5,
            "plateau_cooldown": 3,
            "min_learning_rate": 1e-08
        },
        "hard_keypoint_mining": {
            "online_mining": false,
            "hard_to_easy_ratio": 2.0,
            "min_hard_keypoints": 2,
            "max_hard_keypoints": null,
            "loss_scale": 5.0
        },
        "early_stopping": {
            "stop_training_on_plateau": true,
            "plateau_min_delta": 1e-08,
            "plateau_patience": 10
        }
    },
    "outputs": {
        "save_outputs": true,
        "run_name": "unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0",
        "run_name_prefix": "",
        "run_name_suffix": ".single_instance",
        "runs_folder": "/Users/jeffreymarkowitz/Desktop/basler_plexiglass_sleap_training/labeling_project/models",
        "tags": [
            ""
        ],
        "save_visualizations": true,
        "delete_viz_images": true,
        "zip_outputs": false,
        "log_to_csv": true,
        "checkpointing": {
            "initial_model": false,
            "best_model": true,
            "every_epoch": false,
            "latest_model": false,
            "final_model": false
        },
        "tensorboard": {
            "write_logs": false,
            "loss_frequency": "epoch",
            "architecture_graph": false,
            "profile_graph": false,
            "visualizations": true
        },
        "zmq": {
            "subscribe_to_controller": true,
            "controller_address": "tcp://127.0.0.1:9000",
            "controller_polling_timeout": 10,
            "publish_updates": true,
            "publish_address": "tcp://127.0.0.1:9001"
        }
    },
    "name": "",
    "description": "",
    "sleap_version": "1.3.3",
    "filename": "single_instance.json"
}

and log from run,

---------------------------------------
Begin Slurm Prolog: Apr-03-2024 11:36:26
Job ID:    5503480
User ID:   jmarkowitz30
Account:   gts-jmarkowitz30-paid
Job name:  wrap
Partition: gpu-rtx6000
QOS:       inferno
---------------------------------------
INFO:sleap.nn.training:Versions:
SLEAP: 1.3.3
TensorFlow: 2.7.0
Numpy: 1.19.5
Python: 3.7.12
OS: Linux-3.10.0-1160.49.1.el7.x86_64-x86_64-with-redhat-7.9-Maipo
INFO:sleap.nn.training:Training labels file: None
INFO:sleap.nn.training:Training profile: /storage/coda1/p-jmarkowitz30/0/shared/active_lab_members/markowitz_jeffrey/active_projects/keypoints_basler_nir_plexiglass_arena/sleap_training/gridsearch_pretrained_1/unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0.json
INFO:sleap.nn.training:
INFO:sleap.nn.training:Arguments:
INFO:sleap.nn.training:{
    "training_job_path": "/storage/coda1/p-jmarkowitz30/0/shared/active_lab_members/markowitz_jeffrey/active_projects/keypoints_basler_nir_plexiglass_arena/sleap_training/gridsearch_pretrained_1/unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0.json",
    "labels_path": null,
    "video_paths": [
        ""
    ],
    "val_labels": null,
    "test_labels": null,
    "base_checkpoint": null,
    "tensorboard": false,
    "save_viz": false,
    "zmq": false,
    "run_name": "",
    "prefix": "",
    "suffix": "",
    "cpu": false,
    "first_gpu": false,
    "last_gpu": false,
    "gpu": "auto"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Training job:
INFO:sleap.nn.training:{
    "data": {
        "labels": {
            "training_labels": "../_labels/merged_labels_Basler-267601741000-24383488.slp",
            "validation_labels": null,
            "validation_fraction": 0.2,
            "test_labels": null,
            "split_by_inds": false,
            "training_inds": null,
            "validation_inds": null,
            "test_inds": null,
            "search_path_hints": [
                "../../training_data/Basler-267601746223-24404515",
                "../../training_data/Basler-267601746219-24404505",
                "../../training_data/Basler-2676017611A4-24514980",
                "../../training_data/Basler-26760175421F-24461855",
                "../../training_data/Basler-267601741000-24383488"
            ],
            "skeletons": []
        },
        "preprocessing": {
            "ensure_rgb": false,
            "ensure_grayscale": false,
            "imagenet_mode": null,
            "input_scaling": 1.0,
            "pad_to_stride": 16,
            "resize_and_pad_to_target": true,
            "target_height": 1023,
            "target_width": 1008
        },
        "instance_cropping": {
            "center_on_part": null,
            "crop_size": null,
            "crop_size_detection_padding": 16
        }
    },
    "model": {
        "backbone": {
            "leap": null,
            "unet": null,
            "hourglass": null,
            "resnet": null,
            "pretrained_encoder": {
                "encoder": "densenet121",
                "pretrained": true,
                "decoder_filters": 256,
                "decoder_filters_rate": 1.0,
                "output_stride": 2,
                "decoder_batchnorm": true
            }
        },
        "heads": {
            "single_instance": {
                "part_names": [
                    "nose",
                    "ear_L",
                    "ear_R",
                    "foreleg_L",
                    "foreleg_R",
                    "hindleg_L",
                    "hindleg_R",
                    "back_top",
                    "back_middle",
                    "back_bottom",
                    "tail_base",
                    "tail_middle",
                    "tail_tip"
                ],
                "sigma": 2.5,
                "output_stride": 4,
                "loss_weight": 1.0,
                "offset_refinement": false
            },
            "centroid": null,
            "centered_instance": null,
            "multi_instance": null,
            "multi_class_bottomup": null,
            "multi_class_topdown": null
        },
        "base_checkpoint": null
    },
    "optimization": {
        "preload_data": true,
        "augmentation_config": {
            "rotate": true,
            "rotation_min_angle": -30.0,
            "rotation_max_angle": 30.0,
            "translate": true,
            "translate_min": -100,
            "translate_max": 100,
            "scale": true,
            "scale_min": 0.7,
            "scale_max": 1.3,
            "uniform_noise": false,
            "uniform_noise_min_val": 0.0,
            "uniform_noise_max_val": 10.0,
            "gaussian_noise": true,
            "gaussian_noise_mean": 5.0,
            "gaussian_noise_stddev": 5.0,
            "contrast": true,
            "contrast_min_gamma": 0.5,
            "contrast_max_gamma": 2.0,
            "brightness": true,
            "brightness_min_val": -30.0,
            "brightness_max_val": 30.0,
            "random_crop": false,
            "random_crop_height": 256,
            "random_crop_width": 256,
            "random_flip": false,
            "flip_horizontal": false
        },
        "online_shuffling": true,
        "shuffle_buffer_size": 128,
        "prefetch": true,
        "batch_size": 4,
        "batches_per_epoch": 200,
        "min_batches_per_epoch": 400,
        "val_batches_per_epoch": 10,
        "min_val_batches_per_epoch": 10,
        "epochs": 100,
        "optimizer": "adam",
        "initial_learning_rate": 0.0001,
        "learning_rate_schedule": {
            "reduce_on_plateau": true,
            "reduction_factor": 0.5,
            "plateau_min_delta": 1e-06,
            "plateau_patience": 5,
            "plateau_cooldown": 3,
            "min_learning_rate": 1e-08
        },
        "hard_keypoint_mining": {
            "online_mining": false,
            "hard_to_easy_ratio": 2.0,
            "min_hard_keypoints": 2,
            "max_hard_keypoints": null,
            "loss_scale": 5.0
        },
        "early_stopping": {
            "stop_training_on_plateau": true,
            "plateau_min_delta": 1e-08,
            "plateau_patience": 10
        }
    },
    "outputs": {
        "save_outputs": true,
        "run_name": "unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0",
        "run_name_prefix": "",
        "run_name_suffix": ".single_instance",
        "runs_folder": "/Users/jeffreymarkowitz/Desktop/basler_plexiglass_sleap_training/labeling_project/models",
        "tags": [
            ""
        ],
        "save_visualizations": true,
        "delete_viz_images": true,
        "zip_outputs": false,
        "log_to_csv": true,
        "checkpointing": {
            "initial_model": false,
            "best_model": true,
            "every_epoch": false,
            "latest_model": false,
            "final_model": false
        },
        "tensorboard": {
            "write_logs": false,
            "loss_frequency": "epoch",
            "architecture_graph": false,
            "profile_graph": false,
            "visualizations": true
        },
        "zmq": {
            "subscribe_to_controller": true,
            "controller_address": "tcp://127.0.0.1:9000",
            "controller_polling_timeout": 10,
            "publish_updates": true,
            "publish_address": "tcp://127.0.0.1:9001"
        }
    },
    "name": "",
    "description": "",
    "sleap_version": "1.3.3",
    "filename": "/storage/coda1/p-jmarkowitz30/0/shared/active_lab_members/markowitz_jeffrey/active_projects/keypoints_basler_nir_plexiglass_arena/sleap_training/gridsearch_pretrained_1/unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0.json"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Auto-selected GPU 0 with 24208 MiB of free memory.
INFO:sleap.nn.training:Using GPU 0 for acceleration.
INFO:sleap.nn.training:Disabled GPU memory pre-allocation.
INFO:sleap.nn.training:System:
GPUs: 1/1 available
  Device: /physical_device:GPU:0
         Available: True
        Initalized: False
     Memory growth: True
INFO:sleap.nn.training:
INFO:sleap.nn.training:Initializing trainer...
INFO:sleap.nn.training:Loading training labels from: ../_labels/merged_labels_Basler-267601741000-24383488.slp
INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.2
INFO:sleap.nn.training:  Splits: Training = 331 / Validation = 83.
INFO:sleap.nn.training:Setting up for training...
INFO:sleap.nn.training:Setting up pipeline builders...
INFO:sleap.nn.training:Setting up model...
INFO:sleap.nn.training:Building test pipeline...
2024-04-03 11:36:47.108576: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2024-04-03 11:36:47.978719: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 22842 MB memory:  -> device: 0, name: Quadro RTX 6000, pci bus id: 0000:af:00.0, compute capability: 7.5
INFO:sleap.nn.training:Loaded test example. [4.204s]
INFO:sleap.nn.training:  Input shape: (1024, 1008, 1)
Downloading data from https://github.com/keras-team/keras-applications/releases/download/densenet/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5

   16384/29084464 [..............................] - ETA: 1s
  491520/29084464 [..............................] - ETA: 2s
 5062656/29084464 [====>.........................] - ETA: 0s
22724608/29084464 [======================>.......] - ETA: 0s
29089792/29084464 [==============================] - 0s 0us/step

29097984/29084464 [==============================] - 0s 0us/step
Traceback (most recent call last):
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/bin/sleap-train", line 33, in <module>
    sys.exit(load_entry_point('sleap==1.3.3', 'console_scripts', 'sleap-train')())
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 2014, in main
    trainer.train()
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 924, in train
    self.setup()
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 910, in setup
    self._setup_model()
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 734, in _setup_model
    self.model.make_model(input_shape)
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/model.py", line 326, in make_model
    x_main, x_mid = self.backbone.make_backbone(x_in=x_in)
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/architectures/pretrained_encoders.py", line 226, in make_backbone
    utils=tf.keras.utils,
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/segmentation_models/models/unet.py", line 240, in Unet
    use_batchnorm=decoder_use_batchnorm,
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/segmentation_models/models/unet.py", line 139, in build_unet
    x = decoder_block(decoder_filters[i], stage=i, use_batchnorm=use_batchnorm)(x, skip)
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/segmentation_models/models/unet.py", line 60, in wrapper
    x = layers.Concatenate(axis=concat_axis, name=concat_name)([x, skip])
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/keras/layers/merge.py", line 525, in build
    raise ValueError(err_msg)
ValueError: A `Concatenate` layer requires inputs with matching shapes except for the concatenation axis. Received: input_shape=[(None, 64, 62, 1024), (None, 64, 63, 512)]
---------------------------------------
Begin Slurm Epilog: Apr-03-2024 11:36:55
Job ID:        5503480
Array Job ID:  _4294967294
User ID:       jmarkowitz30
Account:       gts-jmarkowitz30-paid
Job name:      wrap
Resources:     cpu=12,gres/gpu:rtx_6000=1,mem=100G,node=1
Rsrc Used:     cput=00:06:00,vmem=800008K,walltime=00:00:30,mem=0,energy_used=0
Partition:     gpu-rtx6000
QOS:           inferno
Nodes:         atl1-1-03-006-27-0
---------------------------------------

If needed output of pip freeze

absl-py==1.0.0
astunparse==1.6.3
attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1640799537051/work
backports.zoneinfo==0.2.1
cached-property @ file:///home/conda/feedstock_root/build_artifacts/cached_property_1615209429212/work
cachetools==4.2.4
cattrs @ file:///home/conda/feedstock_root/build_artifacts/cattrs_1604136207372/work
certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1700303426725/work/certifi
charset-normalizer==2.0.9
cloudpickle @ file:///home/conda/feedstock_root/build_artifacts/cloudpickle_1674202310934/work
cycler @ file:///home/conda/feedstock_root/build_artifacts/cycler_1635519461629/work
cytoolz @ file:///home/conda/feedstock_root/build_artifacts/cytoolz_1657553457169/work
dask @ file:///home/conda/feedstock_root/build_artifacts/dask-core_1644602974678/work
efficientnet==1.0.0
flatbuffers==2.0
fonttools @ file:///home/conda/feedstock_root/build_artifacts/fonttools_1666389892786/work
fsspec @ file:///home/conda/feedstock_root/build_artifacts/fsspec_1674184942191/work
gast==0.4.0
google-auth==2.3.3
google-auth-oauthlib==0.4.6
google-pasta==0.2.0
grpcio==1.43.0
h5py @ file:///home/conda/feedstock_root/build_artifacts/h5py_1604753641401/work
hdmf==3.6.1
idna==3.3
image-classifiers==1.0.0
imagecodecs-lite @ file:///home/conda/feedstock_root/build_artifacts/imagecodecs-lite_1665171202021/work
imageio @ file:///home/conda/feedstock_root/build_artifacts/imageio_1702571712725/work
imgaug @ file:///home/conda/feedstock_root/build_artifacts/imgaug_1640909786103/work
imgstore==0.2.9
importlib-metadata==4.10.0
importlib-resources==5.12.0
joblib @ file:///home/conda/feedstock_root/build_artifacts/joblib_1691577114857/work
jsmin @ file:///home/conda/feedstock_root/build_artifacts/jsmin_1642532731678/work
jsonpickle==1.2
jsonschema==4.17.3
keras==2.7.0
Keras-Applications==1.0.8
Keras-Preprocessing==1.1.2
kiwisolver @ file:///home/conda/feedstock_root/build_artifacts/kiwisolver_1657953088445/work
libclang==12.0.0
locket @ file:///home/conda/feedstock_root/build_artifacts/locket_1650660393415/work
Markdown==3.3.6
markdown-it-py @ file:///home/conda/feedstock_root/build_artifacts/markdown-it-py_1677100944732/work
matplotlib @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-suite_1661439848456/work
mdurl @ file:///home/conda/feedstock_root/build_artifacts/mdurl_1704317613764/work
munkres==1.1.4
ndx-pose==0.1.1
networkx @ file:///home/conda/feedstock_root/build_artifacts/networkx_1646092782768/work
nixio==1.5.3
numpy==1.19.5
oauthlib==3.1.1
opencv-python-headless==4.2.0.34
opt-einsum==3.3.0
packaging==21.3
pandas==1.3.5
partd @ file:///home/conda/feedstock_root/build_artifacts/partd_1695667515973/work
patsy @ file:///home/conda/feedstock_root/build_artifacts/patsy_1704469236901/work
Pillow @ file:///home/conda/feedstock_root/build_artifacts/pillow_1660385854171/work
pkgutil_resolve_name==1.3.10
protobuf==3.16.0
psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1666155398032/work
pyasn1==0.4.8
pyasn1-modules==0.2.8
Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1700607939962/work
pykalman==0.9.5
pynwb==2.3.3
pyparsing==3.0.6
pyrsistent==0.19.3
PySide2==5.13.2
python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/python-dateutil_1626286286081/work
python-rapidjson @ file:///home/conda/feedstock_root/build_artifacts/python-rapidjson_1665999896718/work
pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1693930252784/work
PyWavelets @ file:///home/conda/feedstock_root/build_artifacts/pywavelets_1649616401885/work
PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1648757092905/work
pyzmq @ file:///home/conda/feedstock_root/build_artifacts/pyzmq_1663830492333/work
qimage2ndarray==1.10.0
QtPy @ file:///home/conda/feedstock_root/build_artifacts/qtpy_1698112029416/work
requests==2.26.0
requests-oauthlib==1.3.0
rich @ file:///home/conda/feedstock_root/build_artifacts/rich-split_1700160075651/work/dist
rsa==4.8
ruamel.yaml==0.17.32
ruamel.yaml.clib==0.2.7
scikit-image @ file:///home/conda/feedstock_root/build_artifacts/scikit-image_1645196656256/work
scikit-learn @ file:///home/conda/feedstock_root/build_artifacts/scikit-learn_1632611341839/work
scikit-video==1.1.11
scipy @ file:///opt/conda/conda-bld/scipy_1661390393401/work
seaborn @ file:///home/conda/feedstock_root/build_artifacts/seaborn-split_1672497695270/work
segmentation-models==1.0.1
setuptools-scm==6.3.2
Shapely @ file:///home/conda/feedstock_root/build_artifacts/shapely_1665624546039/work
shiboken2==5.13.2
six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work
sleap==1.3.3
statsmodels @ file:///home/conda/feedstock_root/build_artifacts/statsmodels_1654787101575/work
tensorboard==2.7.0
tensorboard-data-server==0.6.1
tensorboard-plugin-wit==1.8.0
tensorflow==2.7.0
tensorflow-estimator==2.7.0
tensorflow-hub @ file:///home/conda/feedstock_root/build_artifacts/tensorflow-hub_1678880940235/work/wheel_dir/tensorflow_hub-0.13.0-py2.py3-none-any.whl
tensorflow-io-gcs-filesystem==0.23.1
termcolor==1.1.0
threadpoolctl @ file:///home/conda/feedstock_root/build_artifacts/threadpoolctl_1643647933166/work
tifffile @ file:///home/conda/feedstock_root/build_artifacts/tifffile_1591280222285/work
tomli==2.0.0
toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1706112571092/work
typing_extensions==4.0.1
tzlocal==5.0.1
unicodedata2 @ file:///home/conda/feedstock_root/build_artifacts/unicodedata2_1649111917568/work
urllib3==1.26.7
Werkzeug==2.0.2
wrapt==1.13.3
zipp==3.15.0
talmo commented 3 months ago

Hi @jmarkow,

We'll dig into why we're not calculating the padding correctly, but can you try adjusting these lines in your config and seeing if you still have the same issue?

Old:

            "resize_and_pad_to_target": true,
            "target_height": 1023,
            "target_width": 1008

New:

            "resize_and_pad_to_target": true,
            "target_height": 1024,
            "target_width": 1024

Thanks!

Talmo

jmarkow commented 3 months ago

Such fast troubleshooting! Works now :shipit:

Also hey from sunny Atlanta @talmo .