search

talmolab / sleap

A deep learning framework for multi-animal pose tracking.
https://sleap.ai
Other
435 stars 97 forks source link

ValueError when using downloaded resnet50 on HPC cluster #1549

Closed agrayseal closed 10 months ago

agrayseal commented 1 year ago

I'm trying to use a resnet50 backbone on an hpc cluster without network access, so downloaded it as shown in the FAQs. However, I now get this error that some dimensions don't match (edited out my id):

Running: sleap-train /hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/configs/10_R_8-single_instance.json /hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/labels.v001.pkg.slp
INFO:sleap.nn.training:Versions:
SLEAP: 1.3.3
TensorFlow: 2.8.4
Numpy: 1.21.6
Python: 3.7.12
OS: Linux-4.18.0-305.el8.x86_64-x86_64-with-redhat-8.4-Ootpa
INFO:sleap.nn.training:Training labels file: /hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/labels.v001.pkg.slp
INFO:sleap.nn.training:Training profile: /hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/configs/10_R_8-single_instance.json
INFO:sleap.nn.training:
INFO:sleap.nn.training:Arguments:
INFO:sleap.nn.training:{
    "training_job_path": "/hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/configs/10_R_8-single_instance.json",
    "labels_path": "/hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/labels.v001.pkg.slp",
    "video_paths": [
        ""
    ],
    "val_labels": null,
    "test_labels": null,
    "base_checkpoint": null,
    "tensorboard": false,
    "save_viz": false,
    "zmq": false,
    "run_name": "",
    "prefix": "",
    "suffix": "",
    "cpu": false,
    "first_gpu": false,
    "last_gpu": false,
    "gpu": "auto"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Training job:
INFO:sleap.nn.training:{
    "data": {
        "labels": {
            "training_labels": null,
            "validation_labels": null,
            "validation_fraction": 0.1,
            "test_labels": null,
            "split_by_inds": false,
            "training_inds": [
                39,
                131,
                14,
                74,
                199,
                25,
                177,
                203,
                246,
                130,
                105,
                73,
                154,
                242,
                30,
                178,
                16,
                34,
                21,
                12,
                234,
                257,
                24,
                77,
                17,
                205,
                86,
                190,
                215,
                151,
                254,
                225,
                113,
                201,
                222,
                15,
                155,
                53,
                187,
                139,
                116,
                140,
                89,
                10,
                90,
                236,
                125,
                45,
                141,
                192,
                176,
                171,
                239,
                49,
                152,
                180,
                143,
                63,
                197,
                186,
                149,
                163,
                195,
                107,
                207,
                160,
                126,
                243,
                168,
                20,
                136,
                189,
                252,
                232,
                11,
                224,
                121,
                47,
                132,
                8,
                66,
                253,
                238,
                148,
                228,
                22,
                258,
                213,
                122,
                91,
                208,
                123,
                191,
                240,
                35,
                146,
                211,
                172,
                247,
                245,
                249,
                219,
                174,
                42,
                221,
                28,
                52,
                229,
                188,
                83,
                156,
                217,
                145,
                31,
                92,
                210,
                98,
                78,
                193,
                233,
                80,
                44,
                40,
                71,
                46,
                150,
                169,
                135,
                251,
                173,
                147,
                129,
                165,
                108,
                96,
                250,
                179,
                209,
                153,
                198,
                2,
                55,
                235,
                6,
                104,
                206,
                56,
                200,
                170,
                231,
                51,
                84,
                13,
                110,
                65,
                175,
                194,
                82,
                41,
                1,
                162,
                144,
                185,
                88,
                43,
                112,
                212,
                157,
                97,
                114,
                103,
                50,
                3,
                23,
                119,
                159,
                75,
                58,
                244,
                62,
                85,
                182,
                7,
                256,
                134,
                19,
                214,
                100,
                0,
                18,
                94,
                181,
                101,
                54,
                127,
                93,
                241,
                102,
                128,
                33,
                68,
                183,
                227,
                69,
                87,
                196,
                106,
                184,
                118,
                218,
                59,
                81,
                255,
                220,
                115,
                158,
                202,
                138,
                70,
                204,
                4,
                133,
                223,
                120,
                111,
                38,
                64,
                167,
                109,
                164,
                36,
                48,
                9
            ],
            "validation_inds": [
                230,
                26,
                99,
                117,
                142,
                76,
                161,
                216,
                226,
                60,
                57,
                248,
                32,
                37,
                79,
                124,
                61,
                67,
                29,
                237,
                95,
                27,
                5,
                166,
                72,
                137
            ],
            "test_inds": null,
            "search_path_hints": [
                ""
            ],
            "skeletons": []
        },
        "preprocessing": {
            "ensure_rgb": false,
            "ensure_grayscale": true,
            "imagenet_mode": null,
            "input_scaling": 1.0,
            "pad_to_stride": 16,
            "resize_and_pad_to_target": true,
            "target_height": 208,
            "target_width": 206
        },
        "instance_cropping": {
            "center_on_part": null,
            "crop_size": null,
            "crop_size_detection_padding": 16
        }
    },
    "model": {
        "backbone": {
            "leap": null,
            "unet": null,
            "hourglass": null,
            "resnet": {
                "version": "ResNet50",
                "weights": "frozen",
                "upsampling": {
                    "method": "interpolation",
                    "skip_connections": null,
                    "block_stride": 2,
                    "filters": 64,
                    "filters_rate": 1.0,
                    "refine_convs": 2,
                    "batch_norm": true,
                    "transposed_conv_kernel_size": 4
                },
                "max_stride": 32,
                "output_stride": 2
            },
            "pretrained_encoder": null
        },
        "heads": {
            "single_instance": {
                "part_names": [
                    "head",
                    "body",
                    "tailbase",
                    "tailmid",
                    "tailtip"
                ],
                "sigma": 2.5,
                "output_stride": 2,
                "loss_weight": 1.0,
                "offset_refinement": false
            },
            "centroid": null,
            "centered_instance": null,
            "multi_instance": null,
            "multi_class_bottomup": null,
            "multi_class_topdown": null
        },
        "base_checkpoint": null
    },
    "optimization": {
        "preload_data": true,
        "augmentation_config": {
            "rotate": true,
            "rotation_min_angle": -15.0,
            "rotation_max_angle": 15.0,
            "translate": false,
            "translate_min": -5,
            "translate_max": 5,
            "scale": false,
            "scale_min": 0.9,
            "scale_max": 1.1,
            "uniform_noise": false,
            "uniform_noise_min_val": 0.0,
            "uniform_noise_max_val": 10.0,
            "gaussian_noise": false,
            "gaussian_noise_mean": 5.0,
            "gaussian_noise_stddev": 1.0,
            "contrast": false,
            "contrast_min_gamma": 0.5,
            "contrast_max_gamma": 2.0,
            "brightness": false,
            "brightness_min_val": 0.0,
            "brightness_max_val": 10.0,
            "random_crop": false,
            "random_crop_height": 256,
            "random_crop_width": 256,
            "random_flip": false,
            "flip_horizontal": false
        },
        "online_shuffling": true,
        "shuffle_buffer_size": 128,
        "prefetch": true,
        "batch_size": 8,
        "batches_per_epoch": 200,
        "min_batches_per_epoch": 200,
        "val_batches_per_epoch": 10,
        "min_val_batches_per_epoch": 10,
        "epochs": 200,
        "optimizer": "adam",
        "initial_learning_rate": 0.0001,
        "learning_rate_schedule": {
            "reduce_on_plateau": true,
            "reduction_factor": 0.5,
            "plateau_min_delta": 1e-06,
            "plateau_patience": 5,
            "plateau_cooldown": 3,
            "min_learning_rate": 1e-08
        },
        "hard_keypoint_mining": {
            "online_mining": false,
            "hard_to_easy_ratio": 2.0,
            "min_hard_keypoints": 2,
            "max_hard_keypoints": null,
            "loss_scale": 5.0
        },
        "early_stopping": {
            "stop_training_on_plateau": true,
            "plateau_min_delta": 1e-08,
            "plateau_patience": 10
        }
    },
    "outputs": {
        "save_outputs": true,
        "run_name": "231013_084326",
        "run_name_prefix": "10_R_8-",
        "run_name_suffix": ".single_instance",
        "runs_folder": "models",
        "tags": [
            ""
        ],
        "save_visualizations": true,
        "delete_viz_images": false,
        "zip_outputs": false,
        "log_to_csv": true,
        "checkpointing": {
            "initial_model": false,
            "best_model": true,
            "every_epoch": false,
            "latest_model": false,
            "final_model": false
        },
        "tensorboard": {
            "write_logs": false,
            "loss_frequency": "epoch",
            "architecture_graph": false,
            "profile_graph": false,
            "visualizations": true
        },
        "zmq": {
            "subscribe_to_controller": true,
            "controller_address": "tcp://127.0.0.1:9000",
            "controller_polling_timeout": 10,
            "publish_updates": true,
            "publish_address": "tcp://127.0.0.1:9001"
        }
    },
    "name": "",
    "description": "",
    "sleap_version": "1.3.3",
    "filename": "/hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/configs/10_R_8-single_instance.json"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Auto-selected GPU 0 with 40351 MiB of free memory.
INFO:sleap.nn.training:Using GPU 0 for acceleration.
INFO:sleap.nn.training:Disabled GPU memory pre-allocation.
INFO:sleap.nn.training:System:
GPUs: 1/1 available
  Device: /physical_device:GPU:0
         Available: True
        Initalized: False
     Memory growth: True
INFO:sleap.nn.training:
INFO:sleap.nn.training:Initializing trainer...
INFO:sleap.nn.training:Loading training labels from: /hpcfs/users/id/adult-behaviour/sleap/experiments/2023-10-10_labels/labels.v001.pkg.slp
INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1
INFO:sleap.nn.training:  Splits: Training = 300 / Validation = 33.
INFO:sleap.nn.training:Setting up for training...
INFO:sleap.nn.training:Setting up pipeline builders...
INFO:sleap.nn.training:Setting up model...
INFO:sleap.nn.training:Building test pipeline...
2023-10-13 16:11:51.389582: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-10-13 16:11:51.952873: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 38238 MB memory:  -> device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:31:00.0, compute capability: 8.0
INFO:sleap.nn.training:Loaded test example. [1.815s]
INFO:sleap.nn.training:  Input shape: (208, 208, 1)
INFO:sleap.nn.training:Created Keras model.
INFO:sleap.nn.training:  Backbone: ResNet50(upsampling_stack=UpsamplingStack(output_stride=2, upsampling_stride=2, transposed_conv=False, transposed_conv_filters=64, transposed_conv_filters_rate=1.0, transposed_conv_kernel_size=4, transposed_conv_batchnorm=True, make_skip_connection=False, skip_add=False, refine_convs=2, refine_convs_filters=64, refine_convs_filters_rate=1.0, refine_convs_batchnorm=True), features_output_stride=32, pretrained=True, frozen=True, skip_connections=False, model_name='resnet50', stack_configs=[{'filters': 64, 'blocks': 3, 'stride1': 1, 'name': 'conv2', 'dilation_rate': 1}, {'filters': 128, 'blocks': 4, 'stride1': 2, 'name': 'conv3', 'dilation_rate': 1}, {'filters': 256, 'blocks': 6, 'stride1': 2, 'name': 'conv4', 'dilation_rate': 1}, {'filters': 512, 'blocks': 3, 'stride1': 2, 'name': 'conv5', 'dilation_rate': 1}])
INFO:sleap.nn.training:  Max stride: 32
INFO:sleap.nn.training:  Parameters: 25,028,293
INFO:sleap.nn.training:  Heads: 
INFO:sleap.nn.training:    [0] = SingleInstanceConfmapsHead(part_names=['head', 'body', 'tailbase', 'tailmid', 'tailtip'], sigma=2.5, output_stride=2, loss_weight=1.0)
INFO:sleap.nn.training:  Outputs: 
INFO:sleap.nn.training:    [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 112, 112, 5), dtype=tf.float32, name=None), name='SingleInstanceConfmapsHead/BiasAdd:0', description="created by layer 'SingleInstanceConfmapsHead'")
INFO:sleap.nn.training:Training from scratch
INFO:sleap.nn.training:Setting up data pipelines...
INFO:sleap.nn.training:Training set: n = 300
INFO:sleap.nn.training:Validation set: n = 33
INFO:sleap.nn.training:Setting up optimization...
INFO:sleap.nn.training:  Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-06, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08)
INFO:sleap.nn.training:  Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=10)
INFO:sleap.nn.training:Setting up outputs...
INFO:sleap.nn.callbacks:Training controller subscribed to: tcp://127.0.0.1:9000 (topic: )
INFO:sleap.nn.training:  ZMQ controller subcribed to: tcp://127.0.0.1:9000
INFO:sleap.nn.callbacks:Progress reporter publishing on: tcp://127.0.0.1:9001 for: not_set
INFO:sleap.nn.training:  ZMQ progress reporter publish on: tcp://127.0.0.1:9001
INFO:sleap.nn.training:Created run path: models/10_R_8-231013_084326.single_instance
INFO:sleap.nn.training:Setting up visualization...
Unable to use Qt backend for matplotlib. This probably means Qt is running headless.
Unable to use Qt backend for matplotlib. This probably means Qt is running headless.
INFO:sleap.nn.training:Finished trainer set up. [4.3s]
INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation...
INFO:sleap.nn.training:Finished creating training datasets. [2.2s]
INFO:sleap.nn.training:Starting training loop...
Epoch 1/200
/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/sleap/nn/inference.py:1183: UserWarning: Model input of shape (None, 208, 208, 1) does not divide evenly with output of shape (None, 112, 112, 5).
  f"Model input of shape {model.inputs[input_ind].shape} does not divide "
Traceback (most recent call last):
  File "/hpcfs/users/id/mambaforge/envs/sleap/bin/sleap-train", line 8, in <module>
    sys.exit(main())
  File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 2014, in main
    trainer.train()
  File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 941, in train
    verbose=2,
  File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py", line 1147, in autograph_handler
    raise e.ag_error_metadata.to_exception(e)
ValueError: in user code:

    File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/keras/engine/training.py", line 1021, in train_function  *
        return step_function(self, iterator)
    File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 301, in loss_fn  *
        loss += loss_fn(y_gt, y_pr)
    File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/keras/losses.py", line 141, in __call__  **
        losses = call_fn(y_true, y_pred)
    File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/keras/losses.py", line 245, in call  **
        return ag_fn(y_true, y_pred, **self._fn_kwargs)
    File "/hpcfs/users/id/mambaforge/envs/sleap/lib/python3.7/site-packages/keras/losses.py", line 1329, in mean_squared_error
        return backend.mean(tf.math.squared_difference(y_pred, y_true), axis=-1)

    ValueError: Dimensions must be equal, but are 112 and 104 for '{{node loss_fn/mean_squared_error/SquaredDifference}} = SquaredDifference[T=DT_FLOAT](model/SingleInstanceConfmapsHead/BiasAdd, IteratorGetNext:1)' with input shapes: [8,112,112,5], [8,104,104,?].

INFO:sleap.nn.callbacks:Closing the reporter controller/context.
INFO:sleap.nn.callbacks:Closing the training controller socket/context.

I've exported the training package through the GUI, then the config was copied to clipboard and the top section removed (as it didn't match the config I've seen in the full training job package .zip folder). Config is pasted below:

{
    "data": {
        "labels": {
            "training_labels": null,
            "validation_labels": null,
            "validation_fraction": 0.1,
            "test_labels": null,
            "split_by_inds": false,
            "training_inds": [
                39,
                131,
                14,
                74,
                199,
                25,
                177,
                203,
                246,
                130,
                105,
                73,
                154,
                242,
                30,
                178,
                16,
                34,
                21,
                12,
                234,
                257,
                24,
                77,
                17,
                205,
                86,
                190,
                215,
                151,
                254,
                225,
                113,
                201,
                222,
                15,
                155,
                53,
                187,
                139,
                116,
                140,
                89,
                10,
                90,
                236,
                125,
                45,
                141,
                192,
                176,
                171,
                239,
                49,
                152,
                180,
                143,
                63,
                197,
                186,
                149,
                163,
                195,
                107,
                207,
                160,
                126,
                243,
                168,
                20,
                136,
                189,
                252,
                232,
                11,
                224,
                121,
                47,
                132,
                8,
                66,
                253,
                238,
                148,
                228,
                22,
                258,
                213,
                122,
                91,
                208,
                123,
                191,
                240,
                35,
                146,
                211,
                172,
                247,
                245,
                249,
                219,
                174,
                42,
                221,
                28,
                52,
                229,
                188,
                83,
                156,
                217,
                145,
                31,
                92,
                210,
                98,
                78,
                193,
                233,
                80,
                44,
                40,
                71,
                46,
                150,
                169,
                135,
                251,
                173,
                147,
                129,
                165,
                108,
                96,
                250,
                179,
                209,
                153,
                198,
                2,
                55,
                235,
                6,
                104,
                206,
                56,
                200,
                170,
                231,
                51,
                84,
                13,
                110,
                65,
                175,
                194,
                82,
                41,
                1,
                162,
                144,
                185,
                88,
                43,
                112,
                212,
                157,
                97,
                114,
                103,
                50,
                3,
                23,
                119,
                159,
                75,
                58,
                244,
                62,
                85,
                182,
                7,
                256,
                134,
                19,
                214,
                100,
                0,
                18,
                94,
                181,
                101,
                54,
                127,
                93,
                241,
                102,
                128,
                33,
                68,
                183,
                227,
                69,
                87,
                196,
                106,
                184,
                118,
                218,
                59,
                81,
                255,
                220,
                115,
                158,
                202,
                138,
                70,
                204,
                4,
                133,
                223,
                120,
                111,
                38,
                64,
                167,
                109,
                164,
                36,
                48,
                9
            ],
            "validation_inds": [
                230,
                26,
                99,
                117,
                142,
                76,
                161,
                216,
                226,
                60,
                57,
                248,
                32,
                37,
                79,
                124,
                61,
                67,
                29,
                237,
                95,
                27,
                5,
                166,
                72,
                137
            ],
            "test_inds": null,
            "search_path_hints": [
                ""
            ],
            "skeletons": []
        },
        "preprocessing": {
            "ensure_rgb": false,
            "ensure_grayscale": true,
            "imagenet_mode": null,
            "input_scaling": 1.0,
            "pad_to_stride": 16,
            "resize_and_pad_to_target": true,
            "target_height": 208,
            "target_width": 206
        },
        "instance_cropping": {
            "center_on_part": null,
            "crop_size": null,
            "crop_size_detection_padding": 16
        }
    },
    "model": {
        "backbone": {
            "leap": null,
            "unet": null,
            "hourglass": null,
            "resnet": {
                "version": "ResNet50",
                "weights": "frozen",
                "upsampling": {
                    "method": "interpolation",
                    "skip_connections": null,
                    "block_stride": 2,
                    "filters": 64,
                    "filters_rate": 1.0,
                    "refine_convs": 2,
                    "batch_norm": true,
                    "transposed_conv_kernel_size": 4
                },
                "max_stride": 32,
                "output_stride": 2
            },
            "pretrained_encoder": null
        },
        "heads": {
            "single_instance": {
                "part_names": [
                    "head",
                    "body",
                    "tailbase",
                    "tailmid",
                    "tailtip"
                ],
                "sigma": 2.5,
                "output_stride": 2,
                "loss_weight": 1.0,
                "offset_refinement": false
            },
            "centroid": null,
            "centered_instance": null,
            "multi_instance": null,
            "multi_class_bottomup": null,
            "multi_class_topdown": null
        },
        "base_checkpoint": null
    },
    "optimization": {
        "preload_data": true,
        "augmentation_config": {
            "rotate": true,
            "rotation_min_angle": -15.0,
            "rotation_max_angle": 15.0,
            "translate": false,
            "translate_min": -5,
            "translate_max": 5,
            "scale": false,
            "scale_min": 0.9,
            "scale_max": 1.1,
            "uniform_noise": false,
            "uniform_noise_min_val": 0.0,
            "uniform_noise_max_val": 10.0,
            "gaussian_noise": false,
            "gaussian_noise_mean": 5.0,
            "gaussian_noise_stddev": 1.0,
            "contrast": false,
            "contrast_min_gamma": 0.5,
            "contrast_max_gamma": 2.0,
            "brightness": false,
            "brightness_min_val": 0.0,
            "brightness_max_val": 10.0,
            "random_crop": false,
            "random_crop_height": 256,
            "random_crop_width": 256,
            "random_flip": false,
            "flip_horizontal": false
        },
        "online_shuffling": true,
        "shuffle_buffer_size": 128,
        "prefetch": true,
        "batch_size": 8,
        "batches_per_epoch": 200,
        "min_batches_per_epoch": 200,
        "val_batches_per_epoch": 10,
        "min_val_batches_per_epoch": 10,
        "epochs": 200,
        "optimizer": "adam",
        "initial_learning_rate": 0.0001,
        "learning_rate_schedule": {
            "reduce_on_plateau": true,
            "reduction_factor": 0.5,
            "plateau_min_delta": 1e-06,
            "plateau_patience": 5,
            "plateau_cooldown": 3,
            "min_learning_rate": 1e-08
        },
        "hard_keypoint_mining": {
            "online_mining": false,
            "hard_to_easy_ratio": 2.0,
            "min_hard_keypoints": 2,
            "max_hard_keypoints": null,
            "loss_scale": 5.0
        },
        "early_stopping": {
            "stop_training_on_plateau": true,
            "plateau_min_delta": 1e-08,
            "plateau_patience": 10
        }
    },
    "outputs": {
        "save_outputs": true,
        "run_name": "231013_084326",
        "run_name_prefix": "10_R_8-",
        "run_name_suffix": ".single_instance",
        "runs_folder": "models",
        "tags": [
            ""
        ],
        "save_visualizations": true,
        "delete_viz_images": false,
        "zip_outputs": false,
        "log_to_csv": true,
        "checkpointing": {
            "initial_model": false,
            "best_model": true,
            "every_epoch": false,
            "latest_model": false,
            "final_model": false
        },
        "tensorboard": {
            "write_logs": false,
            "loss_frequency": "epoch",
            "architecture_graph": false,
            "profile_graph": false,
            "visualizations": true
        },
        "zmq": {
            "subscribe_to_controller": true,
            "controller_address": "tcp://127.0.0.1:9000",
            "controller_polling_timeout": 10,
            "publish_updates": true,
            "publish_address": "tcp://127.0.0.1:9001"
        }
    },
    "name": "",
    "description": "",
    "sleap_version": "1.3.3",
    "filename": "10_R_8-single_instance.json"
}
talmo commented 10 months ago

Hi @agrayseal,

Sorry that we never got back to you here! We're closing this as we will soon be implementing a new neural network backend that should get around this issue.

In the meantime, a good troubleshooting step might be to try this out locally with internet access (even without a GPU) and see if you're still getting the shape error.

Let us know if you're running into any other issues though!

Talmo