I encounter the follow error when testing the docker with ./test.sh
The tested segmentation is the same as the ground truth. What does the following error mean?
Successfully built 4fb1792243c0
Successfully tagged flare22dsc:latest
1+0 records in
1+0 records out
32 bytes copied, 3.6341e-05 s, 881 kB/s
flare22dsc-output-e4ece676164e87ceaa9b8fe6bdd5e76d
Traceback (most recent call last):
File "/usr/local/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/local/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/opt/evaluation/evaluation.py", line 85, in <module>
flare22_eva_dsc().evaluate()
File "/home/evaluator/.local/lib/python3.9/site-packages/evalutils/evalutils.py", line 413, in evaluate
self.validate()
File "/home/evaluator/.local/lib/python3.9/site-packages/evalutils/evalutils.py", line 453, in validate
self._validate_data_frame(df=self._ground_truth_cases)
File "/home/evaluator/.local/lib/python3.9/site-packages/evalutils/evalutils.py", line 458, in _validate_data_frame
validator.validate(df=df)
File "/home/evaluator/.local/lib/python3.9/site-packages/evalutils/validators.py", line 48, in validate
raise ValidationError(
evalutils.exceptions.ValidationError: The first number is each filename is not unique, please check that your files are named correctly.
./test.sh: line 23: c: command not found
cat: /output/metrics.json: No such file or directory
Expecting value: line 1 column 1 (char 0)
flare22dsc-output-e4ece676164e87ceaa9b8fe6bdd5e76d
What I Did
N/A
I'm attaching the evaluation code that I used.
evaluation.py
import SimpleITK
import numpy as np
import scipy.ndimage
from evalutils import ClassificationEvaluation
from evalutils.io import SimpleITKLoader
from evalutils.validators import (
NumberOfCasesValidator, UniquePathIndicesValidator, UniqueImagesValidator
)
def compute_dice_coefficient(mask_gt, mask_pred):
volume_sum = mask_gt.sum() + mask_pred.sum()
if volume_sum == 0:
return np.NaN
volume_intersect = (mask_gt & mask_pred).sum()
return 2 * volume_intersect / volume_sum
class flare22_eva_dsc(ClassificationEvaluation):
def __init__(self):
super().__init__(
file_loader = SimpleITKLoader(),
validators=(
NumberOfCasesValidator(num_cases=50),
UniquePathIndicesValidator(),
UniqueImagesValidator(),
),
)
def score_case(self, *, idx, case):
gt_path = case["path_ground_truth"]
pred_path = case["path_prediction"]
# Load the images for this case
gt = self._file_loader.load_image(gt_path)
pred = self._file_loader.load_image(pred_path)
# Check that they're the right images
if (self._file_loader.hash_image(gt) != case["hash_ground_truth"] or
self._file_loader.hash_image(pred) != case["hash_prediction"]):
raise RuntimeError("Images do not match")
gt_data = SimpleITK.GetArrayFromImage(gt).transpose(2, 1, 0)
gt_data = np.uint8(gt_data)
pred_data = SimpleITK.GetArrayFromImage(pred).transpose(2, 1, 0)
pred_data = np.uint8(pred_data)
# Score the case
DSCs = list()
for i in range(1, 14):
if np.sum(gt_data == i) == 0 and np.sum(pred_data == i) == 0:
DSC_i = 1.0
elif np.sum(gt_data == i) == 0 and np.sum(pred_data == i) > 0:
DSC_i = 0.0
else:
DSC_i = compute_dice_coefficient(gt_data == i, pred_data == i)
DSCs.append(DSC_i)
return {
'Mean': np.mean(DSCs),
'Liver': DSCs[0],
'RightKidney': DSCs[1],
'Spleen': DSCs[2],
'Pancreas': DSCs[3],
'Aorta': DSCs[4],
'IVC': DSCs[5],
'RAG': DSCs[6],
'LAG': DSCs[7],
'Gallbladder': DSCs[8],
'Esophagus': DSCs[9],
'Stomach': DSCs[10],
'Duodenum': DSCs[11],
'LeftKidney': DSCs[12],
'pred_fname': pred_path.name,
'gt_fname': gt_path.name,
}
if __name__ == "__main__":
flare22_eva_dsc().evaluate()
This is the test.sh
#!/usr/bin/env bash
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
./build.sh
VOLUME_SUFFIX=$(dd if=/dev/urandom bs=32 count=1 | md5sum | cut --delimiter=' ' --fields=1)
docker volume create flare22dsc-output-$VOLUME_SUFFIX
# Do not change any of the parameters to docker run, these are fixed
docker run --rm \
--memory="4g" \
--memory-swap="4g" \
--network="none" \
--cap-drop="ALL" \
--security-opt="no-new-privileges" \
--shm-size="128m" \
--pids-limit="256" \
-v $SCRIPTPATH/test/:/input/ \
-v flare22dsc-output-$VOLUME_SUFFIX:/output/ \
flare22dsc
docker run --rm \
-v flare22dsc-output-$VOLUME_SUFFIX:/output/ \
python:3.9-slim cat /output/metrics.json | python3 -m json.tool
docker volume rm flare22dsc-output-$VOLUME_SUFFIX
It means that in the submission or ground truth folder you have two files with the same integer number in them, e.g. segmentation_10.mha and my_segmentation_10.mha.
Description
I encounter the follow error when testing the docker with
./test.sh
The tested segmentation is the same as the ground truth. What does the following error mean?
What I Did
N/A
I'm attaching the evaluation code that I used.
evaluation.py
This is the
test.sh