[Request] Release codalab evaluation script

[digital-idiot]

Would it be possible to release the evaluation script from codalab? File format detail is a bit confusing. For example, if I set empty regions as transparent or embed color palette within the image the evaluation script shows warning:

/opt/conda/lib/python2.7/site-packages/PIL/Image.py:870: UserWarning: Palette images with Transparency   expressed in bytes should be converted to RGBA images
  'to RGBA images')

Even if i remove the color palette I get the following error:

Traceback (most recent call last):
  File "/tmp/codalab/tmpS_IrwU/run/program/evaluate.py", line 157, in <module>
    metric.forward(gt[valid_inds], mask[valid_inds])
  File "/tmp/codalab/tmpS_IrwU/run/program/evaluate.py", line 22, in forward
    cm = sparse.coo_matrix((v, (y_true, y_pred)), shape=(self.num_classes, self.num_classes), dtype=np.float32)
  File "/opt/conda/lib/python2.7/site-packages/scipy/sparse/coo.py", line 182, in __init__
    self._check()
  File "/opt/conda/lib/python2.7/site-packages/scipy/sparse/coo.py", line 219, in _check
    nnz = self.nnz
  File "/opt/conda/lib/python2.7/site-packages/scipy/sparse/coo.py", line 196, in getnnz
    raise ValueError('row, column, and data array must all be the '
ValueError: row, column, and data array must all be the same length

I made sure all my images are 1024 × 1024 with a single uint8 channel. The class ids have been assigned as per the specification, with empty regions assigned with value 15

Classes indexes

Background - 0
Building - 1
Road - 2
Water - 3
Barren - 4
Forest - 5
Agriculture - 6

So, it would be helpful to see the evaluation script and generate compatible prediction images.

[Junjue-Wang]

#!/usr/bin/env python
import sys
import os
from skimage.io import imread
import glob
import numpy as np
from scipy import sparse
import time

EPS = 1e-7

class ConfusionMatrix(object):
    def __init__(self, num_classes):
        self.num_classes = num_classes
        self._total = sparse.coo_matrix((num_classes, num_classes), dtype=np.float32)

    def forward(self, y_true, y_pred):
        y_pred = y_pred.reshape((-1,))
        y_true = y_true.reshape((-1,))

        v = np.ones_like(y_pred)
        cm = sparse.coo_matrix((v, (y_true, y_pred)), shape=(self.num_classes, self.num_classes), dtype=np.float32)
        self._total += cm

        return cm

    @property
    def dense_cm(self):
        return self._total.toarray()

    @property
    def sparse_cm(self):
        return self._total

    def reset(self):
        num_classes = self.num_classes
        self._total = sparse.coo_matrix((num_classes, num_classes), dtype=np.float32)

    def summary_iou(self):
        dense_cm = self._total.toarray()
        sum_over_row = np.sum(dense_cm, axis=0)
        sum_over_col = np.sum(dense_cm, axis=1)
        diag = np.diag(dense_cm)
        denominator = sum_over_row + sum_over_col - diag

        iou_per_class = diag / (denominator + EPS)
        miou = iou_per_class.mean()

        return iou_per_class, miou

classes = ['background','building','road','water','barren','forest','agriculture']

input_dir = sys.argv[1]
output_dir = sys.argv[2]

submit_dir = os.path.join(input_dir, 'res')
truth_dir = os.path.join(input_dir, 'ref')

if not os.path.isdir(submit_dir):
    print("%s doesn't exist" % submit_dir)

if os.path.isdir(submit_dir) and os.path.isdir(truth_dir):
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    output_filename = os.path.join(output_dir, 'scores.txt')
    output_file = open(output_filename, 'wb')

    # truthp_list = glob.glob(os.path.join(truth_dir, '*.png'))
    # subplist = [os.path.join(submit_dir, os.path.basename(truthp)) for truthp in truthp_list]
    # subplist = glob.glob(os.path.join(submit_dir, '*.png'))
    subplist = []
    g = os.walk(submit_dir)
    for path, dir_list, file_list in g:
        for file_name in file_list:
            if file_name.endswith('.png'):
                subplist.append(os.path.join(path, file_name))

    truthp_list = [os.path.join(truth_dir, os.path.basename(subp)) for subp in subplist]

    metric = ConfusionMatrix(len(classes))

    start_time = time.time()
    detailed_results = []
    for truthp, subp in zip(truthp_list, subplist):
        gt = imread(truthp).astype(np.float) - 1
        mask = imread(subp)
        valid_inds = gt != -1
        print(np.unique(gt[valid_inds]))
        print(np.unique(mask[valid_inds]))
        metric.forward(gt[valid_inds], mask[valid_inds])

    iou_per_class, miou = metric.summary_iou()

    output_file.write("mIoU: %s\n" % float(miou))
    detailed_results.append(float(miou))

    for class_name, iou in zip(classes, iou_per_class):
        output_file.write("%s: %s\n" % (class_name,float(iou)))
        detailed_results.append(float(iou))

    execution_time = time.time() - start_time

    output_file.write("ExecutionTime: %s\n" % float(execution_time))
    detailed_results.append(float(execution_time))
    output_file.close()

[digital-idiot]

@Junjue-Wang I am getting following error in Codalab;

Traceback (most recent call last):
  File "/tmp/codalab/tmpmCNvDT/run/program/evaluate.py", line 161, in 
    output_file.write("mIoU: %s\n" % float(miou))
TypeError: a bytes-like object is required, not 'str'

most likely due to opening the file in binary mode but trying to write str:

output_file = open(output_filename, 'wb')

In python2 writing str to a file opened in binary mode works but not in python3.x. At the time of my previous submission the evaluation code was most likely running in python2 but now it seems it is using python3!