A bug in extract_keypoints function

Hi, I found a bug in your function extract_keypoints, please check! Your code is as follows

def extract_keypoints(events, n_kpts=10, width=240, height=180):
    # create uniform pivots
    keypoints = np.linspace(-1, 1, n_kpts)[:, None, None]
    interval = keypoints[1] - keypoints[0]
    left, right = keypoints[0], (keypoints[0] + keypoints[1]) / 2
    index, candidate = 0, np.full((height, width), np.nan)
    keypoints = np.tile(keypoints, (1, height, width))
    changes = np.zeros((height, width), np.uint8)
    if len(events) == 0:
        return keypoints
    # normalize timestamps to [-1, 1]
    events[:, 0] = (events[:, 0] - events[0, 0]) / (events[-1, 0] - events[0, 0])
    for t, x, y, _ in events:
        x, y = int(x), int(y)
        while t >= right:
            for yy in range(height):
                for xx in range(width):
                    if not np.isnan(candidate[yy, xx]):
                        keypoints[index, yy, xx] = candidate[yy, xx]
                        changes[yy, xx] += 1
                        candidate[yy, xx] = np.nan
            left, right = right, right + interval
            index += 1
        if np.isnan(candidate[y, x]):
            candidate[y, x] = t
        else:
            old_dist = np.abs(candidate[y, x] - keypoints[index, y, x])
            new_dist = np.abs(t - keypoints[index, y, x])
            if old_dist > new_dist:
                candidate[y, x] = t
    for yy in range(height):
        for xx in range(width):
            if not np.isnan(candidate[yy, xx]):
                keypoints[index, yy, xx] = candidate[yy, xx]
                changes[yy, xx] += 1
    return keypoints

Where events[:, 0] should be normalized to [-1, 1], but [0, 1] in fact! This would cause keypoints error, keypoints in your code is uniform pivots in [-1, 0] and uniform pivots with a random distub (event not right at that time) in [0, 1]

keypoints(t)=\left\{
\begin{aligned}
uniform\_pivots(t),  t \in [-1, 0]\\
uniform\_pivots(t)+noise,  t \in [0, 1]
\end{aligned}
\right.

Despite the above bug, the difference in keypoints is small, and this make little effect on training

Here is my modified code in which I fixed this bug and made some optimizations

def extract_keypoints_acc(events, n_kpts=10, width=240, height=180):
    # events: [t, x, y, p]
    # create uniform pivots
    keypoints = np.linspace(-1, 1, n_kpts)[:, None, None]          
    interval = keypoints[1] - keypoints[0]
    left, right = keypoints[0], (keypoints[0] + keypoints[1]) / 2   
    index, candidate = 0, np.full((height, width), np.nan)
    keypoints = np.tile(keypoints, (1, height, width))              
    changes = np.zeros((height, width), np.uint8)
    if len(events) == 0:
        return keypoints
    # normalize timestamps to [-1, 1]                              
    events[:, 0] = ((events[:, 0] - events[0, 0]) / (events[-1, 0] - events[0, 0]) - 0.5) * 2
    for t, x, y, _ in events:
        x, y = int(x), int(y)
        if t >= right:
            change_mask = ~np.isnan(candidate)
            keypoints[index][change_mask] = candidate[change_mask]
            changes[change_mask] += 1
            candidate = np.full((height, width), np.nan)
            left, right = right, right + interval
            index += 1

        if np.isnan(candidate[y, x]):
            candidate[y, x] = t
        else:
            old_dist = np.abs(candidate[y, x] - keypoints[index, y, x])
            new_dist = np.abs(t - keypoints[index, y, x])
            if old_dist > new_dist:
                candidate[y, x] = t

    change_mask = ~np.isnan(candidate)
    keypoints[index][change_mask] = candidate[change_mask]
    changes[change_mask] += 1

    return keypoints

By the way, your packed dataset is too big, you can save image in uint8 format and do normalization in training code

chensong1995 / E-CIR

A bug in extract_keypoints function #2