GSI gives multiple track results with same ID for same frame

mikel-brostrom commented 11 months ago

I grab all my txt MOT result files by mot_results_folder.glob('MOT*FRCNN.txt'). When I run them through GSI

import os
import numpy as np
from os.path import join
from pathlib import Path
from collections import defaultdict
from sklearn.gaussian_process.kernels import RBF
from sklearn.gaussian_process import GaussianProcessRegressor as GPR

# 线性插值
def LinearInterpolation(input_, interval):
    print(input_)
    input_ = input_[np.lexsort([input_[:, 0], input_[:, 1]])]  # 按ID和帧排序
    output_ = input_.copy()
    '''线性插值'''
    id_pre, f_pre, row_pre = -1, -1, np.zeros((10,))
    for row in input_:
        f_curr, id_curr = row[:2].astype(int)
        if id_curr == id_pre:  # 同ID
            if f_pre + 1 < f_curr < f_pre + interval:
                for i, f in enumerate(range(f_pre + 1, f_curr), start=1):  # 逐框插值
                    step = (row - row_pre) / (f_curr - f_pre) * i
                    row_new = row_pre + step
                    output_ = np.append(output_, row_new[np.newaxis, :], axis=0)
        else:  # 不同ID
            id_pre = id_curr
        row_pre = row
        f_pre = f_curr
    output_ = output_[np.lexsort([output_[:, 0], output_[:, 1]])]
    return output_

# 高斯平滑
def GaussianSmooth(input_, tau):
    output_ = list()
    ids = set(input_[:, 1])
    for id_ in ids:
        tracks = input_[input_[:, 1] == id_]
        len_scale = np.clip(tau * np.log(tau ** 3 / len(tracks)), tau ** -1, tau ** 2)
        gpr = GPR(RBF(len_scale, 'fixed'))
        t = tracks[:, 0].reshape(-1, 1)
        x = tracks[:, 2].reshape(-1, 1)
        y = tracks[:, 3].reshape(-1, 1)
        w = tracks[:, 4].reshape(-1, 1)
        h = tracks[:, 5].reshape(-1, 1)
        gpr.fit(t, x)
        xx = gpr.predict(t)[0]
        gpr.fit(t, y)
        yy = gpr.predict(t)[0]
        gpr.fit(t, w)
        ww = gpr.predict(t)[0]
        gpr.fit(t, h)
        hh = gpr.predict(t)[0]
        print(t)
        print(id_)
        print('xx', xx)
        output_.extend([
            [int(t[0]), int(id_), int(xx), int(yy), int(ww), int(hh), 1, -1, -1] for i in range(len(t))
        ])
    return output_

def gsi_interpolation(mot_results_folder=Path('examples/runs/val/exp51/labels'), interval=20, tau=10):
    tracking_results_files = mot_results_folder.glob('MOT*FRCNN.txt')
    gsi_result_names = []
    for p in tracking_results_files:
        print(f"Processing: {p.parent / (p.stem + 'proc.txt')}")
        tracking_results = np.loadtxt(p, delimiter=' ')
        li = LinearInterpolation(tracking_results, interval)
        print(li)
        gsi = GaussianSmooth(li, tau)

        # print(gsi)
        np.savetxt(p, gsi, fmt='%d %d %d %d %d %d %d %d %d')

gsi_interpolation()

the evaluation using trackeval fails:

Tracker labels was unable to be evaluated.
Tracker predicts the same ID more than once in a single timestep (seq: MOT17-09-FRCNN, frame: 1, ids: 1 2 3 4 5 6 7 8)

It works fine otherwise

dyhBUPT commented 11 months ago

Hi, I can't reproduce your results.

Specifically, I run the following codes:

import os
import numpy as np
from os.path import join
from pathlib import Path
from collections import defaultdict
from sklearn.gaussian_process.kernels import RBF
from sklearn.gaussian_process import GaussianProcessRegressor as GPR

# 线性插值
def LinearInterpolation(input_, interval):
    input_ = input_[np.lexsort([input_[:, 0], input_[:, 1]])]  # 按ID和帧排序
    output_ = input_.copy()
    '''线性插值'''
    id_pre, f_pre, row_pre = -1, -1, np.zeros((10,))
    for row in input_:
        f_curr, id_curr = row[:2].astype(int)
        if id_curr == id_pre:  # 同ID
            if f_pre + 1 < f_curr < f_pre + interval:
                for i, f in enumerate(range(f_pre + 1, f_curr), start=1):  # 逐框插值
                    step = (row - row_pre) / (f_curr - f_pre) * i
                    row_new = row_pre + step
                    output_ = np.append(output_, row_new[np.newaxis, :], axis=0)
        else:  # 不同ID
            id_pre = id_curr
        row_pre = row
        f_pre = f_curr
    output_ = output_[np.lexsort([output_[:, 0], output_[:, 1]])]
    return output_

# 高斯平滑
def GaussianSmooth(input_, tau):
    output_ = list()
    ids = set(input_[:, 1])
    for id_ in ids:
        tracks = input_[input_[:, 1] == id_]
        len_scale = np.clip(tau * np.log(tau ** 3 / len(tracks)), tau ** -1, tau ** 2)
        gpr = GPR(RBF(len_scale, 'fixed'))
        t = tracks[:, 0].reshape(-1, 1)
        x = tracks[:, 2].reshape(-1, 1)
        y = tracks[:, 3].reshape(-1, 1)
        w = tracks[:, 4].reshape(-1, 1)
        h = tracks[:, 5].reshape(-1, 1)
        gpr.fit(t, x)
        xx = gpr.predict(t)[0]
        gpr.fit(t, y)
        yy = gpr.predict(t)[0]
        gpr.fit(t, w)
        ww = gpr.predict(t)[0]
        gpr.fit(t, h)
        hh = gpr.predict(t)[0]
        output_.extend([
            [int(t[0]), int(id_), int(xx), int(yy), int(ww), int(hh), 1, -1, -1] for i in range(len(t))
        ])
    return output_

def gsi_interpolation(mot_results_folder=Path('examples/runs/val/exp51/labels'), interval=20, tau=10):
    # tracking_results_files = mot_results_folder.glob('MOT*FRCNN.txt')
    # gsi_result_names = []
    # for p in tracking_results_files:
    #     print(f"Processing: {p.parent / (p.stem + 'proc.txt')}")
    #     tracking_results = np.loadtxt(p, delimiter=' ')
    tracking_results = np.array([
        [1, 1, 1046, 140, 60, 179, 0, 0, -1],
        [1, 2, 102, 548, 74, 241, 0, 0, -1],
        [1, 3, 544, 155, 59, 175, 0, 0, -1],
        [1, 4, 372, 413, 81, 231, 0, 0, -1],
        [1, 5, 224, 404, 68, 189, 0, 0, -1],
        [1, 6, 1361, 568, 88, 228, 0, 0, -1],
    ])
    li = LinearInterpolation(tracking_results, interval)
    gsi = GaussianSmooth(li, tau)
    print(gsi)
    # np.savetxt(p, gsi, fmt='%d %d %d %d %d %d %d %d %d')

gsi_interpolation()

and get the following results:

[[1, 1, 1045, 139, 59, 178, 1, -1, -1], [1, 2, 101, 547, 73, 240, 1, -1, -1], [1, 3, 543, 154, 58, 174, 1, -1, -1], [1, 4, 371, 412, 80, 230, 1, -1, -1], [1, 5, 223, 403, 67, 188, 1, -1, -1], [1, 6, 1360, 567, 87, 227, 1, -1, -1]]

Maybe something wrong while loading txt files?

mikel-brostrom commented 11 months ago

Will investigate this further... Both type and dimensions matches...

<class 'numpy.ndarray'>
(69, 9)
<class 'numpy.ndarray'>
(6, 9)

Thank you for your feedback.

mikel-brostrom commented 11 months ago

Input:

[[   1    1 1475  419   75  169    0    0   -1]
 [   1    2  471  429   67  174    0    0   -1]
 [   1    3 1366  395   65  223    0    0   -1]
 [   1    4 1174  418   33  100    0    0   -1]
 [   1    5  730  423   49  171    0    0   -1]
 [   1    6  682  413   61  186    0    0   -1]
 [   1    7 1113  418   28   78    0    0   -1]
 [   1    8 1226  412   35  114    0    0   -1]
 [   1    9  553  434   56  167    0    0   -1]
 [   1   10  636  423   43  140    0    0   -1]
 [   1   11  400  432   23   54    0    0   -1]
 [   1   12  810  428   24   61    0    0   -1]
 [   1   13  584  430   57  170    0    0   -1]
 [   1   14 1201  423   22   75    0    0   -1]
 [   2    1 1464  420   79  168    0    0   -1]
 [   2    2  463  429   68  175    0    0   -1]
 [   2    3 1363  396   66  217    0    0   -1]
 [   2    4 1169  418   31   97    0    0   -1]
 [   2    5  725  417   47  174    0    0   -1]
 [   2    6  678  413   58  187    0    0   -1]
 [   2    7 1107  416   28   81    0    0   -1]
 [   2    8 1220  412   32   90    0    0   -1]
 [   2    9  546  433   45  166    0    0   -1]
 [   2   10  627  419   47  144    0    0   -1]
 [   2   11  392  428   24   57    0    0   -1]
 [   2   12  803  427   21   60    0    0   -1]
 [   2   14 1196  422   22   75    0    0   -1]
 [   2   15 1590  423   37  102    0    0   -1]
 [   3    1 1459  420   74  170    0    0   -1]
 [   3    2  455  430   67  173    0    0   -1]
 [   3    3 1360  397   65  218    0    0   -1]
 [   3    4 1160  419   32  102    0    0   -1]
 [   3    5  714  422   51  173    0    0   -1]
 [   3    6  669  411   63  190    0    0   -1]
 [   3    7 1101  417   27   80    0    0   -1]
 [   3    8 1213  412   34  112    0    0   -1]
 [   3    9  537  430   46  168    0    0   -1]
 [   3   10  621  423   44  140    0    0   -1]
 [   3   11  383  430   26   55    0    0   -1]
 [   3   13  568  427   63  172    0    0   -1]
 [   3   14 1189  426   24   69    0    0   -1]
 [   3   15 1584  424   36  102    0    0   -1]
 [   4    1 1455  422   73  169    0    0   -1]
 [   4    2  449  431   68  174    0    0   -1]
 [   4    3 1358  400   65  217    0    0   -1]
 [   4    4 1155  420   33  103    0    0   -1]
 [   4    5  707  419   51  174    0    0   -1]
 [   4    6  660  413   70  197    0    0   -1]
 [   4    7 1095  419   27   79    0    0   -1]
 [   4    8 1209  414   30  112    0    0   -1]
 [   4    9  532  432   48  168    0    0   -1]
 [   4   10  615  423   45  142    0    0   -1]
 [   4   11  374  428   26   58    0    0   -1]
 [   4   14 1182  420   25   83    0    0   -1]
 [   5    1 1444  425   68  170    0    0   -1]
 [   5    2  436  432   69  176    0    0   -1]
 [   5    3 1355  405   67  224    0    0   -1]
 [   5    4 1146  424   30   99    0    0   -1]
 [   5    6  652  415   73  195    0    0   -1]
 [   5    7 1087  420   27   82    0    0   -1]
 [   5    8 1197  418   31   76    0    0   -1]
 [   5    9  523  434   42  165    0    0   -1]
 [   5   10  603  425   46  138    0    0   -1]
 [   5   13  547  430   62  174    0    0   -1]
 [   5   15 1568  428   39  106    0    0   -1]]

gsi output:

[[1, 1, 1474, 419, 75, 168, 1, -1, -1], [1, 1, 1474, 419, 75, 168, 1, -1, -1], [1, 1, 1474, 419, 75, 168, 1, -1, -1], [1, 1, 1474, 419, 75, 168, 1, -1, -1], [1, 1, 1474, 419, 75, 168, 1, -1, -1], [1, 2, 470, 428, 67, 174, 1, -1, -1], [1, 2, 470, 428, 67, 174, 1, -1, -1], [1, 2, 470, 428, 67, 174, 1, -1, -1], [1, 2, 470, 428, 67, 174, 1, -1, -1], [1, 2, 470, 428, 67, 174, 1, -1, -1], [1, 3, 1365, 395, 65, 222, 1, -1, -1], [1, 3, 1365, 395, 65, 222, 1, -1, -1], [1, 3, 1365, 395, 65, 222, 1, -1, -1], [1, 3, 1365, 395, 65, 222, 1, -1, -1], [1, 3, 1365, 395, 65, 222, 1, -1, -1], [1, 4, 1174, 418, 32, 99, 1, -1, -1], [1, 4, 1174, 418, 32, 99, 1, -1, -1], [1, 4, 1174, 418, 32, 99, 1, -1, -1], [1, 4, 1174, 418, 32, 99, 1, -1, -1], [1, 4, 1174, 418, 32, 99, 1, -1, -1], [1, 5, 730, 422, 48, 171, 1, -1, -1], [1, 5, 730, 422, 48, 171, 1, -1, -1], [1, 5, 730, 422, 48, 171, 1, -1, -1], [1, 5, 730, 422, 48, 171, 1, -1, -1], [1, 6, 682, 413, 60, 185, 1, -1, -1], [1, 6, 682, 413, 60, 185, 1, -1, -1], [1, 6, 682, 413, 60, 185, 1, -1, -1], [1, 6, 682, 413, 60, 185, 1, -1, -1], [1, 6, 682, 413, 60, 185, 1, -1, -1], [1, 7, 1112, 417, 28, 78, 1, -1, -1], [1, 7, 1112, 417, 28, 78, 1, -1, -1], [1, 7, 1112, 417, 28, 78, 1, -1, -1], [1, 7, 1112, 417, 28, 78, 1, -1, -1], [1, 7, 1112, 417, 28, 78, 1, -1, -1], [1, 8, 1225, 412, 34, 109, 1, -1, -1], [1, 8, 1225, 412, 34, 109, 1, -1, -1], [1, 8, 1225, 412, 34, 109, 1, -1, -1], [1, 8, 1225, 412, 34, 109, 1, -1, -1], [1, 8, 1225, 412, 34, 109, 1, -1, -1], [1, 9, 553, 434, 55, 166, 1, -1, -1], [1, 9, 553, 434, 55, 166, 1, -1, -1], [1, 9, 553, 434, 55, 166, 1, -1, -1], [1, 9, 553, 434, 55, 166, 1, -1, -1], [1, 9, 553, 434, 55, 166, 1, -1, -1], [1, 10, 635, 422, 43, 140, 1, -1, -1], [1, 10, 635, 422, 43, 140, 1, -1, -1], [1, 10, 635, 422, 43, 140, 1, -1, -1], [1, 10, 635, 422, 43, 140, 1, -1, -1], [1, 10, 635, 422, 43, 140, 1, -1, -1], [1, 11, 400, 431, 22, 54, 1, -1, -1], [1, 11, 400, 431, 22, 54, 1, -1, -1], [1, 11, 400, 431, 22, 54, 1, -1, -1], [1, 11, 400, 431, 22, 54, 1, -1, -1], [1, 12, 809, 427, 23, 60, 1, -1, -1], [1, 12, 809, 427, 23, 60, 1, -1, -1], [1, 13, 583, 430, 56, 170, 1, -1, -1], [1, 13, 583, 430, 56, 170, 1, -1, -1], [1, 13, 583, 430, 56, 170, 1, -1, -1], [1, 13, 583, 430, 56, 170, 1, -1, -1], [1, 13, 583, 430, 56, 170, 1, -1, -1], [1, 14, 1201, 422, 21, 76, 1, -1, -1], [1, 14, 1201, 422, 21, 76, 1, -1, -1], [1, 14, 1201, 422, 21, 76, 1, -1, -1], [1, 14, 1201, 422, 21, 76, 1, -1, -1], [2, 15, 1590, 422, 36, 101, 1, -1, -1], [2, 15, 1590, 422, 36, 101, 1, -1, -1], [2, 15, 1590, 422, 36, 101, 1, -1, -1], [2, 15, 1590, 422, 36, 101, 1, -1, -1]]

Could it be some format input format error @dyhBUPT ? First column in my input data is frame_id, second is object id

mikel-brostrom commented 11 months ago

Can be reproduced by:

import os
import numpy as np
from os.path import join
from pathlib import Path
from collections import defaultdict
from sklearn.gaussian_process.kernels import RBF
from sklearn.gaussian_process import GaussianProcessRegressor as GPR

# 线性插值
def LinearInterpolation(input_, interval):
    input_ = input_[np.lexsort([input_[:, 0], input_[:, 1]])]  # 按ID和帧排序
    output_ = input_.copy()
    '''线性插值'''
    id_pre, f_pre, row_pre = -1, -1, np.zeros((10,))
    for row in input_:
        f_curr, id_curr = row[:2].astype(int)
        if id_curr == id_pre:  # 同ID
            if f_pre + 1 < f_curr < f_pre + interval:
                for i, f in enumerate(range(f_pre + 1, f_curr), start=1):  # 逐框插值
                    step = (row - row_pre) / (f_curr - f_pre) * i
                    row_new = row_pre + step
                    output_ = np.append(output_, row_new[np.newaxis, :], axis=0)
        else:  # 不同ID
            id_pre = id_curr
        row_pre = row
        f_pre = f_curr
    output_ = output_[np.lexsort([output_[:, 0], output_[:, 1]])]
    print(output_)
    return output_

# 高斯平滑
def GaussianSmooth(input_, tau):
    output_ = list()
    ids = set(input_[:, 1])
    for id_ in ids:
        tracks = input_[input_[:, 1] == id_]
        len_scale = np.clip(tau * np.log(tau ** 3 / len(tracks)), tau ** -1, tau ** 2)
        gpr = GPR(RBF(len_scale, 'fixed'))
        t = tracks[:, 0].reshape(-1, 1)
        x = tracks[:, 2].reshape(-1, 1)
        y = tracks[:, 3].reshape(-1, 1)
        w = tracks[:, 4].reshape(-1, 1)
        h = tracks[:, 5].reshape(-1, 1)
        gpr.fit(t, x)
        xx = gpr.predict(t)[0]
        gpr.fit(t, y)
        yy = gpr.predict(t)[0]
        gpr.fit(t, w)
        ww = gpr.predict(t)[0]
        gpr.fit(t, h)
        hh = gpr.predict(t)[0]
        output_.extend([
            [int(t[0]), int(id_), int(xx), int(yy), int(ww), int(hh), 1, -1, -1] for i in range(len(t))
        ])
    return output_

def gsi_interpolation(mot_results_folder=Path('examples/runs/val/exp51/labels'), interval=20, tau=10):
    tracking_results = np.array([
        [   1,    1, 1475,  419,   75,  169,    0,    0,   -1],
        [   1,    2,  471,  429,   67,  174,    0,    0 ,  -1],
        [   1,    3, 1366,  395,   65,  223,    0,    0 ,  -1],
        [   1,    4, 1174,  418,   33,  100,    0,    0 ,  -1],
        [   1,    5,  730,  423,   49,  171,    0,    0 ,  -1],
        [   1,    6,  682,  413,   61,  186 ,   0,    0 ,  -1],
        [   1,    7, 1113,  418,   28,   78,    0,    0 ,  -1],
        [   2,    1, 1475,  419,   75,  169,    0,    0,   -1],
        [   2,    2,  471,  429,   67,  174,    0,    0 ,  -1],
        [   2,    3, 1366,  395,   65,  223,    0,    0 ,  -1],
        [   2,    4, 1174,  418,   33,  100,    0,    0 ,  -1],
        [   2,    5,  730,  423,   49,  171,    0,    0 ,  -1],
        [   2,    6,  682,  413,   61,  186 ,   0,    0 ,  -1],
        [   2,    7, 1113,  418,   28,   78,    0,    0 ,  -1],
    ])
    li = LinearInterpolation(tracking_results, interval)
    gsi = GaussianSmooth(li, tau)
    print(gsi)

dyhBUPT commented 11 months ago

Hi, there is a mistake in your codes. Please change

output_.extend([
    [int(t[0]), int(id_), int(xx), int(yy), int(ww), int(hh), 1, -1, -1] for i in range(len(t))
])

to

output_.extend([
    [int(t[i, 0]), int(id_), int(xx), int(yy), int(ww), int(hh), 1, -1, -1] for i in range(len(t))
])

That is, t[0] -> t[i, 0].

mikel-brostrom commented 11 months ago

Thanks for your guidance @dyhBUPT. Have it working as this:

from pathlib import Path

import numpy as np
from sklearn.gaussian_process import GaussianProcessRegressor as GPR
from sklearn.gaussian_process.kernels import RBF

from boxmot.utils import logger as LOGGER

def linear_interpolation(input_, interval):
    input_ = input_[np.lexsort([input_[:, 0], input_[:, 1]])]
    output_ = input_.copy()

    id_pre, f_pre, row_pre = -1, -1, np.zeros((10,))
    for row in input_:
        f_curr, id_curr = row[:2].astype(int)
        if id_curr == id_pre:
            if f_pre + 1 < f_curr < f_pre + interval:
                for i, f in enumerate(range(f_pre + 1, f_curr), start=1):
                    step = (row - row_pre) / (f_curr - f_pre) * i
                    row_new = row_pre + step
                    output_ = np.append(output_, row_new[np.newaxis, :], axis=0)
        else:
            id_pre = id_curr
        row_pre = row
        f_pre = f_curr
    output_ = output_[np.lexsort([output_[:, 0], output_[:, 1]])]
    return output_

def gaussian_smooth(input_, tau):
    output_ = list()
    print('input_', input_)
    ids = set(input_[:, 1])
    for i, id_ in enumerate(ids):
        tracks = input_[input_[:, 1] == id_]
        print('tracks', tracks)
        len_scale = np.clip(tau * np.log(tau ** 3 / len(tracks)), tau ** -1, tau ** 2)
        gpr = GPR(RBF(len_scale, 'fixed'))
        t = tracks[:, 0].reshape(-1, 1)
        x = tracks[:, 2].reshape(-1, 1)
        y = tracks[:, 3].reshape(-1, 1)
        w = tracks[:, 4].reshape(-1, 1)
        h = tracks[:, 5].reshape(-1, 1)
        gpr.fit(t, x)
        xx = gpr.predict(t)
        gpr.fit(t, y)
        yy = gpr.predict(t)
        gpr.fit(t, w)
        ww = gpr.predict(t)
        gpr.fit(t, h)
        hh = gpr.predict(t)
        # frame count, id, x, y, w, h, conf, cls, -1 (don't care)
        output_.extend([
            [t[j, 0], id_, xx[j], yy[j], ww[j], hh[j], tracks[j, 6], tracks[j, 7], -1] for j in range(len(t))
        ])
    return output_

def gsi(mot_results_folder=Path('examples/runs/val/exp87/labels'), interval=20, tau=10):
    tracking_results_files = mot_results_folder.glob('MOT*FRCNN.txt')
    for p in tracking_results_files:
        LOGGER.info(f"Applying gaussian smoothed interpolation (GSI) to: {p}")
        tracking_results = np.loadtxt(p, dtype=int, delimiter=' ')
        if tracking_results.size != 0:
            li = linear_interpolation(tracking_results, interval)
            gsi = gaussian_smooth(li, tau)
            np.savetxt(p, gsi, fmt='%d %d %d %d %d %d %d %d %d')
        else:
            print('No tracking result in {p}. Skipping...')

gsi()

Needed some minor modifications though. Made this applicable to any tracker here: https://github.com/mikel-brostrom/yolo_tracking. Hope you are okay with this

dyhBUPT commented 11 months ago

That's good, thanks for your excellent contributions~

dyhBUPT / StrongSORT

GSI gives multiple track results with same ID for same frame #92