uwdata / arquero

Query processing and transformation of array-backed data tables.
https://idl.uw.edu/arquero
BSD 3-Clause "New" or "Revised" License
1.22k stars 64 forks source link

Table concatenation results in empty table #329

Open alexbezhan opened 1 year ago

alexbezhan commented 1 year ago

If I have multiple tables that I want to concatenate, if the first table is empty, then the final concatenated table is also empty. Code:

export async function fetchArrowFilesAndConcat(fetches: Parameters<typeof fetch>[0][]) {
    let table: ColumnTable | undefined = undefined
    for (const f of fetches) {
        const response = await fetch(f)
        const responseData = await response.arrayBuffer()
        const newTable = aq.fromArrow(responseData)
        table = table === undefined ? newTable : table.concat(newTable)
    }
    assertDefined(table, `table must be defined`)
    return table // table is empty here if the first table was empty
}

And this is a fixed version - I skip the first empty table:

export async function fetchArrowFilesAndConcat(fetches: Parameters<typeof fetch>[0][]) {
    let table: ColumnTable | undefined = undefined
    let emptyTableSkipped = false
    for (const f of fetches) {
        const response = await fetch(f)
        const responseData = await response.arrayBuffer()
        const newTable = aq.fromArrow(responseData)
        const numRows = newTable.numRows()
        if (numRows === 0 && emptyTableSkipped === false) {
            emptyTableSkipped = true
            continue // skip first empty table, because it's causing the final table to be empty as well
        }
        table = table === undefined ? newTable : table.concat(newTable)
    }
    assertDefined(table, `table must be defined`)
    return table
}