streaming db data

[prayagupa]

• http://www.pipelinedb.com/
• https://github.com/duwamish-os/data-utkhanan

try (var statement = db.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);) {
                statement.setFetchSize(10);

}

[prayagupa]

-- stream db data

## https://stackoverflow.com/questions/17933344/python-postgres-can-i-fetchall-1-million-rows

import psycopg2

lookup_sql_all = """select * from state_history event_date::DATE > '2020-01-01' LIMIT 5000000"""

con = psycopg2.connect(
    host="localhost",
    port=5439,
    database="history",
    user="admin",
    password="admin"
)

db_cursor.execute(lookup_sql_all)

batch = 1
while True:
    records = cursor.fetchmany(size=20000)

    if not records:
        break

    print("batch: " + str(batch))
    batch = batch + 1