Asynchronous requests

[narlesky]

Explore ayncio options for improving speed of multi-water year requests

see excerpt from realtime-reservoir

import aiohttp
import asyncio
import async_timeout
import os

reservoirs = ['ORDC1','NBBC1','FOLC1','SHDC1']
rivers = ['GRIC1','YUBC1','MRYC1','SAMC1','SACC1','BDBC1','FBLC1','BRWC1','FMWC1']

async def download_coroutine(session, url):
    with async_timeout.timeout(10):
        async with session.get(url) as response:
            filename = '{}'.format(os.path.basename(url))
            with open(filename, 'wb') as f:
                while True:
                    chunk = await response.content.read(1024)
                    if not chunk:
                        break
                    f.write(chunk)
            return await response.release()

async def main(loop):
    reservoir_urls = ['http://www.cnrfc.noaa.gov/csv/{}_hefs_csv_hourly.csv'.format(x) for x in reservoirs]
    forecast_urls = ['http://www.cnrfc.noaa.gov/graphicalRVF_csv.php?id={}'.format(x) for x in rivers]

    async with aiohttp.ClientSession(loop=loop) as session:
        for url in reservoir_urls:
            await download_coroutine(session, url)

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main(loop))