idpy async http client

[peppelinux]

We often talked in our idpy meetings about the possibility to speed up the http requests using an async approach. Well, me and Roland started this with asyncio and an abstract class with the same property of the python requests objects, this allows us to leave the legacy code as it is.

The following PoC was inspired by the code already available in spid-cie-oidc-django and it has the following deps:

aiohttp            3.8.1       
aiosignal          1.2.0       
asgiref            3.5.2

Code:

import aiohttp
import asyncio
from asgiref.sync import sync_to_async

from typing import Union

class IdpyHttpResponse:

    def __init__(self, 
        url: str, httpc_params: Union[dict, None] = None, **kwargs
    ):
        for k,v in kwargs.items():
            setattr(self, k, v) 

        self.url = url
        self.response: bytes = kwargs.get('response', b"")
        self.json: str = kwargs.get('json', "")
        self.status_code: int = kwargs.get('status_code', 0)

    def __str__(self):
        return f"{self.url} [{self.status_code}]"

    def __repr__(self):
        return self.__str__()

async def fetch(session, url, httpc_params: dict = {}):
    result = {}
    try:
        async with session.get(url, **httpc_params.get("connection", {})) as response:
            if response.status != 200: # pragma: no cover
                # response.raise_for_status()
                result = await sync_to_async(dict)(
                    url = url,
                    response = b"", 
                    status_code = response.status
                )
            else:
                result = await sync_to_async(dict)(
                    url = url,
                    response = await response.text(),
                    status_code = response.status
                )
    except Exception as e:
        result = await sync_to_async(dict)(
            url = url,
            response = b"",
            status_code = 0,
            exception = f"{e}"
        )
    return result

async def fetch_all(session, urls, httpc_params):
    tasks = []
    for url in urls:
        task = asyncio.create_task(fetch(session, url, httpc_params))
        tasks.append(task)
    results = await asyncio.gather(*tasks)
    return results

async def http_get(urls, httpc_params: dict = {}):
    async with aiohttp.ClientSession(**httpc_params.get("session", {})) as session:
        text = await fetch_all(session, urls, httpc_params)
        return text

if __name__ == "__main__": # pragma: no cover
    httpc_params = {
        "connection": {"ssl": True},
        "session": {"timeout": aiohttp.ClientTimeout(total=14)},
    }

    urls = [
        "http://google.it",
        "https://aasidhasudhasiusdhsaidhaisud.eu"
    ]
    responses = asyncio.run(http_get(urls, httpc_params=httpc_params))
    print(responses)

    responses_objs = [
        IdpyHttpResponse(**i)
        for i in responses 
    ]
    print(responses_objs)

[infohash]

How will sync methods call this async method? Sync methods cannot spawn another event loop if the underlying framework is already running one. We might have to port this library to async the entire chain. As async is a hot trend in async frameworks like FastAPI and Starlite, they will be benefitted from a complete async OIDC lib.