encode / databases

Async database support for Python. 🗄
https://www.encode.io/databases/
BSD 3-Clause "New" or "Revised" License
3.8k stars 260 forks source link

execute_many insert sql do not have high preformance? #152

Open ShichaoMa opened 4 years ago

ShichaoMa commented 4 years ago
    async def execute_many(self, queries: typing.List[ClauseElement]) -> None:
        assert self._connection is not None, "Connection is not acquired"
        cursor = await self._connection.cursor()
        try:
            for single_query in queries:
                single_query, args, context = self._compile(single_query)
                await cursor.execute(single_query, args)
        finally:
            await cursor.close()

execute single sql many times?

aiomysql sames work better

    async def executemany(self, query, args):
        """Execute the given operation multiple times

        The executemany() method will execute the operation iterating
        over the list of parameters in seq_params.

        Example: Inserting 3 new employees and their phone number

            data = [
                ('Jane','555-001'),
                ('Joe', '555-001'),
                ('John', '555-003')
                ]
            stmt = "INSERT INTO employees (name, phone) VALUES ('%s','%s')"
            await cursor.executemany(stmt, data)

        INSERT or REPLACE statements are optimized by batching the data,
        that is using the MySQL multiple rows syntax.

        :param query: `str`, sql statement
        :param args: ``tuple`` or ``list`` of arguments for sql query
        """
        if not args:
            return

        if self._echo:
            logger.info("CALL %s", query)
            logger.info("%r", args)

        m = RE_INSERT_VALUES.match(query)
        if m:
            q_prefix = m.group(1)
            q_values = m.group(2).rstrip()
            q_postfix = m.group(3) or ''
            assert q_values[0] == '(' and q_values[-1] == ')'
            return (await self._do_execute_many(
                q_prefix, q_values, q_postfix, args, self.max_stmt_length,
                self._get_db().encoding))
        else:
            rows = 0
            for arg in args:
                await self.execute(query, arg)
                rows += self._rowcount
            self._rowcount = rows
        return self._rowcount
ljluestc commented 5 months ago
import typing
from sqlalchemy.sql import ClauseElement

async def execute_many(self, queries: typing.List[ClauseElement]) -> None:
    assert self._connection is not None, "Connection is not acquired"
    cursor = await self._connection.cursor()
    try:
        prepared_queries = []
        for single_query in queries:
            single_query, args, context = self._compile(single_query)
            prepared_queries.append((single_query, args))
        await cursor.executemany(prepared_queries)
    finally:
        await cursor.close()