PennyDreadfulMTG / Penny-Dreadful-Tools

A suite of tools for the Penny Dreadful MTGO community
https://pennydreadfulmagic.com
MIT License
40 stars 28 forks source link

Command failed with DatabaseException: !spoiler chemister's insight #5355

Closed vorpal-buildbot closed 6 years ago

vorpal-buildbot commented 6 years ago
Traceback (most recent call last):
  File "/home/discord/Penny-Dreadful-Discord-Bot/magic/oracle.py", line 100, in scryfall_import
    valid_name(sfcard['name'])
  File "/home/discord/Penny-Dreadful-Discord-Bot/magic/oracle.py", line 28, in valid_name
    raise InvalidDataException('Did not find any cards looking for `{name}`'.format(name=name))
shared.pd_exception.InvalidDataException: Did not find any cards looking for `Chemister's Insight`

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/discord/Penny-Dreadful-Discord-Bot/shared/database.py", line 52, in execute_anything
    return self.execute_with_reconnect(sql, args, fetch_rows)
  File "/home/discord/Penny-Dreadful-Discord-Bot/shared/database.py", line 82, in execute_with_reconnect
    raise e
  File "/home/discord/Penny-Dreadful-Discord-Bot/shared/database.py", line 68, in execute_with_reconnect
    n = self.cursor.execute(sql, args)
  File "/home/discord/.local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 250, in execute
    self.errorhandler(self, exc, value)
  File "/home/discord/.local/lib/python3.6/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
    raise errorvalue
  File "/home/discord/.local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 247, in execute
    res = self._query(query)
  File "/home/discord/.local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 412, in _query
    rowcount = self._do_query(q)
  File "/home/discord/.local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 375, in _do_query
    db.query(q)
  File "/home/discord/.local/lib/python3.6/site-packages/MySQLdb/connections.py", line 276, in query
    _mysql.connection.query(self, query)
_mysql_exceptions.OperationalError: (1048, "Column 'position' cannot be null")

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/discord/Penny-Dreadful-Discord-Bot/discordbot/command.py", line 75, in handle_command
    await method(Commands, client=client, channel=message.channel, args=args, author=message.author)
  File "/home/discord/Penny-Dreadful-Discord-Bot/discordbot/command.py", line 410, in spoiler
    oracle.scryfall_import(sfcard['name'])
  File "/home/discord/Penny-Dreadful-Discord-Bot/magic/oracle.py", line 103, in scryfall_import
    insert_scryfall_card(sfcard)
  File "/home/discord/Penny-Dreadful-Discord-Bot/magic/oracle.py", line 137, in insert_scryfall_card
    multiverse.update_cache()
  File "/home/discord/Penny-Dreadful-Discord-Bot/magic/multiverse.py", line 271, in update_cache
    db().execute(create_table_def('_new_cache_card', card.base_query_properties(), base_query()))
  File "/home/discord/Penny-Dreadful-Discord-Bot/shared/database.py", line 45, in execute
    [n, _] = self.execute_anything(sql, args, False)
  File "/home/discord/Penny-Dreadful-Discord-Bot/shared/database.py", line 60, in execute_anything
    raise DatabaseException('Failed to execute `{sql}` with `{args}` because of `{e}`'.format(sql=sql, args=args, e=e))
shared.pd_exception.DatabaseException: Failed to execute `CREATE TABLE IF NOT EXISTS `_new_cache_card` (`id` INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT  , `name` VARCHAR(190) NOT NULL   , `mana_cost` VARCHAR(190)    , `cmc` REAL    , `power` VARCHAR(190)    , `toughness` VARCHAR(190)    , `loyalty` VARCHAR(190)    , `type` VARCHAR(190) NOT NULL   , `text` LONGTEXT NOT NULL   , `search_text` LONGTEXT NOT NULL   , `image_name` VARCHAR(190)    , `hand` INTEGER    , `life` INTEGER    , `starter` INTEGER    , `position` INTEGER NOT NULL   , `name_ascii` VARCHAR(190)    , `card_id` INTEGER    , `layout` VARCHAR(190) NOT NULL   , `names` LONGTEXT    , `legalities` LONGTEXT    , `pd_legal` BOOLEAN    , `bugs` LONGTEXT    , FOREIGN KEY(`card_id`) REFERENCES `card`(`id`)) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci AS 
        SELECT
            `u`.`id` AS `id`, 
        CASE
        WHEN layout = 'double-faced' OR layout = 'flip' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN face_name ELSE '' END SEPARATOR '')
        WHEN layout = 'meld' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN face_name ELSE '' END SEPARATOR '')
        ELSE
            GROUP_CONCAT(face_name SEPARATOR ' // ' )
        END
     AS name, GROUP_CONCAT(`u`.`mana_cost` SEPARATOR '|') AS `mana_cost`, 
        CASE
        WHEN layout = 'split' OR layout = 'aftermath' THEN
            SUM(`u`.cmc)
        WHEN layout = 'meld' THEN
            SUM(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN cmc ELSE 0 END)
        ELSE
            SUM(CASE WHEN `u`.position = 1 THEN `u`.cmc ELSE 0 END)
        END
     AS cmc, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`power` ELSE '' END SEPARATOR '') AS `power`, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`toughness` ELSE '' END SEPARATOR '') AS `toughness`, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`loyalty` ELSE '' END SEPARATOR '') AS `loyalty`, 
        CASE
        WHEN layout = 'meld' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN type ELSE '' END SEPARATOR '')
        ELSE
            GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN type ELSE '' END SEPARATOR '')
        END
     AS type, GROUP_CONCAT(`u`.`text` SEPARATOR '
-----
') AS `text`, GROUP_CONCAT(`u`.`search_text` SEPARATOR '
-----
') AS `search_text`, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`image_name` ELSE '' END SEPARATOR '') AS `image_name`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`hand` ELSE NULL END) AS `hand`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`life` ELSE NULL END) AS `life`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`starter` ELSE NULL END) AS `starter`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`position` ELSE NULL END) AS `position`, 
        CASE
        WHEN layout = 'double-faced' OR layout = 'flip' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN name_ascii ELSE '' END SEPARATOR '')
        WHEN layout = 'meld' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN name_ascii ELSE '' END SEPARATOR '')
        ELSE
            GROUP_CONCAT(name_ascii SEPARATOR ' // ' )
        END
     AS name_ascii, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`card_id` ELSE NULL END) AS `card_id`, `u`.`layout` AS `layout`, GROUP_CONCAT(face_name SEPARATOR '|') AS names, legalities, `u`.`pd_legal` AS `pd_legal`, bugs
        FROM (
            SELECT c.id, c.layout, f.mana_cost, f.cmc, f.power, f.toughness, f.loyalty, f.type, f.text, f.search_text, f.image_name, f.hand, f.life, f.starter, f.position, f.name_ascii, f.card_id, f.name AS face_name,
                pd_legal,
                legalities
            FROM
                card AS c
            INNER JOIN
                face AS f ON c.id = f.card_id
            LEFT JOIN (
                SELECT
                    cl.card_id,
                    SUM(CASE WHEN cl.format_id = 21 THEN 1 ELSE 0 END) > 0 AS pd_legal,
                    GROUP_CONCAT(CONCAT(fo.name, ':', cl.legality)) AS legalities
                FROM
                    card_legality AS cl
                LEFT JOIN
                    format AS fo ON cl.format_id = fo.id
                GROUP BY
                    cl.card_id
            ) AS cl ON cl.card_id = c.id
            GROUP BY
                f.id
            ORDER BY
                f.card_id, f.position
        ) AS u
        LEFT JOIN (
            SELECT
                cb.card_id,
                GROUP_CONCAT(CONCAT(cb.description, '|', cb.classification, '|', cb.last_confirmed, '|', cb.url, '|', cb.from_bug_blog, '|', cb.bannable) SEPARATOR '_SEPARATOR_') AS bugs
            FROM
                card_bug AS cb
            GROUP BY
                cb.card_id
        ) AS bugs ON u.id = bugs.card_id
        WHERE u.id IN (SELECT c.id FROM card AS c INNER JOIN face AS f ON c.id = f.card_id WHERE (1 = 1))
        GROUP BY u.id
    ` with `[]` because of `(1048, "Column 'position' cannot be null")`

Reported on Discord by bakert#2193

bakert commented 6 years ago

This does not occur on dev nor in a private message with the bot, which is kind of weird.

bakert commented 6 years ago

Issue seems to be multiple entries for new GRN cards in discordbot cards db:

MariaDB [discordbot]> SELECT * FROM face WHERE name = 'Chemister''s Insight';
+-------+---------------------+-----------+------+-------+-----------+---------+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------+------+------+---------+----------+---------------------+---------+
| id    | name                | mana_cost | cmc  | power | toughness | loyalty | type    | text                                                                                                                                                      | search_text                 | image_name | hand | life | starter | position | name_ascii          | card_id |
+-------+---------------------+-----------+------+-------+-----------+---------+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------+------+------+---------+----------+---------------------+---------+
| 18839 | Chemister's Insight | {3}{U}    |    4 | NULL  | NULL      | NULL    | Instant | Draw two cards.
Jump-start (You may cast this card from your graveyard by discarding a card in addition to paying its other costs. Then exile this card.) | Draw two cards.
Jump-start  | grn_32     | NULL | NULL |    NULL |        1 | Chemister's Insight |   18639 |
| 18841 | Chemister's Insight | {3}{U}    |    4 | NULL  | NULL      | NULL    | Instant | Draw two cards.
Jump-start (You may cast this card from your graveyard by discarding a card in addition to paying its other costs. Then exile this card.) | Draw two cards.
Jump-start  | grn_32     | NULL | NULL |    NULL |        1 | Chemister's Insight |   18641 |
| 18843 | Chemister's Insight | {3}{U}    |    4 | NULL  | NULL      | NULL    | Instant | Draw two cards.
Jump-start (You may cast this card from your graveyard by discarding a card in addition to paying its other costs. Then exile this card.) | Draw two cards.
Jump-start  | grn_32     | NULL | NULL |    NULL |        1 | Chemister's Insight |   18641 |
| 18844 | Chemister's Insight | {3}{U}    |    4 | NULL  | NULL      | NULL    | Instant | Draw two cards.
Jump-start (You may cast this card from your graveyard by discarding a card in addition to paying its other costs. Then exile this card.) | Draw two cards.
Jump-start  | grn_32     | NULL | NULL |    NULL |        1 | Chemister's Insight |   18641 |
| 18845 | Chemister's Insight | {3}{U}    |    4 | NULL  | NULL      | NULL    | Instant | Draw two cards.
Jump-start (You may cast this card from your graveyard by discarding a card in addition to paying its other costs. Then exile this card.) | Draw two cards.
Jump-start  | grn_32     | NULL | NULL |    NULL |        1 | Chemister's Insight |   18641 |
| 18847 | Chemister's Insight | {3}{U}    |    4 | NULL  | NULL      | NULL    | Instant | Draw two cards.
Jump-start (You may cast this card from your graveyard by discarding a card in addition to paying its other costs. Then exile this card.) | Draw two cards.
Jump-start  | grn_32     | NULL | NULL |    NULL |        1 | Chemister's Insight |   18644 |
| 18848 | Chemister's Insight | {3}{U}    |    4 | NULL  | NULL      | NULL    | Instant | Draw two cards.
Jump-start (You may cast this card from your graveyard by discarding a card in addition to paying its other costs. Then exile this card.) | Draw two cards.
Jump-start  | grn_32     | NULL | NULL |    NULL |        1 | Chemister's Insight |   18645 |
+-------+---------------------+-----------+------+-------+-----------+---------+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------+------------+------+------+---------+----------+---------------------+---------+
7 rows in set (0.06 sec)
bakert commented 6 years ago
MariaDB [discordbot]> SELECT name, COUNT(*) FROM face GROUP BY name HAVING COUNT(name) > 1;
+--------------------------------+----------+
| name                           | COUNT(*) |
+--------------------------------+----------+
| Brisela, Voice of Nightmares   |        2 |
| Chemister's Insight            |        7 |
| Chittering Host                |        2 |
| Hanweir, the Writhing Township |        2 |
| Vraska, Golgari Queen Emblem   |        4 |
+--------------------------------+----------+
5 rows in set (0.07 sec)
bakert commented 6 years ago

The root cause here is that we have bad data for the partner cards from battlebond coming from mtgjson. I patched this in multiverse but the bot db hasn't been rebuilt since then. I have fake updated the version of mtgjson to a prior version in the bot's card db and restarted the bot. That should do the trick.

bakert commented 6 years ago
SELECT
            `u`.`id` AS `id`, 
        CASE
        WHEN layout = 'double-faced' OR layout = 'flip' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN face_name ELSE '' END SEPARATOR '')
        WHEN layout = 'meld' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN face_name ELSE '' END SEPARATOR '')
        ELSE
            GROUP_CONCAT(face_name SEPARATOR ' // ' )
        END
     AS name, GROUP_CONCAT(`u`.`mana_cost` SEPARATOR '|') AS `mana_cost`, 
        CASE
        WHEN layout = 'split' OR layout = 'aftermath' THEN
            SUM(`u`.cmc)
        WHEN layout = 'meld' THEN
            SUM(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN cmc ELSE 0 END)
        ELSE
            SUM(CASE WHEN `u`.position = 1 THEN `u`.cmc ELSE 0 END)
        END
     AS cmc, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`power` ELSE '' END SEPARATOR '') AS `power`, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`toughness` ELSE '' END SEPARATOR '') AS `toughness`, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`loyalty` ELSE '' END SEPARATOR '') AS `loyalty`, 
        CASE
        WHEN layout = 'meld' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN type ELSE '' END SEPARATOR '')
        ELSE
            GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN type ELSE '' END SEPARATOR '')
        END
     AS type, GROUP_CONCAT(`u`.`text` SEPARATOR '
-----
') AS `text`, GROUP_CONCAT(`u`.`search_text` SEPARATOR '
-----
') AS `search_text`, GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN `u`.`image_name` ELSE '' END SEPARATOR '') AS `image_name`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`hand` ELSE NULL END) AS `hand`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`life` ELSE NULL END) AS `life`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`starter` ELSE NULL END) AS `starter`, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`position` ELSE NULL END) AS `position`, 
        CASE
        WHEN layout = 'double-faced' OR layout = 'flip' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 THEN name_ascii ELSE '' END SEPARATOR '')
        WHEN layout = 'meld' THEN
            GROUP_CONCAT(CASE WHEN `u`.position = 1 OR `u`.position = 2 THEN name_ascii ELSE '' END SEPARATOR '')
        ELSE
            GROUP_CONCAT(name_ascii SEPARATOR ' // ' )
        END
     AS name_ascii, SUM(CASE WHEN `u`.position = 1  OR (`u`.position = 2 AND `u`.layout = 'meld') THEN `u`.`card_id` ELSE NULL END) AS `card_id`, `u`.`layout` AS `layout`, GROUP_CONCAT(face_name SEPARATOR '|') AS names, legalities, `u`.`pd_legal` AS `pd_legal`, bugs
        FROM (
            SELECT c.id, c.layout, f.mana_cost, f.cmc, f.power, f.toughness, f.loyalty, f.type, f.text, f.search_text, f.image_name, f.hand, f.life, f.starter, f.position, f.name_ascii, f.card_id, f.name AS face_name,
                pd_legal,
                legalities
            FROM
                card AS c
            INNER JOIN
                face AS f ON c.id = f.card_id
            LEFT JOIN (
                SELECT
                    cl.card_id,
                    SUM(CASE WHEN cl.format_id = 21 THEN 1 ELSE 0 END) > 0 AS pd_legal,
                    GROUP_CONCAT(CONCAT(fo.name, ':', cl.legality)) AS legalities
                FROM
                    card_legality AS cl
                LEFT JOIN
                    format AS fo ON cl.format_id = fo.id
                GROUP BY
                    cl.card_id
            ) AS cl ON cl.card_id = c.id
            GROUP BY
                f.id
            ORDER BY
                f.card_id, f.position
        ) AS u
        LEFT JOIN (
            SELECT
                cb.card_id,
                GROUP_CONCAT(CONCAT(cb.description, '|', cb.classification, '|', cb.last_confirmed, '|', cb.url, '|', cb.from_bug_blog, '|', cb.bannable) SEPARATOR '_SEPARATOR_') AS bugs
            FROM
                card_bug AS cb
            GROUP BY
                cb.card_id
        ) AS bugs ON u.id = bugs.card_id
        WHERE u.id IN (SELECT c.id FROM card AS c INNER JOIN face AS f ON c.id = f.card_id WHERE (1 = 1))
        GROUP BY u.id

***HAVING position IS NULL***

is what revealed the culprits.

bakert commented 6 years ago

Seemed to do it.