datajoint / datajoint-python

Relational data pipelines for the science lab
https://datajoint.com/docs
GNU Lesser General Public License v2.1
169 stars 85 forks source link

`dj.kill` struggles to display blob inserts #718

Open dimitri-yatsenko opened 4 years ago

dimitri-yatsenko commented 4 years ago

sometimes when an insert statement has binary data in it, dj.kill() will raise this error:

---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-3-6ccec7f856cd> in <module>
----> 1 dj.kill()

~/dev/datajoint-python/datajoint/admin.py in kill(restriction, connection)
     45         print('  ID USER         STATE         TIME  INFO')
     46         print('+--+ +----------+ +-----------+ +--+')
---> 47         cur = connection.query(query, as_dict=True)
     48         for process in cur:
     49             try:

~/dev/datajoint-python/datajoint/connection.py in query(self, query, args, as_dict, suppress_warnings, reconnect)
    209         cursor = self._conn.cursor(cursor=cursor_class)
    210         try:
--> 211             self.__execute_query(cursor, query, args, cursor_class, suppress_warnings)
    212         except errors.LostConnectionError:
    213             if not reconnect:

~/dev/datajoint-python/datajoint/connection.py in __execute_query(cursor, query, args, cursor_class, suppress_warnings)
    189                     # suppress all warnings arising from underlying SQL library
    190                     warnings.simplefilter("ignore")
--> 191                 cursor.execute(query, args)
    192         except client_errors as err:
    193             raise translate_query_error(err, query) from None

~/.local/lib/python3.7/site-packages/pymysql/cursors.py in execute(self, query, args)
    168         query = self.mogrify(query, args)
    169 
--> 170         result = self._query(query)
    171         self._executed = query
    172         return result

~/.local/lib/python3.7/site-packages/pymysql/cursors.py in _query(self, q)
    326         self._last_executed = q
    327         self._clear_result()
--> 328         conn.query(q)
    329         self._do_get_result()
    330         return self.rowcount

~/.local/lib/python3.7/site-packages/pymysql/connections.py in query(self, sql, unbuffered)
    515                 sql = sql.encode(self.encoding, 'surrogateescape')
    516         self._execute_command(COMMAND.COM_QUERY, sql)
--> 517         self._affected_rows = self._read_query_result(unbuffered=unbuffered)
    518         return self._affected_rows
    519 

~/.local/lib/python3.7/site-packages/pymysql/connections.py in _read_query_result(self, unbuffered)
    730         else:
    731             result = MySQLResult(self)
--> 732             result.read()
    733         self._result = result
    734         if result.server_status is not None:

~/.local/lib/python3.7/site-packages/pymysql/connections.py in read(self)
   1080                 self._read_load_local_packet(first_packet)
   1081             else:
-> 1082                 self._read_result_packet(first_packet)
   1083         finally:
   1084             self.connection = None

~/.local/lib/python3.7/site-packages/pymysql/connections.py in _read_result_packet(self, first_packet)
   1150         self.field_count = first_packet.read_length_encoded_integer()
   1151         self._get_descriptions()
-> 1152         self._read_rowdata_packet()
   1153 
   1154     def _read_rowdata_packet_unbuffered(self):

~/.local/lib/python3.7/site-packages/pymysql/connections.py in _read_rowdata_packet(self)
   1188                 self.connection = None  # release reference to kill cyclic reference.
   1189                 break
-> 1190             rows.append(self._read_row_from_packet(packet))
   1191 
   1192         self.affected_rows = len(rows)

~/.local/lib/python3.7/site-packages/pymysql/connections.py in _read_row_from_packet(self, packet)
   1204             if data is not None:
   1205                 if encoding is not None:
-> 1206                     data = data.decode(encoding)
   1207                 if DEBUG: print("DEBUG: DATA = ", data)
   1208                 if converter is not None:

UnicodeDecodeError: 'utf-8' codec can't decode byte 0xed in position 528: invalid continuation byte

Seems more of a pymysql error. It should not try to decode the string since it's raw binary data.

ixcat commented 4 years ago

taking to review options to processlist