[BUG] org.tikv.common.exception.TiClientInternalException: Invalid JSON value for Type TiTableInfo:

[melin]

2023-04-20T18:09:33,920 WARN [main] c.p.t.c.CatalogTransaction: fail to parse table from json! org.tikv.common.exception.TiClientInternalException: Invalid JSON value for Type TiTableInfo: {"id":70,"name":{"O":"tidb_mdl_view","L":"tidb_mdl_view"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"job_id","L":"job_id"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":0},{"id":2,"name":{"O":"db_name","L":"db_name"},"offset":1,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":0},{"id":3,"name":{"O":"table_name","L":"table_name"},"offset":2,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":0},{"id":4,"name":{"O":"query","L":"query"},"offset":3,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":0},{"id":5,"name":{"O":"session_id","L":"session_id"},"offset":4,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":0},{"id":6,"name":{"O":"txnstart","L":"txnstart"},"offset":5,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":0},{"id":7,"name":{"O":"SQL_DIGESTS","L":"sql_digests"},"offset":6,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":0}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":7,"max_idx_id":0,"max_fk_id":0,"max_cst_id":0,"update_timestamp":440921600703594503,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":{"view_algorithm":0,"view_definer":{"Username":"","Hostname":"","CurrentUser":true,"AuthUsername":"","AuthHostname":"","AuthPlugin":""},"view_security":0,"view_select":"(SELECT job_id AS job_id,db_name AS db_name,table_name AS table_name,query AS query,session_id AS session_id,txnstart AS txnstart,TIDB_DECODE_SQL_DIGESTS(all_sql_digests, 4096) AS SQL_DIGESTS FROM ((information_schema.ddl_jobs) JOIN information_schema.cluster_tidb_trx) JOIN information_schema.cluster_processlist WHERE (ddl_jobs.state!=_UTF8MB4'synced' AND ddl_jobs.state!=_UTF8MB4'cancelled') AND FIND_IN_SET(ddl_jobs.table_id, cluster_tidb_trx.related_table_ids) AND cluster_tidb_trx.session_id=cluster_processlist.id)","view_checkoption":1,"view_cols":null},"sequence":null,"Lock":null,"version":5,"tiflash_replica":null,"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null}

at com.pingcap.tikv.catalog.CatalogTransaction.parseFromJson(CatalogTransaction.java:60)
at com.pingcap.tikv.catalog.CatalogTransaction.getTables(CatalogTransaction.java:98)
at com.pingcap.tikv.catalog.Catalog$CatalogCache.loadTables(Catalog.java:243)
at com.pingcap.tikv.catalog.Catalog$CatalogCache.getTable(Catalog.java:233)
at com.pingcap.tikv.catalog.Catalog.getTableFromCache(Catalog.java:126)
at com.pingcap.tispark.statistics.StatisticsManager$.initialize(StatisticsManager.scala:287)
at com.pingcap.tispark.statistics.StatisticsManager$.initStatisticsManager(StatisticsManager.scala:273)
at org.apache.spark.sql.TiContext.<init>(TiContext.scala:65)
at org.apache.spark.sql.TiExtensions.getOrCreateTiContext(TiExtensions.scala:55)
at org.apache.spark.sql.TiExtensions.$anonfun$apply$2(TiExtensions.scala:42)
at org.apache.spark.sql.catalyst.rule.TiAuthorizationRule.<init>(TiAuthorizationRule.scala:38)
at org.apache.spark.sql.catalyst.rule.TiAuthRuleFactory.apply(TiAuthRuleFactory.scala:40)
at org.apache.spark.sql.catalyst.rule.TiAuthRuleFactory.apply(TiAuthRuleFactory.scala:26)
at org.apache.spark.sql.SparkSessionExtensions.$anonfun$buildResolutionRules$1(SparkSessionExtensions.scala:174)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at org.apache.spark.sql.SparkSessionExtensions.buildResolutionRules(SparkSessionExtensions.scala:174)
at org.apache.spark.sql.internal.BaseSessionStateBuilder.customResolutionRules(BaseSessionStateBuilder.scala:217)
at org.apache.spark.sql.internal.BaseSessionStateBuilder$$anon$1.<init>(BaseSessionStateBuilder.scala:191)
at org.apache.spark.sql.internal.BaseSessionStateBuilder.analyzer(BaseSessionStateBuilder.scala:183)
at org.apache.spark.sql.internal.BaseSessionStateBuilder.$anonfun$build$2(BaseSessionStateBuilder.scala:363)
at org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:87)
at org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:87)
at org.apache.spark.sql.internal.SessionState.catalogManager(SessionState.scala:97)
at org.apache.spark.sql.catalyst.parser.TiParser.getOrElseInitTiCatalog(TiParser.scala:61)
at org.apache.spark.sql.catalyst.parser.TiParser.updateMetaCatalog(TiParser.scala:66)
at org.apache.spark.sql.catalyst.parser.TiParser.parsePlan(TiParser.scala:35)
at org.apache.spark.sql.SparkSession.$anonfun$sql$2(SparkSession.scala:620)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:620)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:617)
at com.superior.datatunnel.examples.DataTunnelTidb2LogDemo.main(DataTunnelTidb2LogDemo.java:29)

Caused by: com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "AuthPlugin" (class com.pingcap.tikv.meta.TiUserIdentity), not marked as ignorable (5 known properties: "Hostname", "CurrentUser", "Username", "AuthHostname", "AuthUsername"]) at [Source: (String)"{"id":70,"name":{"O":"tidb_mdl_view","L":"tidb_mdl_view"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"job_id","L":"job_id"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":0,"Flag":0,"Flen":0,"Decimal":0,"Charset":"","Collate":"","Elems":null,"ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":fals"[truncated 4118 chars]; line: 1, column: 3740] (through reference chain: com.pingcap.tikv.meta.TiTableInfo["view"]->com.pingcap.tikv.meta.TiViewInfo["view_definer"]->com.pingcap.tikv.meta.TiUserIdentity["AuthPlugin"]) at com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:61) at com.fasterxml.jackson.databind.DeserializationContext.handleUnknownProperty(DeserializationContext.java:1127) at com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:2036) at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1700) at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownVanilla(BeanDeserializerBase.java:1678) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:284) at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:463) at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1405) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:352) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:185) at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:542) at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping(BeanDeserializer.java:564) at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:439) at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1405) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:352) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:185) at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:542) at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping(BeanDeserializer.java:564) at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:439) at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1405) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:352) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:185) at com.fasterxml.jackson.databind.deser.DefaultDeserializationContext.readRootValue(DefaultDeserializationContext.java:323) at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4674) at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629) at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597) at com.pingcap.tikv.catalog.CatalogTransaction.parseFromJson(CatalogTransaction.java:55) ... 37 more 2023-04-20T18:09:34,151 INFO [main] o.a.s.s.c.c.TiCatalog: Initialize TiCatalog with name: tidb_catalog, pd address: 172.18.5.45:2379 2023-04-20T18:09:34,155 INFO [main] c.p.t.u.TiUtil$: spark.tispark.stale_read =

[xuanyu66]

Do you use the VIEW? TiSpark doesn't support VIEW.

[melin]

读取了tidb 数据库中 tidb_mdl_view 视图，用的是root账号，tidb 注册catalog的时候，没有办法过滤数据库。

这个视图，在代码中没有使用到：tidb_mdl_view

[melin]

如果不支持，是否自动过滤掉？

[shiyuhang0]

TiSpark 3.0.3 / 3.1.3 and TiSpark master support lazy load table, which means TiSpark will not load tale if it is no be used. Use the config spark.tispark.load_tables to control it. https://github.com/pingcap/tispark/blob/master/docs/userguide_3.0.md#configuration

[melin]

TiSpark 3.0.3 / 3.1.3 and TiSpark master support lazy load table, which means TiSpark will not load tale if it is no be used. Use the config spark.tispark.load_tables to control it. https://github.com/pingcap/tispark/blob/master/docs/userguide_3.0.md#configuration

这是一种办法。如果加载缓存，可以考虑过滤view，避免出现这种错误。

[xuanyu66]

@melin fixed in https://github.com/pingcap/tispark/pull/2690