liufuyang
commented
6 years ago I removed the old data, start again, now TiDB can starts. Tried to add data by using the data file inside ti-spark-master container, success. Then trying to connect with spark, error happens:
docker-compose exec tispark-master /opt/spark/bin/spark-shell
scala> import org.apache.spark.sql.TiContext
scala> val ti = new TiContext(spark)
scala> ti.tidbMapDatabase("TPCH_001")
com.pingcap.tikv.exception.TiClientInternalException: Invalid JSON value for Type TiTableInfo: {"id":11,"name":{"O":"columns_priv","L":"columns_priv"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"Host","L":"host"},"offset":0,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":60,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":2,"name":{"O":"DB","L":"db"},"offset":1,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":3,"name":{"O":"User","L":"user"},"offset":2,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":32,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":4,"name":{"O":"Table_name","L":"table_name"},"offset":3,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":5,"name":{"O":"Column_name","L":"column_name"},"offset":4,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":6,"name":{"O":"Timestamp","L":"timestamp"},"offset":5,"origin_default":null,"default":"CURRENT_TIMESTAMP","generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":7,"Flag":128,"Flen":19,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":""},{"id":7,"name":{"O":"Column_priv","L":"column_priv"},"offset":6,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":248,"Flag":0,"Flen":-1,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":["Select","Insert","Update"]},"state":5,"comment":""}],"index_info":[{"id":1,"idx_name":{"O":"PRIMARY","L":"primary"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"Host","L":"host"},"offset":0,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":60,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"DB","L":"db"},"offset":1,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"User","L":"user"},"offset":2,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":32,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"Table_name","L":"table_name"},"offset":3,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"Column_name","L":"column_name"},"offset":4,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}}],"is_unique":true,"is_primary":true,"state":5,"comment":"","index_type":1}],"fk_info":null,"state":5,"pk_is_handle":false,"comment":"","auto_inc_id":0,"max_col_id":7,"max_idx_id":1,"update_timestamp":401624161340358664,"ShardRowIDBits":0,"partition":null}
at com.pingcap.tikv.catalog.CatalogTransaction.parseFromJson(CatalogTransaction.java:178)
at com.pingcap.tikv.catalog.CatalogTransaction.getTables(CatalogTransaction.java:160)
at com.pingcap.tikv.catalog.Catalog$CatalogCache.loadTables(Catalog.java:94)
at com.pingcap.tikv.catalog.Catalog$CatalogCache.getTable(Catalog.java:88)
at com.pingcap.tikv.catalog.Catalog.getTable(Catalog.java:168)
at com.pingcap.tikv.catalog.Catalog.getTable(Catalog.java:162)
at com.pingcap.tispark.statistics.StatisticsManager.metaTable$lzycompute(StatisticsManager.scala:90)
at com.pingcap.tispark.statistics.StatisticsManager.metaTable(StatisticsManager.scala:90)
at com.pingcap.tispark.statistics.StatisticsHelper$.isManagerReady(StatisticsHelper.scala:62)
at com.pingcap.tispark.statistics.StatisticsManager.loadStatisticsInfo(StatisticsManager.scala:107)
at org.apache.spark.sql.TiContext$$anonfun$tidbMapDatabase$1$$anonfun$apply$3.apply(TiContext.scala:152)
at org.apache.spark.sql.TiContext$$anonfun$tidbMapDatabase$1$$anonfun$apply$3.apply(TiContext.scala:148)
at scala.collection.immutable.List.foreach(List.scala:381)
at org.apache.spark.sql.TiContext$$anonfun$tidbMapDatabase$1.apply(TiContext.scala:148)
at org.apache.spark.sql.TiContext$$anonfun$tidbMapDatabase$1.apply(TiContext.scala:147)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.sql.TiContext.tidbMapDatabase(TiContext.scala:147)
... 48 elided
Caused by: com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "Tp" (class com.pingcap.tikv.meta.TiIndexColumn), not marked as ignorable (3 known properties: "name", "length", "offset"])
at [Source: {"id":11,"name":{"O":"columns_priv","L":"columns_priv"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"Host","L":"host"},"offset":0,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":60,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":2,"name":{"O":"DB","L":"db"},"offset":1,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":3,"name":{"O":"User","L":"user"},"offset":2,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":32,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":4,"name":{"O":"Table_name","L":"table_name"},"offset":3,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":5,"name":{"O":"Column_name","L":"column_name"},"offset":4,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5,"comment":""},{"id":6,"name":{"O":"Timestamp","L":"timestamp"},"offset":5,"origin_default":null,"default":"CURRENT_TIMESTAMP","generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":7,"Flag":128,"Flen":19,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":""},{"id":7,"name":{"O":"Column_priv","L":"column_priv"},"offset":6,"origin_default":null,"default":null,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":248,"Flag":0,"Flen":-1,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":["Select","Insert","Update"]},"state":5,"comment":""}],"index_info":[{"id":1,"idx_name":{"O":"PRIMARY","L":"primary"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"Host","L":"host"},"offset":0,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":60,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"DB","L":"db"},"offset":1,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"User","L":"user"},"offset":2,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":32,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"Table_name","L":"table_name"},"offset":3,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}},{"name":{"O":"Column_name","L":"column_name"},"offset":4,"length":-1,"Tp":{"Tp":254,"Flag":3,"Flen":64,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null}}],"is_unique":true,"is_primary":true,"state":5,"comment":"","index_type":1}],"fk_info":null,"state":5,"pk_is_handle":false,"comment":"","auto_inc_id":0,"max_col_id":7,"max_idx_id":1,"update_timestamp":401624161340358664,"ShardRowIDBits":0,"partition":null}; line: 1, column: 2314] (through reference chain: com.pingcap.tikv.meta.TiTableInfo["index_info"]->java.util.ArrayList[0]->com.pingcap.tikv.meta.TiIndexInfo["idx_cols"]->java.util.ArrayList[0]->com.pingcap.tikv.meta.TiIndexColumn["Tp"])
at com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:51)
at com.fasterxml.jackson.databind.DeserializationContext.reportUnknownProperty(DeserializationContext.java:839)
at com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:1045)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1352)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownVanilla(BeanDeserializerBase.java:1330)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:228)
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:402)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1099)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:296)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:133)
at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:245)
at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:217)
at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:25)
at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:520)
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping(BeanDeserializer.java:463)
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:378)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1099)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:296)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:133)
at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:245)
at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:217)
at com.fasterxml.jackson.databind.deser.std.CollectionDeserializer.deserialize(CollectionDeserializer.java:25)
at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:520)
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping(BeanDeserializer.java:463)
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:378)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1099)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:296)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:133)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:3736)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2726)
at com.pingcap.tikv.catalog.CatalogTransaction.parseFromJson(CatalogTransaction.java:173)
... 64 more
scala>Seems to be the same error mentioned at the end of https://github.com/pingcap/tidb-docker-compose/issues/28
I've got two comments here:
- by deleting the old data works now, but this may implicate that other users might encounter issues when they upgrade TiDB version later on.
- I wonder if we have some type of high level component tests after image build, which basically runs a script let what I manually did above, to check everything works after we build a certain image. I guess if we do it will prevent a faulty/non-compliant image to be pushed into the docker registry.
tennix
Current issue:
TiSpark's new docker image is not working.
Error:
when doing:
Old comments below, please ignore them for now
It used to work. But I tried to do
docker-compose pullearlier then start again, the tidb container failed.I haven't clean up the old data yet. If you want I can try clean up that and try again.
Current output of docker ps:
Current docker logs: