search

pingcap / tispark

TiSpark is built for running Apache Spark on top of TiDB/TiKV
Apache License 2.0
883 stars 244 forks source link

cache df from tispark read tidb error #2771

Closed xufengnian2022 closed 7 months ago

xufengnian2022 commented 8 months ago

General Question

tidb version：v5.0.3 tispark version : 3.3_2.12-3.1.5 operate： val df = tispark read tidb df.cache()

java.lang.ArrayIndexOutOfBoundsException: 3 at java.util.ArrayList.clear(ArrayList.java:562) at com.pingcap.tikv.meta.TiDAGRequest.clearPushDownInfo(TiDAGRequest.java:861) at com.pingcap.tikv.meta.TiDAGRequest.stringTableRangeScan(TiDAGRequest.java:1039) at com.pingcap.tikv.meta.TiDAGRequest.toStringInternal(TiDAGRequest.java:1004) at com.pingcap.tikv.meta.TiDAGRequest.toString(TiDAGRequest.java:970) at java.lang.String.valueOf(String.java:2994) at java.lang.StringBuilder.append(StringBuilder.java:131) at org.apache.spark.sql.execution.LeafColumnarExecRDD.verboseString(CoprocessorRDD.scala:61) at org.apache.spark.sql.execution.LeafColumnarExecRDD.verboseString$(CoprocessorRDD.scala:50) at org.apache.spark.sql.execution.ColumnarCoprocessorRDD.verboseString(CoprocessorRDD.scala:71) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1049) at org.apache.spark.sql.execution.InputAdapter.generateTreeString(WholeStageCodegenExec.scala:554) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.execution.WholeStageCodegenExec.generateTreeString(WholeStageCodegenExec.scala:828) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1067) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1067) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.generateTreeString(AdaptiveSparkPlanExec.scala:413) at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:970) at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:960) at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:952) at org.apache.spark.sql.catalyst.trees.TreeNode.toString(TreeNode.scala:949) at org.apache.spark.sql.execution.columnar.CachedRDDBuilder.$anonfun$cachedName$2(InMemoryRelation.scala:216) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.sql.execution.columnar.CachedRDDBuilder.(InMemoryRelation.scala:216) at org.apache.spark.sql.execution.columnar.InMemoryRelation$.apply(InMemoryRelation.scala:328) at org.apache.spark.sql.execution.CacheManager.$anonfun$cacheQuery$2(CacheManager.scala:124) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) at org.apache.spark.sql.execution.CacheManager.cacheQuery(CacheManager.scala:119) at org.apache.spark.sql.execution.CacheManager.cacheQuery(CacheManager.scala:92) at org.apache.spark.sql.Dataset.persist(Dataset.scala:3360) at org.apache.spark.sql.Dataset.cache(Dataset.scala:3370)

xufengnian2022 commented 8 months ago

java.lang.ArrayIndexOutOfBoundsException at java.lang.System.arraycopy(Native Method) at java.util.ArrayList.addAll(ArrayList.java:584) at com.pingcap.tikv.meta.TiDAGRequest.addPushDownFilters(TiDAGRequest.java:820) at com.pingcap.tikv.meta.TiDAGRequest.pushDownFilters(TiDAGRequest.java:447) at com.pingcap.tikv.meta.TiDAGRequest.addPushDownExecutorToRequest(TiDAGRequest.java:389) at com.pingcap.tikv.meta.TiDAGRequest.buildDAGGetTableData(TiDAGRequest.java:353) at com.pingcap.tikv.meta.TiDAGRequest.stringTableRangeScan(TiDAGRequest.java:1040) at com.pingcap.tikv.meta.TiDAGRequest.toStringInternal(TiDAGRequest.java:1004) at com.pingcap.tikv.meta.TiDAGRequest.toString(TiDAGRequest.java:970) at java.lang.String.valueOf(String.java:2994) at java.lang.StringBuilder.append(StringBuilder.java:131) at org.apache.spark.sql.execution.LeafColumnarExecRDD.verboseString(CoprocessorRDD.scala:61) at org.apache.spark.sql.execution.LeafColumnarExecRDD.verboseString$(CoprocessorRDD.scala:50) at org.apache.spark.sql.execution.ColumnarCoprocessorRDD.verboseString(CoprocessorRDD.scala:71) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1049) at org.apache.spark.sql.execution.InputAdapter.generateTreeString(WholeStageCodegenExec.scala:554) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.execution.WholeStageCodegenExec.generateTreeString(WholeStageCodegenExec.scala:828) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1067) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1067) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:1077) at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.generateTreeString(AdaptiveSparkPlanExec.scala:413) at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:970) at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:960) at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:952) at org.apache.spark.sql.catalyst.trees.TreeNode.toString(TreeNode.scala:949) at org.apache.spark.sql.execution.columnar.CachedRDDBuilder.$anonfun$cachedName$2(InMemoryRelation.scala:216) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.sql.execution.columnar.CachedRDDBuilder.(InMemoryRelation.scala:216) at org.apache.spark.sql.execution.columnar.InMemoryRelation$.apply(InMemoryRelation.scala:328) at org.apache.spark.sql.execution.CacheManager.$anonfun$cacheQuery$2(CacheManager.scala:124) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) at org.apache.spark.sql.execution.CacheManager.cacheQuery(CacheManager.scala:119) at org.apache.spark.sql.execution.CacheManager.cacheQuery(CacheManager.scala:92) at org.apache.spark.sql.Dataset.persist(Dataset.scala:3360) at org.apache.spark.sql.Dataset.cache(Dataset.scala:3370)

github-actions[bot] commented 7 months ago

This issue is stale because it has been open for 30 days with no activity.

github-actions[bot] commented 7 months ago

This issue was closed because it has been inactive for 14 days since being marked as stale.