search

UBOdin / mimir-caveats

3 stars 0 forks source link

Caveat failure on group-by attribute #13

Open okennedy opened 1 year ago

okennedy commented 1 year ago

Describe the bug Caveat failure on group-by aggregate dataset

Specifically note the difference in exprIds between the outer project and the first aggregate: idx#261443 in the former, idx#262323 in the latter.

+- !Project [grp#261406, idx#261443, (__MIMIR_ATTR_CAVEATTED__grp_261406#262188 OR __MIMIR_ALL_GROUPS_CONTAMINATED#262190) AS __MIMIR_ATTR_CAVEATTED__grp_261406#262193, (__MIMIR_ATTR_CAVEATTED__idx_261443#262189 OR __MIMIR_ALL_GROUPS_CONTAMINATED#262190) AS __MIMIR_ATTR_CAVEATTED__idx_261443#262194, __MIMIR_ROW_CAVEATTED#262187 AS __MIMIR_ROW_CAVEATTED#262192]
  +- Join Cross
     :- Aggregate [idx#262323], [min(idx#262319) AS grp#261406, idx#262323, ((sum(if (__MIMIR_ATTR_CAVEATTED__idx_261407#262171) 1 else 0) > cast(0 as bigint)) OR (sum(if ((__MIMIR_ATTR_CAVEATTED__idx_261443#262176 OR __MIMIR_ROW_CAVEATTED#262170)) 1 else 0) > cast(0 as bigint))) AS __MIMIR_ATTR_CAVEATTED__grp_261406#262188, ((sum(if (__MIMIR_ATTR_CAVEATTED__idx_261443#262176) 1 else 0) > cast(0 as bigint)) OR (sum(if (__MIMIR_ATTR_CAVEATTED__idx_261443#262176) 1 else 0) > cast(0 as bigint))) AS __MIMIR_ATTR_CAVEATTED__idx_261443#262189, NOT (sum(if ((NOT __MIMIR_ROW_CAVEATTED#262170 AND NOT __MIMIR_ATTR_CAVEATTED__idx_261443#262176)) 1 else 0) > cast(0 as bigint)) AS __MIMIR_ROW_CAVEATTED#262187]
     :  +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262159 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262171, __MIMIR_ATTR_CAVEATTED__file_261408#262160 AS __MIMIR_ATTR_CAVEATTED__file_261408#262172, __MIMIR_ATTR_CAVEATTED__lat_261409#262161 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262173, __MIMIR_ATTR_CAVEATTED__lon_261410#262162 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262174, __MIMIR_ATTR_CAVEATTED__geometry_261430#262163 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262175, __MIMIR_ATTR_CAVEATTED__idx_261443#262164 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262176, __MIMIR_ATTR_CAVEATTED__file_261444#262165 AS __MIMIR_ATTR_CAVEATTED__file_261444#262177, __MIMIR_ATTR_CAVEATTED__lat_261445#262166 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262178, __MIMIR_ATTR_CAVEATTED__lon_261446#262167 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262179, __MIMIR_ATTR_CAVEATTED__geometry_261461#262168 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262180, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262181, (__MIMIR_ROW_CAVEATTED#262158 OR (__MIMIR_ATTR_CAVEATTED__geometry_261430#262163 OR __MIMIR_ATTR_CAVEATTED__geometry_261461#262168)) AS __MIMIR_ROW_CAVEATTED#262170]
     :     +- Filter ( **org.apache.spark.sql.sedona_sql.expressions.ST_Distance$**   < cast(0.015 as double))
     :        +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262145 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262159, __MIMIR_ATTR_CAVEATTED__file_261408#262146 AS __MIMIR_ATTR_CAVEATTED__file_261408#262160, __MIMIR_ATTR_CAVEATTED__lat_261409#262147 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262161, __MIMIR_ATTR_CAVEATTED__lon_261410#262148 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262162, __MIMIR_ATTR_CAVEATTED__geometry_261430#262149 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262163, __MIMIR_ATTR_CAVEATTED__idx_261443#262151 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262164, __MIMIR_ATTR_CAVEATTED__file_261444#262152 AS __MIMIR_ATTR_CAVEATTED__file_261444#262165, __MIMIR_ATTR_CAVEATTED__lat_261445#262153 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262166, __MIMIR_ATTR_CAVEATTED__lon_261446#262154 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262167, __MIMIR_ATTR_CAVEATTED__geometry_261461#262155 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262168, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262157 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169, __MIMIR_ROW_CAVEATTED#262144 AS __MIMIR_ROW_CAVEATTED#262158]
     :           +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, RHS___MIMIR_ROWID#261860L, cast(hash(ifnull(LHS___MIMIR_ROWID#261859L, 1), ifnull(RHS___MIMIR_ROWID#261860L, 1), 42) as bigint) AS __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262132 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262145, __MIMIR_ATTR_CAVEATTED__file_261408#262133 AS __MIMIR_ATTR_CAVEATTED__file_261408#262146, __MIMIR_ATTR_CAVEATTED__lat_261409#262134 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262147, __MIMIR_ATTR_CAVEATTED__lon_261410#262135 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262148, __MIMIR_ATTR_CAVEATTED__geometry_261430#262136 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262149, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262150, __MIMIR_ATTR_CAVEATTED__idx_261443#262138 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262151, __MIMIR_ATTR_CAVEATTED__file_261444#262139 AS __MIMIR_ATTR_CAVEATTED__file_261444#262152, __MIMIR_ATTR_CAVEATTED__lat_261445#262140 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262153, __MIMIR_ATTR_CAVEATTED__lon_261446#262141 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262154, __MIMIR_ATTR_CAVEATTED__geometry_261461#262142 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262155, ... 3 more fields]
     :              +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261407#262086 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262132, __MIMIR_ATTR_CAVEATTED__file_261408#262087 AS __MIMIR_ATTR_CAVEATTED__file_261408#262133, __MIMIR_ATTR_CAVEATTED__lat_261409#262088 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262134, __MIMIR_ATTR_CAVEATTED__lon_261410#262089 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262135, __MIMIR_ATTR_CAVEATTED__geometry_261430#262090 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262136, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137, __MIMIR_ATTR_CAVEATTED__idx_261443#262125 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262138, __MIMIR_ATTR_CAVEATTED__file_261444#262126 AS __MIMIR_ATTR_CAVEATTED__file_261444#262139, __MIMIR_ATTR_CAVEATTED__lat_261445#262127 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262140, __MIMIR_ATTR_CAVEATTED__lon_261446#262128 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262141, __MIMIR_ATTR_CAVEATTED__geometry_261461#262129 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262142, __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262143, (__MIMIR_ROW_CAVEATTED#262085 OR __MIMIR_ROW_CAVEATTED#262124) AS __MIMIR_ROW_CAVEATTED#262131]
     :                 +- Join Inner
     :                    :- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, __MIMIR_ROWID#261601L AS LHS___MIMIR_ROWID#261859L, __MIMIR_ATTR_CAVEATTED__idx_261407#262079 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262086, __MIMIR_ATTR_CAVEATTED__file_261408#262080 AS __MIMIR_ATTR_CAVEATTED__file_261408#262087, __MIMIR_ATTR_CAVEATTED__lat_261409#262081 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262088, __MIMIR_ATTR_CAVEATTED__lon_261410#262082 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262089, __MIMIR_ATTR_CAVEATTED__geometry_261430#262083 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262090, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091, __MIMIR_ROW_CAVEATTED#262078 AS __MIMIR_ROW_CAVEATTED#262085]
     :                    :  +- Project [idx#262319, file#262320, lat#262321, lon#262322, CASE WHEN (isnull(lon#262322) OR isnull(lat#262321)) THEN cast(null as geometry) ELSE st_point(cast(lon#262322 as decimal(24,20)), cast(lat#262321 as decimal(24,20))) END AS geometry#261430, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262072 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262079, __MIMIR_ATTR_CAVEATTED__file_261408#262073 AS __MIMIR_ATTR_CAVEATTED__file_261408#262080, __MIMIR_ATTR_CAVEATTED__lat_261409#262074 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262081, __MIMIR_ATTR_CAVEATTED__lon_261410#262075 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262082, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND __MIMIR_ATTR_CAVEATTED__lat_261409#262074) OR (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND NOT isnull(lat#262321))) OR (__MIMIR_ATTR_CAVEATTED__lat_261409#262074 AND NOT isnull(lon#262322))) THEN true WHEN (isnull(lon#262322) OR isnull(lat#262321)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 OR __MIMIR_ATTR_CAVEATTED__lat_261409#262074) END AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262083, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084, __MIMIR_ROW_CAVEATTED#262071 AS __MIMIR_ROW_CAVEATTED#262078]
     :                    :     +- Project [idx#262319, file#262320, lat#262321, lon#262322, cast(null as geometry) AS geometry#261420, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262066 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262072, __MIMIR_ATTR_CAVEATTED__file_261408#262067 AS __MIMIR_ATTR_CAVEATTED__file_261408#262073, __MIMIR_ATTR_CAVEATTED__lat_261409#262068 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262074, __MIMIR_ATTR_CAVEATTED__lon_261410#262069 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262075, false AS __MIMIR_ATTR_CAVEATTED__geometry_261420#262076, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077, __MIMIR_ROW_CAVEATTED#262065 AS __MIMIR_ROW_CAVEATTED#262071]
     :                    :        +- Project [idx#262319, file#262320, lat#262321, lon#262322, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) as bigint)), idx#262319, file#262320, lat#262321, lon#262322, 42) as bigint) AS __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262059 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262066, __MIMIR_ATTR_CAVEATTED__file_261408#262060 AS __MIMIR_ATTR_CAVEATTED__file_261408#262067, __MIMIR_ATTR_CAVEATTED__lat_261409#262061 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262068, __MIMIR_ATTR_CAVEATTED__lon_261410#262062 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262069, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063))) OR __MIMIR_ATTR_CAVEATTED__idx_261407#262059) OR __MIMIR_ATTR_CAVEATTED__file_261408#262060) OR __MIMIR_ATTR_CAVEATTED__lat_261409#262061) OR __MIMIR_ATTR_CAVEATTED__lon_261410#262062) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070, __MIMIR_ROW_CAVEATTED#262058 AS __MIMIR_ROW_CAVEATTED#262065]
     :                    :           +- Project [idx#262319, file#262320, lat#262321, lon#262322, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261602L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261604L, __MIMIR_ATTR_CAVEATTED__idx_261407#262054 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262059, __MIMIR_ATTR_CAVEATTED__file_261408#262055 AS __MIMIR_ATTR_CAVEATTED__file_261408#262060, __MIMIR_ATTR_CAVEATTED__lat_261409#262056 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262061, __MIMIR_ATTR_CAVEATTED__lon_261410#262057 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262062, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064, __MIMIR_ROW_CAVEATTED#262053 AS __MIMIR_ROW_CAVEATTED#262058]
     :                    :              +- Project [idx#262319, file#262320, lat#262321, lon#262322, false AS __MIMIR_ATTR_CAVEATTED__idx_261407#262054, false AS __MIMIR_ATTR_CAVEATTED__file_261408#262055, false AS __MIMIR_ATTR_CAVEATTED__lat_261409#262056, false AS __MIMIR_ATTR_CAVEATTED__lon_261410#262057, false AS __MIMIR_ROW_CAVEATTED#262053]
     :                    :                 +- LocalRelation [idx#262319, file#262320, lat#262321, lon#262322]
     :                    +- Project [idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, __MIMIR_ROWID#261778L AS RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261443#262118 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262125, __MIMIR_ATTR_CAVEATTED__file_261444#262119 AS __MIMIR_ATTR_CAVEATTED__file_261444#262126, __MIMIR_ATTR_CAVEATTED__lat_261445#262120 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262127, __MIMIR_ATTR_CAVEATTED__lon_261446#262121 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262128, __MIMIR_ATTR_CAVEATTED__geometry_261461#262122 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262129, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130, __MIMIR_ROW_CAVEATTED#262117 AS __MIMIR_ROW_CAVEATTED#262124]
     :                       +- Project [idx#262323, file#262324, lat#262325, lon#262326, CASE WHEN (isnull(lon#262326) OR isnull(lat#262325)) THEN cast(null as geometry) ELSE st_point(cast(lon#262326 as decimal(24,20)), cast(lat#262325 as decimal(24,20))) END AS geometry#261461, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262111 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262118, __MIMIR_ATTR_CAVEATTED__file_261444#262112 AS __MIMIR_ATTR_CAVEATTED__file_261444#262119, __MIMIR_ATTR_CAVEATTED__lat_261445#262113 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262120, __MIMIR_ATTR_CAVEATTED__lon_261446#262114 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262121, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND __MIMIR_ATTR_CAVEATTED__lat_261445#262113) OR (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND NOT isnull(lat#262325))) OR (__MIMIR_ATTR_CAVEATTED__lat_261445#262113 AND NOT isnull(lon#262326))) THEN true WHEN (isnull(lon#262326) OR isnull(lat#262325)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 OR __MIMIR_ATTR_CAVEATTED__lat_261445#262113) END AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262122, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123, __MIMIR_ROW_CAVEATTED#262110 AS __MIMIR_ROW_CAVEATTED#262117]
     :                          +- Project [idx#262323, file#262324, lat#262325, lon#262326, cast(null as geometry) AS geometry#261453, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262105 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262111, __MIMIR_ATTR_CAVEATTED__file_261444#262106 AS __MIMIR_ATTR_CAVEATTED__file_261444#262112, __MIMIR_ATTR_CAVEATTED__lat_261445#262107 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262113, __MIMIR_ATTR_CAVEATTED__lon_261446#262108 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262114, false AS __MIMIR_ATTR_CAVEATTED__geometry_261453#262115, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116, __MIMIR_ROW_CAVEATTED#262104 AS __MIMIR_ROW_CAVEATTED#262110]
     :                             +- Project [idx#262323, file#262324, lat#262325, lon#262326, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) as bigint)), idx#262323, file#262324, lat#262325, lon#262326, 42) as bigint) AS __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262098 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262105, __MIMIR_ATTR_CAVEATTED__file_261444#262099 AS __MIMIR_ATTR_CAVEATTED__file_261444#262106, __MIMIR_ATTR_CAVEATTED__lat_261445#262100 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262107, __MIMIR_ATTR_CAVEATTED__lon_261446#262101 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262108, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102))) OR __MIMIR_ATTR_CAVEATTED__idx_261443#262098) OR __MIMIR_ATTR_CAVEATTED__file_261444#262099) OR __MIMIR_ATTR_CAVEATTED__lat_261445#262100) OR __MIMIR_ATTR_CAVEATTED__lon_261446#262101) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109, __MIMIR_ROW_CAVEATTED#262097 AS __MIMIR_ROW_CAVEATTED#262104]
     :                                +- Project [idx#262323, file#262324, lat#262325, lon#262326, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261779L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261781L, __MIMIR_ATTR_CAVEATTED__idx_261443#262093 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262098, __MIMIR_ATTR_CAVEATTED__file_261444#262094 AS __MIMIR_ATTR_CAVEATTED__file_261444#262099, __MIMIR_ATTR_CAVEATTED__lat_261445#262095 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262100, __MIMIR_ATTR_CAVEATTED__lon_261446#262096 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262101, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103, __MIMIR_ROW_CAVEATTED#262092 AS __MIMIR_ROW_CAVEATTED#262097]
     :                                   +- Project [idx#262323, file#262324, lat#262325, lon#262326, false AS __MIMIR_ATTR_CAVEATTED__idx_261443#262093, false AS __MIMIR_ATTR_CAVEATTED__file_261444#262094, false AS __MIMIR_ATTR_CAVEATTED__lat_261445#262095, false AS __MIMIR_ATTR_CAVEATTED__lon_261446#262096, false AS __MIMIR_ROW_CAVEATTED#262092]
     :                                      +- LocalRelation [idx#262323, file#262324, lat#262325, lon#262326]
     +- Aggregate [(sum(if (__MIMIR_ATTR_CAVEATTED__idx_261443#262176) 1 else 0) > cast(0 as bigint)) AS __MIMIR_ALL_GROUPS_CONTAMINATED#262190]
        +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262159 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262171, __MIMIR_ATTR_CAVEATTED__file_261408#262160 AS __MIMIR_ATTR_CAVEATTED__file_261408#262172, __MIMIR_ATTR_CAVEATTED__lat_261409#262161 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262173, __MIMIR_ATTR_CAVEATTED__lon_261410#262162 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262174, __MIMIR_ATTR_CAVEATTED__geometry_261430#262163 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262175, __MIMIR_ATTR_CAVEATTED__idx_261443#262164 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262176, __MIMIR_ATTR_CAVEATTED__file_261444#262165 AS __MIMIR_ATTR_CAVEATTED__file_261444#262177, __MIMIR_ATTR_CAVEATTED__lat_261445#262166 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262178, __MIMIR_ATTR_CAVEATTED__lon_261446#262167 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262179, __MIMIR_ATTR_CAVEATTED__geometry_261461#262168 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262180, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262181, (__MIMIR_ROW_CAVEATTED#262158 OR (__MIMIR_ATTR_CAVEATTED__geometry_261430#262163 OR __MIMIR_ATTR_CAVEATTED__geometry_261461#262168)) AS __MIMIR_ROW_CAVEATTED#262170]
           +- Filter ( **org.apache.spark.sql.sedona_sql.expressions.ST_Distance$**   < cast(0.015 as double))
              +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262145 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262159, __MIMIR_ATTR_CAVEATTED__file_261408#262146 AS __MIMIR_ATTR_CAVEATTED__file_261408#262160, __MIMIR_ATTR_CAVEATTED__lat_261409#262147 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262161, __MIMIR_ATTR_CAVEATTED__lon_261410#262148 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262162, __MIMIR_ATTR_CAVEATTED__geometry_261430#262149 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262163, __MIMIR_ATTR_CAVEATTED__idx_261443#262151 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262164, __MIMIR_ATTR_CAVEATTED__file_261444#262152 AS __MIMIR_ATTR_CAVEATTED__file_261444#262165, __MIMIR_ATTR_CAVEATTED__lat_261445#262153 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262166, __MIMIR_ATTR_CAVEATTED__lon_261446#262154 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262167, __MIMIR_ATTR_CAVEATTED__geometry_261461#262155 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262168, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262157 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169, __MIMIR_ROW_CAVEATTED#262144 AS __MIMIR_ROW_CAVEATTED#262158]
                 +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, RHS___MIMIR_ROWID#261860L, cast(hash(ifnull(LHS___MIMIR_ROWID#261859L, 1), ifnull(RHS___MIMIR_ROWID#261860L, 1), 42) as bigint) AS __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262132 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262145, __MIMIR_ATTR_CAVEATTED__file_261408#262133 AS __MIMIR_ATTR_CAVEATTED__file_261408#262146, __MIMIR_ATTR_CAVEATTED__lat_261409#262134 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262147, __MIMIR_ATTR_CAVEATTED__lon_261410#262135 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262148, __MIMIR_ATTR_CAVEATTED__geometry_261430#262136 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262149, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262150, __MIMIR_ATTR_CAVEATTED__idx_261443#262138 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262151, __MIMIR_ATTR_CAVEATTED__file_261444#262139 AS __MIMIR_ATTR_CAVEATTED__file_261444#262152, __MIMIR_ATTR_CAVEATTED__lat_261445#262140 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262153, __MIMIR_ATTR_CAVEATTED__lon_261446#262141 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262154, __MIMIR_ATTR_CAVEATTED__geometry_261461#262142 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262155, ... 3 more fields]
                    +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261407#262086 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262132, __MIMIR_ATTR_CAVEATTED__file_261408#262087 AS __MIMIR_ATTR_CAVEATTED__file_261408#262133, __MIMIR_ATTR_CAVEATTED__lat_261409#262088 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262134, __MIMIR_ATTR_CAVEATTED__lon_261410#262089 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262135, __MIMIR_ATTR_CAVEATTED__geometry_261430#262090 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262136, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137, __MIMIR_ATTR_CAVEATTED__idx_261443#262125 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262138, __MIMIR_ATTR_CAVEATTED__file_261444#262126 AS __MIMIR_ATTR_CAVEATTED__file_261444#262139, __MIMIR_ATTR_CAVEATTED__lat_261445#262127 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262140, __MIMIR_ATTR_CAVEATTED__lon_261446#262128 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262141, __MIMIR_ATTR_CAVEATTED__geometry_261461#262129 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262142, __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262143, (__MIMIR_ROW_CAVEATTED#262085 OR __MIMIR_ROW_CAVEATTED#262124) AS __MIMIR_ROW_CAVEATTED#262131]
                       +- Join Inner
                          :- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, __MIMIR_ROWID#261601L AS LHS___MIMIR_ROWID#261859L, __MIMIR_ATTR_CAVEATTED__idx_261407#262079 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262086, __MIMIR_ATTR_CAVEATTED__file_261408#262080 AS __MIMIR_ATTR_CAVEATTED__file_261408#262087, __MIMIR_ATTR_CAVEATTED__lat_261409#262081 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262088, __MIMIR_ATTR_CAVEATTED__lon_261410#262082 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262089, __MIMIR_ATTR_CAVEATTED__geometry_261430#262083 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262090, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091, __MIMIR_ROW_CAVEATTED#262078 AS __MIMIR_ROW_CAVEATTED#262085]
                          :  +- Project [idx#262327, file#262328, lat#262329, lon#262330, CASE WHEN (isnull(lon#262330) OR isnull(lat#262329)) THEN cast(null as geometry) ELSE st_point(cast(lon#262330 as decimal(24,20)), cast(lat#262329 as decimal(24,20))) END AS geometry#261430, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262072 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262079, __MIMIR_ATTR_CAVEATTED__file_261408#262073 AS __MIMIR_ATTR_CAVEATTED__file_261408#262080, __MIMIR_ATTR_CAVEATTED__lat_261409#262074 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262081, __MIMIR_ATTR_CAVEATTED__lon_261410#262075 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262082, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND __MIMIR_ATTR_CAVEATTED__lat_261409#262074) OR (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND NOT isnull(lat#262329))) OR (__MIMIR_ATTR_CAVEATTED__lat_261409#262074 AND NOT isnull(lon#262330))) THEN true WHEN (isnull(lon#262330) OR isnull(lat#262329)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 OR __MIMIR_ATTR_CAVEATTED__lat_261409#262074) END AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262083, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084, __MIMIR_ROW_CAVEATTED#262071 AS __MIMIR_ROW_CAVEATTED#262078]
                          :     +- Project [idx#262327, file#262328, lat#262329, lon#262330, cast(null as geometry) AS geometry#261420, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262066 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262072, __MIMIR_ATTR_CAVEATTED__file_261408#262067 AS __MIMIR_ATTR_CAVEATTED__file_261408#262073, __MIMIR_ATTR_CAVEATTED__lat_261409#262068 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262074, __MIMIR_ATTR_CAVEATTED__lon_261410#262069 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262075, false AS __MIMIR_ATTR_CAVEATTED__geometry_261420#262076, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077, __MIMIR_ROW_CAVEATTED#262065 AS __MIMIR_ROW_CAVEATTED#262071]
                          :        +- Project [idx#262327, file#262328, lat#262329, lon#262330, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) as bigint)), idx#262327, file#262328, lat#262329, lon#262330, 42) as bigint) AS __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262059 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262066, __MIMIR_ATTR_CAVEATTED__file_261408#262060 AS __MIMIR_ATTR_CAVEATTED__file_261408#262067, __MIMIR_ATTR_CAVEATTED__lat_261409#262061 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262068, __MIMIR_ATTR_CAVEATTED__lon_261410#262062 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262069, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063))) OR __MIMIR_ATTR_CAVEATTED__idx_261407#262059) OR __MIMIR_ATTR_CAVEATTED__file_261408#262060) OR __MIMIR_ATTR_CAVEATTED__lat_261409#262061) OR __MIMIR_ATTR_CAVEATTED__lon_261410#262062) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070, __MIMIR_ROW_CAVEATTED#262058 AS __MIMIR_ROW_CAVEATTED#262065]
                          :           +- Project [idx#262327, file#262328, lat#262329, lon#262330, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261602L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261604L, __MIMIR_ATTR_CAVEATTED__idx_261407#262054 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262059, __MIMIR_ATTR_CAVEATTED__file_261408#262055 AS __MIMIR_ATTR_CAVEATTED__file_261408#262060, __MIMIR_ATTR_CAVEATTED__lat_261409#262056 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262061, __MIMIR_ATTR_CAVEATTED__lon_261410#262057 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262062, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064, __MIMIR_ROW_CAVEATTED#262053 AS __MIMIR_ROW_CAVEATTED#262058]
                          :              +- Project [idx#262327, file#262328, lat#262329, lon#262330, false AS __MIMIR_ATTR_CAVEATTED__idx_261407#262054, false AS __MIMIR_ATTR_CAVEATTED__file_261408#262055, false AS __MIMIR_ATTR_CAVEATTED__lat_261409#262056, false AS __MIMIR_ATTR_CAVEATTED__lon_261410#262057, false AS __MIMIR_ROW_CAVEATTED#262053]
                          :                 +- LocalRelation [idx#262327, file#262328, lat#262329, lon#262330]
                          +- Project [idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, __MIMIR_ROWID#261778L AS RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261443#262118 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262125, __MIMIR_ATTR_CAVEATTED__file_261444#262119 AS __MIMIR_ATTR_CAVEATTED__file_261444#262126, __MIMIR_ATTR_CAVEATTED__lat_261445#262120 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262127, __MIMIR_ATTR_CAVEATTED__lon_261446#262121 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262128, __MIMIR_ATTR_CAVEATTED__geometry_261461#262122 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262129, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130, __MIMIR_ROW_CAVEATTED#262117 AS __MIMIR_ROW_CAVEATTED#262124]
                             +- Project [idx#262331, file#262332, lat#262333, lon#262334, CASE WHEN (isnull(lon#262334) OR isnull(lat#262333)) THEN cast(null as geometry) ELSE st_point(cast(lon#262334 as decimal(24,20)), cast(lat#262333 as decimal(24,20))) END AS geometry#261461, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262111 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262118, __MIMIR_ATTR_CAVEATTED__file_261444#262112 AS __MIMIR_ATTR_CAVEATTED__file_261444#262119, __MIMIR_ATTR_CAVEATTED__lat_261445#262113 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262120, __MIMIR_ATTR_CAVEATTED__lon_261446#262114 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262121, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND __MIMIR_ATTR_CAVEATTED__lat_261445#262113) OR (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND NOT isnull(lat#262333))) OR (__MIMIR_ATTR_CAVEATTED__lat_261445#262113 AND NOT isnull(lon#262334))) THEN true WHEN (isnull(lon#262334) OR isnull(lat#262333)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 OR __MIMIR_ATTR_CAVEATTED__lat_261445#262113) END AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262122, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123, __MIMIR_ROW_CAVEATTED#262110 AS __MIMIR_ROW_CAVEATTED#262117]
                                +- Project [idx#262331, file#262332, lat#262333, lon#262334, cast(null as geometry) AS geometry#261453, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262105 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262111, __MIMIR_ATTR_CAVEATTED__file_261444#262106 AS __MIMIR_ATTR_CAVEATTED__file_261444#262112, __MIMIR_ATTR_CAVEATTED__lat_261445#262107 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262113, __MIMIR_ATTR_CAVEATTED__lon_261446#262108 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262114, false AS __MIMIR_ATTR_CAVEATTED__geometry_261453#262115, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116, __MIMIR_ROW_CAVEATTED#262104 AS __MIMIR_ROW_CAVEATTED#262110]
                                   +- Project [idx#262331, file#262332, lat#262333, lon#262334, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) as bigint)), idx#262331, file#262332, lat#262333, lon#262334, 42) as bigint) AS __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262098 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262105, __MIMIR_ATTR_CAVEATTED__file_261444#262099 AS __MIMIR_ATTR_CAVEATTED__file_261444#262106, __MIMIR_ATTR_CAVEATTED__lat_261445#262100 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262107, __MIMIR_ATTR_CAVEATTED__lon_261446#262101 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262108, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102))) OR __MIMIR_ATTR_CAVEATTED__idx_261443#262098) OR __MIMIR_ATTR_CAVEATTED__file_261444#262099) OR __MIMIR_ATTR_CAVEATTED__lat_261445#262100) OR __MIMIR_ATTR_CAVEATTED__lon_261446#262101) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109, __MIMIR_ROW_CAVEATTED#262097 AS __MIMIR_ROW_CAVEATTED#262104]
                                      +- Project [idx#262331, file#262332, lat#262333, lon#262334, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261779L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261781L, __MIMIR_ATTR_CAVEATTED__idx_261443#262093 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262098, __MIMIR_ATTR_CAVEATTED__file_261444#262094 AS __MIMIR_ATTR_CAVEATTED__file_261444#262099, __MIMIR_ATTR_CAVEATTED__lat_261445#262095 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262100, __MIMIR_ATTR_CAVEATTED__lon_261446#262096 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262101, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103, __MIMIR_ROW_CAVEATTED#262092 AS __MIMIR_ROW_CAVEATTED#262097]
                                         +- Project [idx#262331, file#262332, lat#262333, lon#262334, false AS __MIMIR_ATTR_CAVEATTED__idx_261443#262093, false AS __MIMIR_ATTR_CAVEATTED__file_261444#262094, false AS __MIMIR_ATTR_CAVEATTED__lat_261445#262095, false AS __MIMIR_ATTR_CAVEATTED__lon_261446#262096, false AS __MIMIR_ROW_CAVEATTED#262092]
                                            +- LocalRelation [idx#262331, file#262332, lat#262333, lon#262334]
okennedy commented 1 year ago

I think the problem is at: https://github.com/UBOdin/mimir-caveats/blob/master/src/main/scala/org/mimirdb/caveats/annotate/CaveatExistsInPlan.scala#L465

Specifically, annotate is getting a plan that references a different set of attributes... I'm reasonably certain that it should be taking ret and not plan, but making that change breaks unit tests, so I need to dig further.