search

deephaven / deephaven-core

Deephaven Community Core
Other
257 stars 80 forks source link

Renaming a partitioning column on a `PartitionAwareSourceTable` causes permanent `lastBy` failures #6359

Closed nbauernfeind closed 1 week ago

nbauernfeind commented 1 week ago 
import io.deephaven.parquet.table.ParquetTools

def partition = emptyTable(10).update("II = ii")
ParquetTools.writeTable(partition, "/tmp/test/A=1/data.parquet")
ParquetTools.writeTable(partition, "/tmp/test/A=2/data.parquet")

table = ParquetTools.readTable("/tmp/test").renameColumns("B = A").lastBy("B")

Note that neither a coalesce() nor an update/updateView work. You can explicitly select("B", "II").lastBy("B"), but this isn't particularly useful.

The error:

Error running script: io.deephaven.engine.table.impl.NoSuchColumnException: Unknown column names [B], available column names are [A, dh_row_set]
    at io.deephaven.engine.table.impl.select.SourceColumn.initDef(SourceColumn.java:67)
    at io.deephaven.engine.table.impl.select.SelectColumn.initDef(SelectColumn.java:126)
    at io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer.createContext(SelectAndViewAnalyzer.java:128)
    at io.deephaven.engine.table.impl.QueryTable.lambda$selectOrUpdate$34(QueryTable.java:1578)
    at io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder.withNugget(QueryPerformanceRecorder.java:369)
    at io.deephaven.engine.table.impl.QueryTable.lambda$selectOrUpdate$35(QueryTable.java:1560)
    at io.deephaven.engine.table.impl.QueryTable$MemoizedResult.getOrCompute(QueryTable.java:3732)
    at io.deephaven.engine.table.impl.QueryTable.memoizeResult(QueryTable.java:3701)
    at io.deephaven.engine.table.impl.QueryTable$CopiedTable.lambda$memoizeResult$0(QueryTable.java:3641)
    at io.deephaven.engine.table.impl.QueryTable$MemoizedResult.getOrCompute(QueryTable.java:3732)
    at io.deephaven.engine.table.impl.QueryTable.memoizeResult(QueryTable.java:3701)
    at io.deephaven.engine.table.impl.QueryTable$CopiedTable.memoizeResult(QueryTable.java:3652)
    at io.deephaven.engine.table.impl.QueryTable$CopiedTable.lambda$memoizeResult$0(QueryTable.java:3641)
    at io.deephaven.engine.table.impl.QueryTable$MemoizedResult.getOrCompute(QueryTable.java:3732)
    at io.deephaven.engine.table.impl.QueryTable.memoizeResult(QueryTable.java:3701)
    at io.deephaven.engine.table.impl.QueryTable$CopiedTable.memoizeResult(QueryTable.java:3652)
    at io.deephaven.engine.table.impl.QueryTable.selectOrUpdate(QueryTable.java:1559)
    at io.deephaven.engine.table.impl.QueryTable.selectInternal(QueryTable.java:1527)
    at io.deephaven.engine.table.impl.QueryTable.select(QueryTable.java:1517)
    at io.deephaven.engine.table.impl.QueryTable$CopiedTable.select(QueryTable.java:3619)
    at io.deephaven.api.TableOperationsDefaults.select(TableOperationsDefaults.java:114)
    at io.deephaven.engine.table.impl.by.ChunkedOperatorAggregationHelper.staticIndexedAggregation(ChunkedOperatorAggregationHelper.java:1632)
    at io.deephaven.engine.table.impl.by.ChunkedOperatorAggregationHelper.aggregation(ChunkedOperatorAggregationHelper.java:166)
    at io.deephaven.engine.table.impl.by.ChunkedOperatorAggregationHelper.lambda$aggregation$2(ChunkedOperatorAggregationHelper.java:130)
    at io.deephaven.engine.table.impl.BaseTable.initializeWithSnapshot(BaseTable.java:1293)
    at io.deephaven.engine.table.impl.by.ChunkedOperatorAggregationHelper.lambda$aggregation$3(ChunkedOperatorAggregationHelper.java:127)
    at io.deephaven.engine.liveness.LivenessScopeStack.computeEnclosed(LivenessScopeStack.java:179)
    at io.deephaven.engine.table.impl.by.ChunkedOperatorAggregationHelper.aggregation(ChunkedOperatorAggregationHelper.java:115)
    at io.deephaven.engine.table.impl.by.ChunkedOperatorAggregationHelper.aggregation(ChunkedOperatorAggregationHelper.java:70)
    at io.deephaven.engine.table.impl.QueryTable.lambda$aggNoMemo$18(QueryTable.java:861)
    at io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder.withNugget(QueryPerformanceRecorder.java:369)
    at io.deephaven.engine.table.impl.QueryTable.aggNoMemo(QueryTable.java:860)
    at io.deephaven.engine.table.impl.QueryTable.lambda$aggAllBy$16(QueryTable.java:789)
    at io.deephaven.engine.table.impl.QueryTable$MemoizedResult.getOrCompute(QueryTable.java:3732)
    at io.deephaven.engine.table.impl.QueryTable.memoizeResult(QueryTable.java:3701)
    at io.deephaven.engine.table.impl.QueryTable.aggAllBy(QueryTable.java:787)
    at io.deephaven.engine.table.impl.QueryTable.aggAllBy(QueryTable.java:98)
    at io.deephaven.engine.table.impl.UncoalescedTable.aggAllBy(UncoalescedTable.java:360)
    at io.deephaven.engine.table.impl.UncoalescedTable.aggAllBy(UncoalescedTable.java:43)
    at io.deephaven.api.TableOperationsDefaults.lastBy(TableOperationsDefaults.java:464)
    at io.deephaven.api.TableOperationsDefaults.lastBy(TableOperationsDefaults.java:458)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:564)
    at org.codehaus.groovy.runtime.callsite.PlainObjectMetaMethodSite.doInvoke(PlainObjectMetaMethodSite.java:48)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite$PojoCachedMethodSite.invoke(PojoMetaMethodSite.java:186)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite.call(PojoMetaMethodSite.java:51)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:47)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:125)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:139)
    at io.deephaven.dynamic.Script_12.run(Script_12.groovy:8)
    at groovy.lang.GroovyShell.evaluate(GroovyShell.java:427)
    at groovy.lang.GroovyShell.evaluate(GroovyShell.java:461)
    at groovy.lang.GroovyShell.evaluate(GroovyShell.java:436)
    at io.deephaven.engine.util.GroovyDeephavenSession.lambda$evaluate$0(GroovyDeephavenSession.java:352)
    at io.deephaven.util.locks.FunctionalLock.doLockedInterruptibly(FunctionalLock.java:51)
    at io.deephaven.engine.util.GroovyDeephavenSession.evaluate(GroovyDeephavenSession.java:352)
    at io.deephaven.engine.util.AbstractScriptSession.lambda$evaluateScript$0(AbstractScriptSession.java:165)
    at io.deephaven.engine.context.ExecutionContext.lambda$apply$0(ExecutionContext.java:196)
    at io.deephaven.engine.context.ExecutionContext.apply(ExecutionContext.java:207)
    at io.deephaven.engine.context.ExecutionContext.apply(ExecutionContext.java:195)
    at io.deephaven.engine.util.AbstractScriptSession.evaluateScript(AbstractScriptSession.java:165)
    at io.deephaven.engine.util.DelegatingScriptSession.evaluateScript(DelegatingScriptSession.java:72)
    at io.deephaven.engine.util.ScriptSession.evaluateScript(ScriptSession.java:75)
    at io.deephaven.server.console.ConsoleServiceGrpcImpl.lambda$executeCommand$5(ConsoleServiceGrpcImpl.java:192)
    at io.deephaven.server.session.SessionState$ExportObject.doExport(SessionState.java:993)
    at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
    at io.deephaven.server.runner.scheduler.SchedulerModule$ThreadFactory.lambda$newThread$0(SchedulerModule.java:100)
    at java.base/java.lang.Thread.run(Thread.java:832)