search

trinodb / trino

Official repository of Trino, the distributed SQL query engine for big data, formerly known as PrestoSQL (https://trino.io)
https://trino.io
Apache License 2.0
10.49k stars 3.02k forks source link

Failed to list information_schema.tables when metadata location is wrong #11764

Open ebyhr opened 2 years ago

ebyhr commented 2 years ago

TestSharedGlueMetastore.testReadInformationSchema keeps failing on master. MetadataManager.getRedirectionAwareTableHandle tries to get the table handle and fails due to missing metadata location. 

Error:  io.trino.plugin.iceberg.TestSharedGlueMetastore.testReadInformationSchema  Time elapsed: 2.317 s  <<< FAILURE!
java.lang.RuntimeException: Failed to open input stream for file: /tmp/TrinoTest12782318577807722697/iceberg_data/nation/metadata/00000-55f3cefd-7639-40a0-b33f-0c93fd0cb533.metadata.json
    at io.trino.testing.AbstractTestingTrinoClient.execute(AbstractTestingTrinoClient.java:122)
    at io.trino.testing.DistributedQueryRunner.execute(DistributedQueryRunner.java:527)
    at io.trino.sql.query.QueryAssertions$QueryAssert.newQueryAssert(QueryAssertions.java:257)
    at io.trino.sql.query.QueryAssertions.query(QueryAssertions.java:105)
    at io.trino.testing.AbstractTestQueryFramework.query(AbstractTestQueryFramework.java:238)
    at io.trino.testing.AbstractTestQueryFramework.query(AbstractTestQueryFramework.java:233)
    at io.trino.plugin.iceberg.TestSharedGlueMetastore.testReadInformationSchema(TestSharedGlueMetastore.java:146)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:566)
    at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:104)
    at org.testng.internal.Invoker.invokeMethod(Invoker.java:645)
    at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:851)
    at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1177)
    at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:129)
    at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:112)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
    at java.base/java.lang.Thread.run(Thread.java:829)
    Suppressed: java.lang.Exception: SQL: SELECT table_schema FROM iceberg.information_schema.tables WHERE table_name = 'nation'
        at io.trino.testing.DistributedQueryRunner.execute(DistributedQueryRunner.java:530)
        ... 18 more
Caused by: org.apache.iceberg.exceptions.NotFoundException: Failed to open input stream for file: /tmp/TrinoTest12782318577807722697/iceberg_data/nation/metadata/00000-55f3cefd-7639-40a0-b33f-0c93fd0cb533.metadata.json
    at org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:177)
    at io.trino.plugin.hive.authentication.NoHdfsAuthentication.doAs(NoHdfsAuthentication.java:25)
    at io.trino.plugin.hive.HdfsEnvironment.doAs(HdfsEnvironment.java:97)
    at io.trino.plugin.iceberg.HdfsInputFile.newStream(HdfsInputFile.java:62)
    at org.apache.iceberg.TableMetadataParser.read(TableMetadataParser.java:250)
    at io.trino.plugin.iceberg.catalog.AbstractIcebergTableOperations.lambda$refreshFromMetadataLocation$1(AbstractIcebergTableOperations.java:218)
    at org.apache.iceberg.util.Tasks$Builder.runTaskWithRetry(Tasks.java:404)
    at org.apache.iceberg.util.Tasks$Builder.runSingleThreaded(Tasks.java:214)
    at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:198)
    at org.apache.iceberg.util.Tasks$Builder.run(Tasks.java:190)
    at io.trino.plugin.iceberg.catalog.AbstractIcebergTableOperations.refreshFromMetadataLocation(AbstractIcebergTableOperations.java:217)
    at io.trino.plugin.iceberg.catalog.AbstractIcebergTableOperations.refresh(AbstractIcebergTableOperations.java:122)
    at io.trino.plugin.iceberg.catalog.AbstractIcebergTableOperations.current(AbstractIcebergTableOperations.java:110)
    at io.trino.plugin.iceberg.catalog.glue.TrinoGlueCatalog.lambda$loadTable$5(TrinoGlueCatalog.java:267)
    at java.base/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705)
    at io.trino.plugin.iceberg.catalog.glue.TrinoGlueCatalog.loadTable(TrinoGlueCatalog.java:257)
    at io.trino.plugin.iceberg.IcebergMetadata.getTableHandle(IcebergMetadata.java:212)
    at io.trino.plugin.iceberg.IcebergMetadata.getTableHandle(IcebergMetadata.java:158)
    at io.trino.plugin.base.classloader.ClassLoaderSafeConnectorMetadata.getTableHandle(ClassLoaderSafeConnectorMetadata.java:182)
    at io.trino.metadata.MetadataManager.lambda$getTableHandle$5(MetadataManager.java:279)
    at java.base/java.util.Optional.flatMap(Optional.java:294)
    at io.trino.metadata.MetadataManager.getTableHandle(MetadataManager.java:259)
    at io.trino.metadata.MetadataManager.getRedirectionAwareTableHandle(MetadataManager.java:1484)
    at io.trino.metadata.MetadataManager.getRedirectionAwareTableHandle(MetadataManager.java:1476)
    at io.trino.metadata.MetadataManager.isExistingRelationForListing(MetadataManager.java:544)
    at io.trino.metadata.MetadataManager.listTables(MetadataManager.java:510)
    at io.trino.metadata.MetadataListing.listTables(MetadataListing.java:112)
    at io.trino.connector.informationschema.InformationSchemaPageSource.addTablesRecords(InformationSchemaPageSource.java:281)
    at io.trino.connector.informationschema.InformationSchemaPageSource.buildPages(InformationSchemaPageSource.java:219)
    at io.trino.connector.informationschema.InformationSchemaPageSource.getNextPage(InformationSchemaPageSource.java:183)
    at io.trino.operator.ScanFilterAndProjectOperator$ConnectorPageSourceToPages.process(ScanFilterAndProjectOperator.java:392)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:240)
    at io.trino.operator.WorkProcessorUtils$YieldingProcess.process(WorkProcessorUtils.java:182)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:325)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:325)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:325)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:325)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils.lambda$flatten$7(WorkProcessorUtils.java:296)
    at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:338)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:325)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:240)
    at io.trino.operator.WorkProcessorUtils.lambda$processStateMonitor$3(WorkProcessorUtils.java:219)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:240)
    at io.trino.operator.WorkProcessorUtils.lambda$finishWhen$4(WorkProcessorUtils.java:234)
    at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:391)
    at io.trino.operator.WorkProcessorSourceOperatorAdapter.getOutput(WorkProcessorSourceOperatorAdapter.java:150)
    at io.trino.operator.Driver.processInternal(Driver.java:388)
    at io.trino.operator.Driver.lambda$processFor$9(Driver.java:292)
    at io.trino.operator.Driver.tryWithLock(Driver.java:693)
    at io.trino.operator.Driver.processFor(Driver.java:285)
    at io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:[1092](https://github.com/trinodb/trino/runs/5801261030?check_suite_focus=true#step:13:1092))
    at io.trino.execution.executor.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:163)
    at io.trino.execution.executor.TaskExecutor$TaskRunner.run(TaskExecutor.java:488)
    at io.trino.$gen.Trino_testversion____20220402_190409_6.run(Unknown Source)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:[1128](https://github.com/trinodb/trino/runs/5801261030?check_suite_focus=true#step:13:1128))
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
    at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.io.FileNotFoundException: File /tmp/TrinoTest12782318577807722697/iceberg_data/nation/metadata/00000-55f3cefd-7639-40a0-b33f-0c93fd0cb533.metadata.json does not exist
    at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:666)
    at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:987)
    at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:656)
    at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:454)
    at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker.<init>(ChecksumFileSystem.java:146)
    at org.apache.hadoop.fs.ChecksumFileSystem.open(ChecksumFileSystem.java:347)
    at io.trino.plugin.hive.fs.TrinoFileSystemCache$FileSystemWrapper.open(TrinoFileSystemCache.java:307)
    at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:899)
    at org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:175)
    ... 65 more
findepi commented 2 years ago

cc @alexjo2144 @homar

findepi commented 2 years ago

This looks like a product, not test's, problem. @ebyhr can you please reformulate issue with this in bind?

ebyhr commented 2 years ago

Sure, I will look into and update the title and description.

findepi commented 2 years ago

Relates to https://github.com/trinodb/trino/issues/6551