search

StarRocks / starrocks

The world's fastest open query engine for sub-second analytics both on and off the data lakehouse. With the flexibility to support nearly any scenario, StarRocks provides best-in-class performance for multi-dimensional analytics, real-time analytics, and ad-hoc queries. A Linux Foundation project.
https://starrocks.io
Apache License 2.0
9.01k stars 1.81k forks source link

Apache Iceberg rest catalog. Query problem #24569

Closed alberttwong closed 1 year ago

alberttwong commented 1 year ago

Using

drop catalog tabular;
create external catalog 'tabular'
PROPERTIES
(
    "aws.s3.use_instance_profile" = "false",
    "aws.s3.access_key" = "XXXX",
    "aws.s3.secret_key" = "YYYY",    
    "aws.s3.region" = "us-east-1",
    "uri" = "https://api.tabular.io/ws",
    "warehouse" = "test301",
    "type" = "iceberg",
    "iceberg.catalog.type" = "rest",
    "security" = "oauth2",
    "session" = "user",
    "credential" = "t-ZZZZZ"
);
show databases from tabular;
set catalog tabular;
select count(*) from `default`.movies;
2023-06-01 21:15:59,667 WARN (starrocks-mysql-nio-pool-39|1415) [StmtExecutor.execute():558] execute Exception, sql select count(*) from `default`.movies
org.apache.iceberg.exceptions.RuntimeIOException: Failed to open input stream for file: s3://celerdata-4864/a3076908-c737-4d42-b104-581b6b170254/e3730253-692e-474d-a473-4dfb23f537e1/metadata/snap-2346046352581096739-1-3eac4786-95da-483a-8f72-2565b17f78ee.avro
    at org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:187) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.avro.AvroIterable.newFileReader(AvroIterable.java:100) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:76) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:36) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.relocated.com.google.common.collect.Iterables.addAll(Iterables.java:337) ~[iceberg-bundled-guava-1.1.0.jar:?]
    at org.apache.iceberg.relocated.com.google.common.collect.Lists.newLinkedList(Lists.java:241) ~[iceberg-bundled-guava-1.1.0.jar:?]
    at org.apache.iceberg.ManifestLists.read(ManifestLists.java:45) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.BaseSnapshot.cacheManifests(BaseSnapshot.java:148) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.BaseSnapshot.dataManifests(BaseSnapshot.java:174) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.DataTableScan.doPlanFiles(DataTableScan.java:82) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.BaseTableScan.planFiles(BaseTableScan.java:152) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.DataTableScan.planFiles(DataTableScan.java:27) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.BaseTableScan.planTasks(BaseTableScan.java:178) ~[iceberg-core-1.1.0.jar:?]
    at org.apache.iceberg.DataTableScan.planTasks(DataTableScan.java:27) ~[iceberg-core-1.1.0.jar:?]
    at com.starrocks.connector.iceberg.IcebergMetadata.getRemoteFileInfos(IcebergMetadata.java:178) ~[starrocks-fe.jar:?]
    at com.starrocks.connector.iceberg.IcebergMetadata.getRemoteFileInfos(IcebergMetadata.java:157) ~[starrocks-fe.jar:?]
    at com.starrocks.server.MetadataMgr.getRemoteFileInfos(MetadataMgr.java:183) ~[starrocks-fe.jar:?]
    at com.starrocks.connector.iceberg.cost.IcebergStatisticProvider.generateIcebergFileStats(IcebergStatisticProvider.java:80) ~[starrocks-fe.jar:?]
    at com.starrocks.connector.iceberg.cost.IcebergStatisticProvider.getTableStatistics(IcebergStatisticProvider.java:60) ~[starrocks-fe.jar:?]
    at com.starrocks.connector.iceberg.IcebergMetadata.getTableStatistics(IcebergMetadata.java:198) ~[starrocks-fe.jar:?]
    at com.starrocks.server.MetadataMgr.lambda$getTableStatistics$3(MetadataMgr.java:170) ~[starrocks-fe.jar:?]
    at java.util.Optional.map(Optional.java:265) ~[?:?]
    at com.starrocks.server.MetadataMgr.getTableStatistics(MetadataMgr.java:169) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.computeIcebergScanNode(StatisticsCalculator.java:284) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalIcebergScan(StatisticsCalculator.java:272) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalIcebergScan(StatisticsCalculator.java:153) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator.accept(LogicalIcebergScanOperator.java:72) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.estimatorStats(StatisticsCalculator.java:169) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.task.DeriveStatsTask.execute(DeriveStatsTask.java:57) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.task.SeriallyTaskScheduler.executeTasks(SeriallyTaskScheduler.java:68) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.Optimizer.memoOptimize(Optimizer.java:461) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.Optimizer.optimizeByCost(Optimizer.java:168) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.optimizer.Optimizer.optimize(Optimizer.java:110) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.StatementPlanner.createQueryPlan(StatementPlanner.java:140) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.StatementPlanner.planQuery(StatementPlanner.java:115) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:90) ~[starrocks-fe.jar:?]
    at com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:55) ~[starrocks-fe.jar:?]
    at com.starrocks.qe.StmtExecutor.execute(StmtExecutor.java:396) ~[starrocks-fe.jar:?]
    at com.starrocks.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:349) ~[starrocks-fe.jar:?]
    at com.starrocks.qe.ConnectProcessor.dispatch(ConnectProcessor.java:463) ~[starrocks-fe.jar:?]
    at com.starrocks.qe.ConnectProcessor.processOnce(ConnectProcessor.java:729) ~[starrocks-fe.jar:?]
    at com.starrocks.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:69) ~[starrocks-fe.jar:?]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
    at java.lang.Thread.run(Thread.java:829) ~[?:?]
Caused by: java.nio.file.AccessDeniedException: s3://celerdata-4864/a3076908-c737-4d42-b104-581b6b170254/e3730253-692e-474d-a473-4dfb23f537e1/metadata/snap-2346046352581096739-1-3eac4786-95da-483a-8f72-2565b17f78ee.avro: org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException: No AWS Credentials provided by InstanceProfileCredentialsProvider : com.amazonaws.SdkClientException: Failed to connect to service endpoint:
    at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:212) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:175) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3799) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3688) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.extractOrFetchSimpleFileStatus(S3AFileSystem.java:5401) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1465) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1441) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:976) ~[hadoop-common-3.3.3.jar:?]
    at org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:183) ~[iceberg-core-1.1.0.jar:?]
    ... 44 more
Caused by: org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException: No AWS Credentials provided by InstanceProfileCredentialsProvider : com.amazonaws.SdkClientException: Failed to connect to service endpoint:
    at org.apache.hadoop.fs.s3a.AWSCredentialProviderList.getCredentials(AWSCredentialProviderList.java:216) ~[hadoop-aws-3.3.3.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.getCredentialsFromContext(AmazonHttpClient.java:1257) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.runBeforeRequestHandlers(AmazonHttpClient.java:833) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:783) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5437) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.getBucketRegionViaHeadRequest(AmazonS3Client.java:6408) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.fetchRegionFromCache(AmazonS3Client.java:6381) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5422) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5384) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1367) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getObjectMetadata$10(S3AFileSystem.java:2545) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:377) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2533) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2513) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3776) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3688) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.extractOrFetchSimpleFileStatus(S3AFileSystem.java:5401) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1465) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1441) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:976) ~[hadoop-common-3.3.3.jar:?]
    at org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:183) ~[iceberg-core-1.1.0.jar:?]
    ... 44 more
Caused by: com.amazonaws.SdkClientException: Failed to connect to service endpoint:
    at com.amazonaws.internal.EC2ResourceFetcher.doReadResource(EC2ResourceFetcher.java:100) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.internal.InstanceMetadataServiceResourceFetcher.getToken(InstanceMetadataServiceResourceFetcher.java:91) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.internal.InstanceMetadataServiceResourceFetcher.readResource(InstanceMetadataServiceResourceFetcher.java:69) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.internal.EC2ResourceFetcher.readResource(EC2ResourceFetcher.java:66) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.InstanceMetadataServiceCredentialsFetcher.getCredentialsEndpoint(InstanceMetadataServiceCredentialsFetcher.java:58) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.InstanceMetadataServiceCredentialsFetcher.getCredentialsResponse(InstanceMetadataServiceCredentialsFetcher.java:46) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.BaseCredentialsFetcher.fetchCredentials(BaseCredentialsFetcher.java:112) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.BaseCredentialsFetcher.getCredentials(BaseCredentialsFetcher.java:68) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.InstanceProfileCredentialsProvider.getCredentials(InstanceProfileCredentialsProvider.java:165) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at org.apache.hadoop.fs.s3a.AWSCredentialProviderList.getCredentials(AWSCredentialProviderList.java:177) ~[hadoop-aws-3.3.3.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.getCredentialsFromContext(AmazonHttpClient.java:1257) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.runBeforeRequestHandlers(AmazonHttpClient.java:833) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:783) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5437) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.getBucketRegionViaHeadRequest(AmazonS3Client.java:6408) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.fetchRegionFromCache(AmazonS3Client.java:6381) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5422) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5384) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1367) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getObjectMetadata$10(S3AFileSystem.java:2545) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:377) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2533) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2513) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3776) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3688) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.extractOrFetchSimpleFileStatus(S3AFileSystem.java:5401) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1465) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1441) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:976) ~[hadoop-common-3.3.3.jar:?]
    at org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:183) ~[iceberg-core-1.1.0.jar:?]
    ... 44 more
Caused by: java.net.ConnectException: Connection refused (Connection refused)
    at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:?]
    at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:412) ~[?:?]
    at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:255) ~[?:?]
    at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:237) ~[?:?]
    at java.net.Socket.connect(Socket.java:609) ~[?:?]
    at sun.net.NetworkClient.doConnect(NetworkClient.java:177) ~[?:?]
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:507) ~[?:?]
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:602) ~[?:?]
    at sun.net.www.http.HttpClient.<init>(HttpClient.java:275) ~[?:?]
    at sun.net.www.http.HttpClient.New(HttpClient.java:374) ~[?:?]
    at sun.net.www.http.HttpClient.New(HttpClient.java:395) ~[?:?]
    at sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(HttpURLConnection.java:1253) ~[?:?]
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1232) ~[?:?]
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1081) ~[?:?]
    at sun.net.www.protocol.http.HttpURLConnection.connect(HttpURLConnection.java:1015) ~[?:?]
    at com.amazonaws.internal.ConnectionUtils.connectToEndpoint(ConnectionUtils.java:52) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.internal.EC2ResourceFetcher.doReadResource(EC2ResourceFetcher.java:80) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.internal.InstanceMetadataServiceResourceFetcher.getToken(InstanceMetadataServiceResourceFetcher.java:91) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.internal.InstanceMetadataServiceResourceFetcher.readResource(InstanceMetadataServiceResourceFetcher.java:69) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.internal.EC2ResourceFetcher.readResource(EC2ResourceFetcher.java:66) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.InstanceMetadataServiceCredentialsFetcher.getCredentialsEndpoint(InstanceMetadataServiceCredentialsFetcher.java:58) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.InstanceMetadataServiceCredentialsFetcher.getCredentialsResponse(InstanceMetadataServiceCredentialsFetcher.java:46) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.BaseCredentialsFetcher.fetchCredentials(BaseCredentialsFetcher.java:112) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.BaseCredentialsFetcher.getCredentials(BaseCredentialsFetcher.java:68) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.auth.InstanceProfileCredentialsProvider.getCredentials(InstanceProfileCredentialsProvider.java:165) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at org.apache.hadoop.fs.s3a.AWSCredentialProviderList.getCredentials(AWSCredentialProviderList.java:177) ~[hadoop-aws-3.3.3.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.getCredentialsFromContext(AmazonHttpClient.java:1257) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.runBeforeRequestHandlers(AmazonHttpClient.java:833) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:783) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5437) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.getBucketRegionViaHeadRequest(AmazonS3Client.java:6408) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.fetchRegionFromCache(AmazonS3Client.java:6381) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5422) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5384) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1367) ~[aws-java-sdk-bundle-1.11.1026.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getObjectMetadata$10(S3AFileSystem.java:2545) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:377) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2533) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2513) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3776) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3688) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.extractOrFetchSimpleFileStatus(S3AFileSystem.java:5401) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1465) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1441) ~[hadoop-aws-3.3.3.jar:?]
    at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:976) ~[hadoop-common-3.3.3.jar:?]
    at org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:183) ~[iceberg-core-1.1.0.jar:?]
    ... 44 more
alberttwong commented 1 year ago

same results if I use

drop catalog tabular;
create external catalog 'tabular'
PROPERTIES
(
    "aws.s3.use_instance_profile" = "true",
    "aws.s3.region" = "us-east-1",
    "uri" = "https://api.tabular.io/ws",
    "warehouse" = "test301",
    "type" = "iceberg",
    "iceberg.catalog.type" = "rest",
    "security" = "oauth2",
    "session" = "user",
    "credential" = "t-zzzzz"
);
show databases from tabular;
set catalog tabular;
select count(*) from `default`.movies;
alberttwong commented 1 year ago

tried

drop catalog tabular;
create external catalog 'tabular'
PROPERTIES
(
    "aws.s3.access_key" = "xxxxxx",
    "aws.s3.secret_key" = "yyyyyyy",    
    "aws.s3.region" = "us-west-2",
    "uri" = "https://api.tabular.io/ws",
    "warehouse" = "test301",
    "type" = "iceberg",
    "iceberg.catalog.type" = "rest",
    "security" = "oauth2",
    "session" = "user",
    "credential" = "t-zzzz:zzzzzz"
);
show databases from tabular;
set catalog tabular;
select count(*) from `default`.movies;
alberttwong commented 1 year ago 
drop catalog tabular;
create external catalog 'tabular'
PROPERTIES
(
    "header.x-tabular-s3-access" = "vended_credentials",   
    "uri" = "https://api.tabular.io/ws",
    "warehouse" = "test301",
    "type" = "iceberg",
    "iceberg.catalog.type" = "rest",
    "io-impl" = "org.apache.iceberg.aws.s3.S3FileIO",
    "credential" = "t-ZZZ"
);
show databases from tabular;
set catalog tabular;
select count(*) from `default`.movies;

Then I get the AWS region issue in the default AWS profile so then I modified the container startup

ERROR 1064 (HY000): Unable to load region from any of the providers in the chain software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain@6b27437: [software.amazon.awssdk.regions.providers.SystemSettingsRegionProvider@7da35c1a: region must not be blank or empty., software.amazon.awssdk.regions.providers.AwsProfileRegionProvider@50d2e48b: No region provided in profile: default, software.amazon.awssdk.regions.providers.InstanceProfileRegionProvider@b3fb45b: Unable to contact EC2 metadata service.]
docker run -p 9030:9030 -p 8030:8030 -p 8040:8040 -itd --name=starrocks -v starrocks-storage-be:/data/deploy/starrocks/be/storage -v starrocks-storage-fe:/data/deploy/starrocks/fe/meta -e "AWS_REGION=us-west-2"  registry.starrocks.io/starrocks/allin1-ubuntu
2023-06-08 22:36:58,673 WARN (starrocks-mysql-nio-pool-1|119) [StmtExecutor.execute():558] execute Exception, sql select count(*) from `default`.movies
software.amazon.awssdk.core.exception.SdkClientException: Received an UnknownHostException when attempting to interact with a service. See cause for the exact endpoint that is failing to resolve. If this is happening on an endpoint that previously worked, there may be a network connectivity issue or your DNS cache could be storing endpoints for too long.
        at software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:98) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.awscore.interceptor.HelpfulUnknownHostExceptionInterceptor.modifyException(HelpfulUnknownHostExceptionInterceptor.java:59) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain.modifyException(ExecutionInterceptorChain.java:199) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.runModifyException(ExceptionReportingUtils.java:54) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.reportFailureToInterceptors(ExceptionReportingUtils.java:38) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:39) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:167) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:175) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.services.s3.DefaultS3Client.headObject(DefaultS3Client.java:5219) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at org.apache.iceberg.aws.s3.BaseS3File.getObjectMetadata(BaseS3File.java:85) ~[iceberg-aws-1.1.0.jar:?]
        at org.apache.iceberg.aws.s3.S3InputFile.getLength(S3InputFile.java:75) ~[iceberg-aws-1.1.0.jar:?]
        at org.apache.iceberg.avro.AvroIterable.newFileReader(AvroIterable.java:100) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:76) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:36) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.relocated.com.google.common.collect.Iterables.addAll(Iterables.java:337) ~[iceberg-bundled-guava-1.1.0.jar:?]
        at org.apache.iceberg.relocated.com.google.common.collect.Lists.newLinkedList(Lists.java:241) ~[iceberg-bundled-guava-1.1.0.jar:?]
        at org.apache.iceberg.ManifestLists.read(ManifestLists.java:45) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.BaseSnapshot.cacheManifests(BaseSnapshot.java:148) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.BaseSnapshot.dataManifests(BaseSnapshot.java:174) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.DataTableScan.doPlanFiles(DataTableScan.java:82) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.BaseTableScan.planFiles(BaseTableScan.java:152) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.DataTableScan.planFiles(DataTableScan.java:27) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.BaseTableScan.planTasks(BaseTableScan.java:178) ~[iceberg-core-1.1.0.jar:?]
        at org.apache.iceberg.DataTableScan.planTasks(DataTableScan.java:27) ~[iceberg-core-1.1.0.jar:?]
        at com.starrocks.connector.iceberg.IcebergMetadata.getRemoteFileInfos(IcebergMetadata.java:178) ~[starrocks-fe.jar:?]
        at com.starrocks.connector.iceberg.IcebergMetadata.getRemoteFileInfos(IcebergMetadata.java:157) ~[starrocks-fe.jar:?]
        at com.starrocks.server.MetadataMgr.getRemoteFileInfos(MetadataMgr.java:183) ~[starrocks-fe.jar:?]
        at com.starrocks.connector.iceberg.cost.IcebergStatisticProvider.generateIcebergFileStats(IcebergStatisticProvider.java:80) ~[starrocks-fe.jar:?]
        at com.starrocks.connector.iceberg.cost.IcebergStatisticProvider.getTableStatistics(IcebergStatisticProvider.java:60) ~[starrocks-fe.jar:?]
        at com.starrocks.connector.iceberg.IcebergMetadata.getTableStatistics(IcebergMetadata.java:198) ~[starrocks-fe.jar:?]
        at com.starrocks.server.MetadataMgr.lambda$getTableStatistics$3(MetadataMgr.java:170) ~[starrocks-fe.jar:?]
        at java.util.Optional.map(Optional.java:265) ~[?:?]
        at com.starrocks.server.MetadataMgr.getTableStatistics(MetadataMgr.java:169) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.computeIcebergScanNode(StatisticsCalculator.java:284) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalIcebergScan(StatisticsCalculator.java:272) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalIcebergScan(StatisticsCalculator.java:153) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator.accept(LogicalIcebergScanOperator.java:72) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.estimatorStats(StatisticsCalculator.java:169) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.task.DeriveStatsTask.execute(DeriveStatsTask.java:57) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.task.SeriallyTaskScheduler.executeTasks(SeriallyTaskScheduler.java:68) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.Optimizer.memoOptimize(Optimizer.java:461) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.Optimizer.optimizeByCost(Optimizer.java:168) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.optimizer.Optimizer.optimize(Optimizer.java:110) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.StatementPlanner.createQueryPlan(StatementPlanner.java:140) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.StatementPlanner.planQuery(StatementPlanner.java:115) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:90) ~[starrocks-fe.jar:?]
        at com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:55) ~[starrocks-fe.jar:?]
        at com.starrocks.qe.StmtExecutor.execute(StmtExecutor.java:396) ~[starrocks-fe.jar:?]
        at com.starrocks.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:349) ~[starrocks-fe.jar:?]
        at com.starrocks.qe.ConnectProcessor.dispatch(ConnectProcessor.java:463) ~[starrocks-fe.jar:?]
        at com.starrocks.qe.ConnectProcessor.processOnce(ConnectProcessor.java:729) ~[starrocks-fe.jar:?]
        at com.starrocks.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:69) ~[starrocks-fe.jar:?]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
        at java.lang.Thread.run(Thread.java:829) ~[?:?]
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: celerdata-4864.us-west-2.amazonaws.com
        at software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:98) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:43) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:204) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
        ... 56 more
Caused by: java.net.UnknownHostException: celerdata-4864.us-west-2.amazonaws.com
        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:229) ~[?:?]
        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:?]
        at java.net.Socket.connect(Socket.java:609) ~[?:?]