Closed alberttwong closed 1 year ago
same results if I use
drop catalog tabular;
create external catalog 'tabular'
PROPERTIES
(
"aws.s3.use_instance_profile" = "true",
"aws.s3.region" = "us-east-1",
"uri" = "https://api.tabular.io/ws",
"warehouse" = "test301",
"type" = "iceberg",
"iceberg.catalog.type" = "rest",
"security" = "oauth2",
"session" = "user",
"credential" = "t-zzzzz"
);
show databases from tabular;
set catalog tabular;
select count(*) from `default`.movies;
tried
drop catalog tabular;
create external catalog 'tabular'
PROPERTIES
(
"aws.s3.access_key" = "xxxxxx",
"aws.s3.secret_key" = "yyyyyyy",
"aws.s3.region" = "us-west-2",
"uri" = "https://api.tabular.io/ws",
"warehouse" = "test301",
"type" = "iceberg",
"iceberg.catalog.type" = "rest",
"security" = "oauth2",
"session" = "user",
"credential" = "t-zzzz:zzzzzz"
);
show databases from tabular;
set catalog tabular;
select count(*) from `default`.movies;
drop catalog tabular;
create external catalog 'tabular'
PROPERTIES
(
"header.x-tabular-s3-access" = "vended_credentials",
"uri" = "https://api.tabular.io/ws",
"warehouse" = "test301",
"type" = "iceberg",
"iceberg.catalog.type" = "rest",
"io-impl" = "org.apache.iceberg.aws.s3.S3FileIO",
"credential" = "t-ZZZ"
);
show databases from tabular;
set catalog tabular;
select count(*) from `default`.movies;
Then I get the AWS region issue in the default AWS profile so then I modified the container startup
ERROR 1064 (HY000): Unable to load region from any of the providers in the chain software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain@6b27437: [software.amazon.awssdk.regions.providers.SystemSettingsRegionProvider@7da35c1a: region must not be blank or empty., software.amazon.awssdk.regions.providers.AwsProfileRegionProvider@50d2e48b: No region provided in profile: default, software.amazon.awssdk.regions.providers.InstanceProfileRegionProvider@b3fb45b: Unable to contact EC2 metadata service.]
docker run -p 9030:9030 -p 8030:8030 -p 8040:8040 -itd --name=starrocks -v starrocks-storage-be:/data/deploy/starrocks/be/storage -v starrocks-storage-fe:/data/deploy/starrocks/fe/meta -e "AWS_REGION=us-west-2" registry.starrocks.io/starrocks/allin1-ubuntu
2023-06-08 22:36:58,673 WARN (starrocks-mysql-nio-pool-1|119) [StmtExecutor.execute():558] execute Exception, sql select count(*) from `default`.movies
software.amazon.awssdk.core.exception.SdkClientException: Received an UnknownHostException when attempting to interact with a service. See cause for the exact endpoint that is failing to resolve. If this is happening on an endpoint that previously worked, there may be a network connectivity issue or your DNS cache could be storing endpoints for too long.
at software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:98) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.awscore.interceptor.HelpfulUnknownHostExceptionInterceptor.modifyException(HelpfulUnknownHostExceptionInterceptor.java:59) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain.modifyException(ExecutionInterceptorChain.java:199) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.runModifyException(ExceptionReportingUtils.java:54) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.ExceptionReportingUtils.reportFailureToInterceptors(ExceptionReportingUtils.java:38) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:39) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:167) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:175) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.services.s3.DefaultS3Client.headObject(DefaultS3Client.java:5219) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at org.apache.iceberg.aws.s3.BaseS3File.getObjectMetadata(BaseS3File.java:85) ~[iceberg-aws-1.1.0.jar:?]
at org.apache.iceberg.aws.s3.S3InputFile.getLength(S3InputFile.java:75) ~[iceberg-aws-1.1.0.jar:?]
at org.apache.iceberg.avro.AvroIterable.newFileReader(AvroIterable.java:100) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:76) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:36) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.relocated.com.google.common.collect.Iterables.addAll(Iterables.java:337) ~[iceberg-bundled-guava-1.1.0.jar:?]
at org.apache.iceberg.relocated.com.google.common.collect.Lists.newLinkedList(Lists.java:241) ~[iceberg-bundled-guava-1.1.0.jar:?]
at org.apache.iceberg.ManifestLists.read(ManifestLists.java:45) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.BaseSnapshot.cacheManifests(BaseSnapshot.java:148) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.BaseSnapshot.dataManifests(BaseSnapshot.java:174) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.DataTableScan.doPlanFiles(DataTableScan.java:82) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.BaseTableScan.planFiles(BaseTableScan.java:152) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.DataTableScan.planFiles(DataTableScan.java:27) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.BaseTableScan.planTasks(BaseTableScan.java:178) ~[iceberg-core-1.1.0.jar:?]
at org.apache.iceberg.DataTableScan.planTasks(DataTableScan.java:27) ~[iceberg-core-1.1.0.jar:?]
at com.starrocks.connector.iceberg.IcebergMetadata.getRemoteFileInfos(IcebergMetadata.java:178) ~[starrocks-fe.jar:?]
at com.starrocks.connector.iceberg.IcebergMetadata.getRemoteFileInfos(IcebergMetadata.java:157) ~[starrocks-fe.jar:?]
at com.starrocks.server.MetadataMgr.getRemoteFileInfos(MetadataMgr.java:183) ~[starrocks-fe.jar:?]
at com.starrocks.connector.iceberg.cost.IcebergStatisticProvider.generateIcebergFileStats(IcebergStatisticProvider.java:80) ~[starrocks-fe.jar:?]
at com.starrocks.connector.iceberg.cost.IcebergStatisticProvider.getTableStatistics(IcebergStatisticProvider.java:60) ~[starrocks-fe.jar:?]
at com.starrocks.connector.iceberg.IcebergMetadata.getTableStatistics(IcebergMetadata.java:198) ~[starrocks-fe.jar:?]
at com.starrocks.server.MetadataMgr.lambda$getTableStatistics$3(MetadataMgr.java:170) ~[starrocks-fe.jar:?]
at java.util.Optional.map(Optional.java:265) ~[?:?]
at com.starrocks.server.MetadataMgr.getTableStatistics(MetadataMgr.java:169) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.computeIcebergScanNode(StatisticsCalculator.java:284) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalIcebergScan(StatisticsCalculator.java:272) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalIcebergScan(StatisticsCalculator.java:153) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator.accept(LogicalIcebergScanOperator.java:72) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.statistics.StatisticsCalculator.estimatorStats(StatisticsCalculator.java:169) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.task.DeriveStatsTask.execute(DeriveStatsTask.java:57) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.task.SeriallyTaskScheduler.executeTasks(SeriallyTaskScheduler.java:68) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.Optimizer.memoOptimize(Optimizer.java:461) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.Optimizer.optimizeByCost(Optimizer.java:168) ~[starrocks-fe.jar:?]
at com.starrocks.sql.optimizer.Optimizer.optimize(Optimizer.java:110) ~[starrocks-fe.jar:?]
at com.starrocks.sql.StatementPlanner.createQueryPlan(StatementPlanner.java:140) ~[starrocks-fe.jar:?]
at com.starrocks.sql.StatementPlanner.planQuery(StatementPlanner.java:115) ~[starrocks-fe.jar:?]
at com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:90) ~[starrocks-fe.jar:?]
at com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:55) ~[starrocks-fe.jar:?]
at com.starrocks.qe.StmtExecutor.execute(StmtExecutor.java:396) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:349) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.dispatch(ConnectProcessor.java:463) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.processOnce(ConnectProcessor.java:729) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:69) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:829) ~[?:?]
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: celerdata-4864.us-west-2.amazonaws.com
at software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:98) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:43) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:204) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37) ~[cloudfs-hadoop-with-dependencies-1.1.21.jar:?]
... 56 more
Caused by: java.net.UnknownHostException: celerdata-4864.us-west-2.amazonaws.com
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:229) ~[?:?]
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:?]
at java.net.Socket.connect(Socket.java:609) ~[?:?]
Using