search

trinodb / trino

Official repository of Trino, the distributed SQL query engine for big data, formerly known as PrestoSQL (https://trino.io)
https://trino.io
Apache License 2.0
10.49k stars 3.02k forks source link

Allow large results in BigQuery connector #23570

Open ebyhr opened 1 month ago

ebyhr commented 1 month ago

The connector throws an exception when the result from BigQuery is too large. We should enable QueryJobConfiguration.Builder#setAllowLargeResults with/without config property. 

        "com.google.api.client.googleapis.json.GoogleJsonResponseException.from(GoogleJsonResponseException.java:146)",
        "com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:118)",
        "com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:37)",
        "com.google.api.client.googleapis.services.AbstractGoogleClientRequest$3.interceptResponse(AbstractGoogleClientRequest.java:479)",
        "com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1111)",
        "com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:565)",
        "com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:506)",
        "com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:616)",
        "com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc.getQueryResults(HttpBigQueryRpc.java:743)",
        "com.google.cloud.bigquery.BigQueryImpl$36.call(BigQueryImpl.java:1500)",
        "com.google.cloud.bigquery.BigQueryImpl$36.call(BigQueryImpl.java:1495)",
        "com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:102)",
        "com.google.cloud.bigquery.BigQueryRetryHelper.run(BigQueryRetryHelper.java:86)",
        "com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries(BigQueryRetryHelper.java:49)",
        "com.google.cloud.bigquery.BigQueryImpl.getQueryResults(BigQueryImpl.java:1494)",
        "com.google.cloud.bigquery.BigQueryImpl.getQueryResults(BigQueryImpl.java:1478)",
        "com.google.cloud.bigquery.Job$1.call(Job.java:390)",
        "com.google.cloud.bigquery.Job$1.call(Job.java:387)",
        "com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:102)",
        "com.google.cloud.bigquery.BigQueryRetryHelper.run(BigQueryRetryHelper.java:86)",
        "com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries(BigQueryRetryHelper.java:49)",
        "com.google.cloud.bigquery.Job.waitForQueryResults(Job.java:386)",
        "com.google.cloud.bigquery.Job.getQueryResults(Job.java:333)",
        "com.google.cloud.bigquery.BigQueryImpl.query(BigQueryImpl.java:1360)",
        "io.trino.plugin.bigquery.BigQueryClient.execute(BigQueryClient.java:407)",
        "io.trino.plugin.bigquery.BigQueryClient.executeQuery(BigQueryClient.java:398)",
        "io.trino.plugin.bigquery.BigQueryQueryPageSource.<init>(BigQueryQueryPageSource.java:105)",
        "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createQueryPageSource(BigQueryPageSourceProvider.java:125)",
        "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createPageSource(BigQueryPageSourceProvider.java:95)",
        "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createPageSource(BigQueryPageSourceProvider.java:84)",
        "io.trino.spi.connector.ConnectorPageSourceProvider.createPageSource(ConnectorPageSourceProvider.java:46)",
        "io.trino.split.PageSourceManager$PageSourceProviderInstance.createPageSource(PageSourceManager.java:82)",
        "io.trino.split.TableAwarePageSourceProvider.lambda$createPageSource$1(TableAwarePageSourceProvider.java:72)",
        "java.base/java.util.Optional.orElseGet(Optional.java:364)",
        "io.trino.split.TableAwarePageSourceProvider.createPageSource(TableAwarePageSourceProvider.java:72)",
        "io.trino.operator.TableScanOperator.getOutput(TableScanOperator.java:285)",
        "io.trino.operator.Driver.processInternal(Driver.java:400)",
        "io.trino.operator.Driver.lambda$process$8(Driver.java:303)",
        "io.trino.operator.Driver.tryWithLock(Driver.java:706)",
        "io.trino.operator.Driver.process(Driver.java:295)",
        "io.trino.operator.Driver.processForDuration(Driver.java:266)",
        "io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:891)",
        "io.trino.execution.executor.timesharing.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:189)",
        "io.trino.execution.executor.timesharing.TimeSharingTaskExecutor$TaskRunner.run(TimeSharingTaskExecutor.java:651)",
        "io.trino.$gen.xxx.run(Unknown Source)",
        "java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)",
        "java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)",
        "java.base/java.lang.Thread.run(Thread.java:1570)"
      ],
      "errorCode": {
        "code": 65536,
        "name": "GENERIC_INTERNAL_ERROR",
        "type": "INTERNAL_ERROR",
        "fatal": false
      }
    },
    "suppressed": [],
    "stack": [
      "com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc.translate(HttpBigQueryRpc.java:116)",
      "com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc.getQueryResults(HttpBigQueryRpc.java:745)",
      "com.google.cloud.bigquery.BigQueryImpl$36.call(BigQueryImpl.java:1500)",
      "com.google.cloud.bigquery.BigQueryImpl$36.call(BigQueryImpl.java:1495)",
      "com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:102)",
      "com.google.cloud.bigquery.BigQueryRetryHelper.run(BigQueryRetryHelper.java:86)",
      "com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries(BigQueryRetryHelper.java:49)",
      "com.google.cloud.bigquery.BigQueryImpl.getQueryResults(BigQueryImpl.java:1494)",
      "com.google.cloud.bigquery.BigQueryImpl.getQueryResults(BigQueryImpl.java:1478)",
      "com.google.cloud.bigquery.Job$1.call(Job.java:390)",
      "com.google.cloud.bigquery.Job$1.call(Job.java:387)",
      "com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:102)",
      "com.google.cloud.bigquery.BigQueryRetryHelper.run(BigQueryRetryHelper.java:86)",
      "com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries(BigQueryRetryHelper.java:49)",
      "com.google.cloud.bigquery.Job.waitForQueryResults(Job.java:386)",
      "com.google.cloud.bigquery.Job.getQueryResults(Job.java:333)",
      "com.google.cloud.bigquery.BigQueryImpl.query(BigQueryImpl.java:1360)",
      "io.trino.plugin.bigquery.BigQueryClient.execute(BigQueryClient.java:407)",
      "io.trino.plugin.bigquery.BigQueryClient.executeQuery(BigQueryClient.java:398)",
      "io.trino.plugin.bigquery.BigQueryQueryPageSource.<init>(BigQueryQueryPageSource.java:105)",
      "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createQueryPageSource(BigQueryPageSourceProvider.java:125)",
      "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createPageSource(BigQueryPageSourceProvider.java:95)",
      "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createPageSource(BigQueryPageSourceProvider.java:84)",
      "io.trino.spi.connector.ConnectorPageSourceProvider.createPageSource(ConnectorPageSourceProvider.java:46)",
      "io.trino.split.PageSourceManager$PageSourceProviderInstance.createPageSource(PageSourceManager.java:82)",
      "io.trino.split.TableAwarePageSourceProvider.lambda$createPageSource$1(TableAwarePageSourceProvider.java:72)",
      "java.base/java.util.Optional.orElseGet(Optional.java:364)",
      "io.trino.split.TableAwarePageSourceProvider.createPageSource(TableAwarePageSourceProvider.java:72)",
      "io.trino.operator.TableScanOperator.getOutput(TableScanOperator.java:285)",
      "io.trino.operator.Driver.processInternal(Driver.java:400)",
      "io.trino.operator.Driver.lambda$process$8(Driver.java:303)",
      "io.trino.operator.Driver.tryWithLock(Driver.java:706)",
      "io.trino.operator.Driver.process(Driver.java:295)",
      "io.trino.operator.Driver.processForDuration(Driver.java:266)",
      "io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:891)",
      "io.trino.execution.executor.timesharing.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:189)",
      "io.trino.execution.executor.timesharing.TimeSharingTaskExecutor$TaskRunner.run(TimeSharingTaskExecutor.java:651)",
      "io.trino.$gen.xxx.run(Unknown Source)",
      "java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)",
      "java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)",
      "java.base/java.lang.Thread.run(Thread.java:1570)"
    ],
    "errorCode": {
      "code": 65536,
      "name": "GENERIC_INTERNAL_ERROR",
      "type": "INTERNAL_ERROR",
      "fatal": false
    }
  },
  "suppressed": [],
  "stack": [
    "io.trino.plugin.bigquery.BigQueryClient.execute(BigQueryClient.java:410)",
    "io.trino.plugin.bigquery.BigQueryClient.executeQuery(BigQueryClient.java:398)",
    "io.trino.plugin.bigquery.BigQueryQueryPageSource.<init>(BigQueryQueryPageSource.java:105)",
    "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createQueryPageSource(BigQueryPageSourceProvider.java:125)",
    "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createPageSource(BigQueryPageSourceProvider.java:95)",
    "io.trino.plugin.bigquery.BigQueryPageSourceProvider.createPageSource(BigQueryPageSourceProvider.java:84)",
    "io.trino.spi.connector.ConnectorPageSourceProvider.createPageSource(ConnectorPageSourceProvider.java:46)",
    "io.trino.split.PageSourceManager$PageSourceProviderInstance.createPageSource(PageSourceManager.java:82)",
    "io.trino.split.TableAwarePageSourceProvider.lambda$createPageSource$1(TableAwarePageSourceProvider.java:72)",
    "java.base/java.util.Optional.orElseGet(Optional.java:364)",
    "io.trino.split.TableAwarePageSourceProvider.createPageSource(TableAwarePageSourceProvider.java:72)",
    "io.trino.operator.TableScanOperator.getOutput(TableScanOperator.java:285)",
    "io.trino.operator.Driver.processInternal(Driver.java:400)",
    "io.trino.operator.Driver.lambda$process$8(Driver.java:303)",
    "io.trino.operator.Driver.tryWithLock(Driver.java:706)",
    "io.trino.operator.Driver.process(Driver.java:295)",
    "io.trino.operator.Driver.processForDuration(Driver.java:266)",
    "io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:891)",
    "io.trino.execution.executor.timesharing.PrioritizedSplitRunner.process(PrioritizedSplitRunner.java:189)",
    "io.trino.execution.executor.timesharing.TimeSharingTaskExecutor$TaskRunner.run(TimeSharingTaskExecutor.java:651)",
    "io.trino.$gen.xxx.run(Unknown Source)",
    "java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)",
    "java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)",
    "java.base/java.lang.Thread.run(Thread.java:1570)"
  ]
ebyhr commented 1 month ago

The past attempt https://github.com/trinodb/trino/pull/17069