search

apache / pinot

Apache Pinot - A realtime distributed OLAP datastore
https://pinot.apache.org/
Apache License 2.0
5.38k stars 1.26k forks source link

Pinot return no response when upload data through API ingestFromFile #11645

Open duongtruongtrong opened 11 months ago

duongtruongtrong commented 11 months ago

Using API ingestFromFile to upload a json file to Pinot, but there is no response from Pinot and the data is not uploaded as well. I have tried different files and tables, no upload work.

Pinot version:

{
  "pinot-protobuf": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-kafka-2.0": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-avro": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-distribution": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-csv": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-s3": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-segment-uploader-default": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-yammer": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-thrift": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-confluent-avro": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-batch-ingestion-standalone": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-orc": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-clp-log": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-pulsar": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-gcs": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-dropwizard": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-azure": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-hdfs": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-adls": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-kinesis": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-minion-builtin-tasks": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-json": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-parquet": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76",
  "pinot-segment-writer-file-based": "1.1.0-SNAPSHOT-a8411c02d92b10af462592f49e7b10c0c2cb8f76"
}

Example json file: 

{
    "1": {
        "midpoint_id": "1",
        "midpoint_org_id_lvl_1": "1",
        "midpoint_org_id_lvl_2": "2",
        "midpoint_org_id_lvl_3": "6",
        "midpoint_org_id_lvl_4": "13",
        "midpoint_org_id_lvl_5": "22",
        "midpoint_geo_bat_id": "1",
        "midpoint_geo_site_id": "1",
        "midpoint_period_version_id": "1",
        "midpoint_period_id": "1",
        "midpoint_ges_poste_id": "1",
        "midpoint_ges_category_id": "1",
        "midpoint_ges_scope_id": "1",
        "midpoint_transaction_id": "1",
        "midpoint_flow_id": "1",
        "midpoint_co2": "8",
        "midpoint_co2f": "0",
        "midpoint_ch4f": "0",
        "midpoint_ch4b": "0",
        "midpoint_no2": "0",
        "midpoint_uncertainty": "67",
        "midpoint_calculation_date": "2023-01-01 20:34:36.360",
        "midpoint_em_date": "2023-01-01 20:34:36.360"
    },
    "2": {
        "midpoint_id": "2",
        "midpoint_org_id_lvl_1": "1",
        "midpoint_org_id_lvl_2": "2",
        "midpoint_org_id_lvl_3": "6",
        "midpoint_org_id_lvl_4": "13",
        "midpoint_org_id_lvl_5": "22",
        "midpoint_geo_bat_id": "1",
        "midpoint_geo_site_id": "1",
        "midpoint_period_version_id": "1",
        "midpoint_period_id": "1",
        "midpoint_ges_poste_id": "1",
        "midpoint_ges_category_id": "1",
        "midpoint_ges_scope_id": "1",
        "midpoint_transaction_id": "2",
        "midpoint_flow_id": "1",
        "midpoint_co2": "11",
        "midpoint_co2f": "0",
        "midpoint_ch4f": "0",
        "midpoint_ch4b": "0",
        "midpoint_no2": "0",
        "midpoint_uncertainty": "18",
        "midpoint_calculation_date": "2023-01-02 14:21:16.750",
        "midpoint_em_date": "2023-01-02 14:21:16.750"
    }
}

Schema:

{
    "schemaName": "midpoint",
    "dimensionFieldSpecs": [
      {
        "name": "midpoint_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_org_id_lvl_1",
        "dataType": "INT"
      },
      {
        "name": "midpoint_org_id_lvl_2",
        "dataType": "INT"
      },
      {
        "name": "midpoint_org_id_lvl_3",
        "dataType": "INT"
      },
      {
        "name": "midpoint_org_id_lvl_4",
        "dataType": "INT"
      },
      {
        "name": "midpoint_org_id_lvl_5",
        "dataType": "INT"
      },
      {
        "name": "midpoint_geo_bat_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_geo_site_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_period_version_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_period_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_ges_poste_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_ges_category_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_ges_scope_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_transaction_id",
        "dataType": "INT"
      },
      {
        "name": "midpoint_flow_id",
        "dataType": "INT"
      }
    ],
    "metricFieldSpecs": [
      {
        "name": "midpoint_co2",
        "dataType": "FLOAT"
      },
      {
        "name": "midpoint_co2f",
        "dataType": "FLOAT"
      },
      {
        "name": "midpoint_ch4f",
        "dataType": "FLOAT"
      },
      {
        "name": "midpoint_ch4b",
        "dataType": "FLOAT"
      },
      {
        "name": "midpoint_no2",
        "dataType": "FLOAT"
      },
      {
        "name": "midpoint_uncertainty",
        "dataType": "FLOAT"
      }
    ],
    "dateTimeFieldSpecs": [
      {
        "name": "midpoint_calculation_date",
        "dataType": "TIMESTAMP",
        "format": "1:MILLISECONDS:EPOCH",
        "granularity": "1:MILLISECONDS"
      },
      {
        "name": "midpoint_em_date",
        "dataType": "TIMESTAMP",
        "format": "1:MILLISECONDS:EPOCH",
        "granularity": "1:MILLISECONDS"
      }
    ]
  }

Table config:

{
  "tableName": "midpoint",
  "tableType": "OFFLINE",
  "tenants": {
    "broker": "DefaultTenant",
    "server": "DefaultTenant"
  },
  "segmentsConfig": {
    "schemaName": "midpoint",
    "timeColumnName": "midpoint_em_date",
    "replication": "2",
    "replicasPerPartition": "2",
    "retentionTimeUnit": "DAYS",
    "retentionTimeValue": "366",
    "segmentPushType": "APPEND",
    "segmentPushFrequency": "HOURLY",
    "crypterClassName": null,
    "peerSegmentDownloadScheme": null
  },
  "tableIndexConfig": {
    "loadMode": "MMAP",
    "invertedIndexColumns": [],
    "createInvertedIndexDuringSegmentGeneration": false,
    "rangeIndexColumns": [],
    "sortedColumn": [
      "midpoint_em_date",
      "midpoint_calculation_date",
      "midpoint_flow_id",
      "midpoint_geo_bat_id",
      "midpoint_geo_site_id",
      "midpoint_period_version_id",
      "midpoint_period_id",
      "midpoint_ges_poste_id",
      "midpoint_ges_category_id",
      "midpoint_ges_scope_id",
      "midpoint_transaction_id",
      "midpoint_org_id_lvl_1",
      "midpoint_org_id_lvl_2",
      "midpoint_org_id_lvl_3",
      "midpoint_org_id_lvl_4",
      "midpoint_org_id_lvl_5"
    ],
    "bloomFilterColumns": [],
    "bloomFilterConfigs": null,
    "noDictionaryColumns": [],
    "onHeapDictionaryColumns": [],
    "varLengthDictionaryColumns": [],
    "enableDefaultStarTree": false,
    "starTreeIndexConfigs": null,
    "enableDynamicStarTreeCreation": false,
    "segmentPartitionConfig": null,
    "columnMinMaxValueGeneratorMode": null,
    "nullHandlingEnabled": false,
    "DictionaryColumns": []
  },
  "metadata": {},
  "ingestionConfig": {
    "filterConfig": null,
    "transformConfigs": []
  },
  "quota": {
    "storage": null,
    "maxQueriesPerSecond": null
  },
  "task": {
    "taskTypeConfigsMap": {
      "MergeRollupTask": {
        "1day.mergeType": "rollup",
        "1day.bucketTimePeriod": "1d",
        "1day.bufferTimePeriod": "1d",
        "1day.roundBucketTimePeriod": "1d"
      },
      "SegmentGenerationAndPushTask": {
        "schedule": "0 */10 * * * ?",
        "tableMaxNumTasks": "10"
      }
    }
  },
  "routing": {
    "segmentPrunerTypes": null,
    "instanceSelectorType": null
  },
  "instanceAssignmentConfigMap": null,
  "query": {
    "timeoutMs": null
  },
  "fieldConfigList": [],
  "upsertConfig": null,
  "tierConfigs": null
}

image

Jackie-Jiang commented 11 months ago

cc @snleee @swaminathanmanish