hortonworks-spark / spark-llap

Apache License 2.0
102 stars 68 forks source link

Table have partition column start with underscore #262

Closed thucpk closed 3 years ago

thucpk commented 5 years ago

I don't have a problem when using hive to create table, but builder interface is error when table have partition column start with underscore.

library: pyspark_hwc-1.0.0.3.1.0.0-78.zip, hive-warehouse-connector-assembly-1.0.0.3.1.0.0-78.jar

CREATE TABLE IF NOT EXISTS tbl1 (
  `country` string, 
  `device` string
) PARTITIONED BY (`ds` string);

CREATE TABLE IF NOT EXISTS tbl2 (
  `country` string, 
  `device` string
) PARTITIONED BY (`__ds` string);
# success
df.selectExpr("country", "device", "__ds as ds").write.format(
        HiveWarehouseSession().HIVE_WAREHOUSE_CONNECTOR
).mode("append").option("table", "tbl1").save()

# failed
df.selectExpr("country", "device", "__ds").write.format(
        HiveWarehouseSession().HIVE_WAREHOUSE_CONNECTOR
).mode("append").option("table", "tbl2").save()

# success
hive.createTable(
"tbl1"
).ifNotExists().column(
    "country", "string"
).column(
    "device", "string"
).partition(
    "ds", "string"
).create()

# failed
hive.createTable(
"tbl2"
).ifNotExists().column(
    "country", "string"
).column(
    "device", "string"
).partition(
    "__ds", "string"
).create()