search

xuchuanyin / workbench

0 stars 0 forks source link

2018-07-31 bloom preagg serial #55

Open xuchuanyin opened 5 years ago

xuchuanyin commented 5 years ago

test("CARBONDATA-2802") { sql( s""" | CREATE TABLE $bloomDMSampleTable (id int, name string, salary float,dob date) | STORED BY 'carbondata' | TBLPROPERTIES('dictionary_include'='id') """.stripMargin) sql( s""" | INSERT INTO $bloomDMSampleTable VALUES | ('1', 'name1', '11.1', '2018-07-01'), | ('2', 'name2', '21.1', '2018-07-02'), | ('3', 'name3', '31.1', '2018-07-03'), | ('4', 'name4', '41.1', '2018-07-04') """.stripMargin) val preAggOnBase = "preagg_on_base" sql( s""" | CREATE DATAMAP $preAggOnBase ON TABLE $bloomDMSampleTable | USING 'preaggregate' AS | select id, count(id) from $bloomDMSampleTable group by id """.stripMargin) // query the preagg table sql(s"SELECT id, count(id) FROM $bloomDMSampleTable group by id").show(false) sql(s"SELECT * FROM $bloomDMSampleTable ").show(false) sql( s""" | CREATE DATAMAP $dataMapName ON TABLE $bloomDMSampleTable | USING 'bloomfilter' | DMPROPERTIES('INDEX_COLUMNS'='id', 'BLOOM_SIZE'='320000', 'BLOOM_FPP'='0.01', 'BLOOM_COMPRESS'='TRUE') """.stripMargin) sql(s"SELECT id, count(id) FROM $bloomDMSampleTable group by id").show(false) sql(s"SELECT id, name FROM $bloomDMSampleTable where id = 1").show(false) }

xuchuanyin commented 5 years ago

test("CARBONDATA-2802-2: fix serialization error when creating bloom datamap after selecting preagg") { sql( s""" | CREATE TABLE $bloomDMSampleTable (id int, name string, salary float,dob date) | STORED BY 'carbondata' | TBLPROPERTIES('dictionary_include'='id') """.stripMargin) sql( s""" | CREATE DATAMAP $dataMapName ON TABLE $bloomDMSampleTable | USING 'bloomfilter' | DMPROPERTIES('INDEX_COLUMNS'='id', 'BLOOM_SIZE'='320000', 'BLOOM_FPP'='0.01', 'BLOOM_COMPRESS'='TRUE') """.stripMargin) sql( s""" | INSERT INTO $bloomDMSampleTable VALUES | ('1', 'name1', '11.1', '2018-07-01'), | ('2', 'name2', '21.1', '2018-07-02'), | ('3', 'name3', '31.1', '2018-07-03'), | ('4', 'name4', '41.1', '2018-07-04') """.stripMargin) val preAggOnBase = "preagg_on_base" sql( s""" | CREATE DATAMAP $preAggOnBase ON TABLE $bloomDMSampleTable | USING 'preaggregate' AS | select id, count(id) from $bloomDMSampleTable group by id """.stripMargin) // query the preagg table sql(s"SELECT id, count(id) FROM $bloomDMSampleTable group by id").show(false) sql(s"SELECT * FROM $bloomDMSampleTable ").show(false) val carbonTable = CarbonEnv.getCarbonTable(None, bloomDMSampleTable)(sqlContext.sparkSession) val tableIdentifier = carbonTable.getAbsoluteTableIdentifier DataMapStoreManager.getInstance().clearDataMaps(tableIdentifier)

sql(s"SELECT id, count(id) FROM $bloomDMSampleTable group by id").show(false)
sql(s"SELECT id, name FROM $bloomDMSampleTable where id = 1").show(false)

}