search

xuchuanyin / workbench

0 stars 0 forks source link

2018-11-16 ecs #79

Open xuchuanyin opened 5 years ago

xuchuanyin commented 5 years ago 
/home/xu/hadoop_related/spark-2.2.1-bin-hadoop2.7/bin/beeline  -u jdbc:hive2://10.90.51.97:10000 -nroot -e "drop table if exists lineitem;create table if not exists LINEITEM(L_ORDERKEY INT , L_PARTKEY INT , L_SUPPKEY string,       L_LINENUMBER int, L_QUANTITY double, L_EXTENDEDPRICE double, L_DISCOUNT double, L_TAX double,              L_RETURNFLAG string, L_LINESTATUS string, L_SHIPDATE date, L_COMMITDATE date, L_RECEIPTDATE date,          L_SHIPINSTRUCT string, L_SHIPMODE string, L_COMMENT string) STORED AS carbondata                           TBLPROPERTIES('table_blocksize'='256', 'SORT_COLUMNS'='L_SHIPDATE,L_RETURNFLAG,L_SHIPMODE,                 L_RECEIPTDATE,L_SHIPINSTRUCT');"

/home/xu/hadoop_related/spark-2.2.1-bin-hadoop2.7/bin/beeline  -u jdbc:hive2://10.90.51.97:10000 -nroot -e "load data inpath 'hdfs://slave1:9000/tpch-data/lineitem' into table LINEITEM options ('header'='false','delimiter'='|');load data inpath 'hdfs://slave1:9000/tpch-data/lineitem' into table LINEITEM options ('header'='false','delimiter'='|');load data inpath 'hdfs://slave1:9000/tpch-data/lineitem' into table LINEITEM options ('header'='false','delimiter'='|');alter table LINEITEM compact 'MAJOR';show segments for table LINEITEM;"
xuchuanyin commented 5 years ago 
hws操作

 set TIMEOUT=1000000000

 # carbon git source
/opt/carbon/carbondata

# run  mysql
service mysql start
或
${MYSQL_HOME}/bin/mysqld_safe --user=mysql &

# run hadoop
$HADOOP_HOME/sbin/start-all.sh

# run carbon

/opt/carbon/carbon.properties

cd /opt/carbon/
$SPARK_HOME/bin/spark-submit --master yarn --class org.apache.carbondata.spark.thriftserver.CarbonThriftServer /opt/carbon/apache-carbondata-1.5.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar hdfs://master:9000/user/hive/warehouse/carbon.store

$SPARK_HOME/bin/spark-submit --master yarn --class org.apache.carbondata.spark.thriftserver.CarbonThriftServer /opt/carbon/apache-carbondata-1.6.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar hdfs://master:9000/user/hive/warehouse/carbon.store

## run carbon in more general way
rm /opt/carbon/carbon.jar
cp /opt/carbon/carbondata/assembly/target/scala-2.11/apache-carbondata-1.6.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar /opt/carbon/carbon.jar

$SPARK_HOME/bin/spark-submit --master yarn --class org.apache.carbondata.spark.thriftserver.CarbonThriftServer /opt/carbon/carbon.jar hdfs://master:9000/user/hive/warehouse/carbon.store

# run beeline
$SPARK_HOME/bin/beeline -u jdbc:hive2://127.0.0.1:10000

# nmon for sql file
/opt/nmon_related/load.sh test.sql

java -cp '/home/xu/ws/tpch-java/target/tpch-java-1.0-SNAPSHOT-jar-with-dependencies.jar:/usr/local/share/spark/jars/*' ind.xuchuanyin.tpch.cli.CliTool -c exec_sql -f /srv/BigData/hadoop/xu/scripts/carbondata/carbon_tpch_create_table_perf_2.json

load data inpath 'hdfs://master:9000/tpch-100g-data/supplier' into table SUPPLIER options ('header'='false','delimiter'='|');

cd /srv/BigData/hadoop/xu
time sh /srv/BigData/hadoop/xu/query.sh /srv/BigData/hadoop/xu/scripts/carbondata/carbon_tpch_create_table_perf.json
time sh /srv/BigData/hadoop/xu/query.sh /srv/BigData/hadoop/xu/scripts/carbondata/carbon_tpch_load_table_perf.json
time sh /srv/BigData/hadoop/xu/query.sh /srv/BigData/hadoop/xu/scripts/carbondata/carbon_tpch_query_table_perf.json

time sh /srv/BigData/hadoop/xu/query.sh /srv/BigData/hadoop/xu/scripts/carbondata/carbon_tpch_create_table_perf_2.json
time sh /srv/BigData/hadoop/xu/query.sh /srv/BigData/hadoop/xu/scripts/carbondata/carbon_tpch_load_table_perf_2.json
time sh /srv/BigData/hadoop/xu/query.sh /srv/BigData/hadoop/xu/scripts/carbondata/carbon_tpch_query_table_perf_2.json

$SPARK_HOME/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -nroot -e "drop table if exists LINEITEM;create table if not exists LINEITEM(L_ORDERKEY INT , L_PARTKEY INT , L_SUPPKEY string, L_LINENUMBER int, L_QUANTITY double, L_EXTENDEDPRICE double, L_DISCOUNT double, L_TAX double, L_RETURNFLAG string, L_LINESTATUS string, L_SHIPDATE date, L_COMMITDATE date, L_RECEIPTDATE date, L_SHIPINSTRUCT string, L_SHIPMODE string, L_COMMENT string) STORED AS carbondata TBLPROPERTIES('table_blocksize'='256', 'SORT_COLUMNS'='L_SHIPDATE,L_RETURNFLAG,L_SHIPMODE,L_RECEIPTDATE,L_SHIPINSTRUCT')"

$SPARK_HOME/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -nroot -e "load data inpath 'hdfs://master:9000/tpch-100g-data/lineitem' into table LINEITEM options ('header'='false','delimiter'='|');load data inpath 'hdfs://master:9000/tpch-100g-data/lineitem' into table LINEITEM options ('header'='false','delimiter'='|');load data inpath 'hdfs://master:9000/tpch-100g-data/lineitem' into table LINEITEM options ('header'='false','delimiter'='|');alter table LINEITEM compact 'MAJOR';show segments for table LINEITEM"

$SPARK_HOME/bin/beeline -u jdbc:hive2://127.0.0.1:10000 -nroot -e "drop table if exists LINEITEM"