Spark - intellij idea中不能运行最大行单词个数（harli补充：不能直接设置master地址，启动提交到集群的调试）

1.集群环境安装完好 2.同样函数在集群shell中可以正常运行 3.在intellij 不能正常运行 QuickStart.scala 4.控制以日志输出 console.log

QuickStart.scala

import org.apache.spark.{SparkContext, SparkConf}

/**

Created by hadoop on 16/1/14. */ object QuickStart extends App{

val conf = new SparkConf conf.setAppName("快速开始2").setMaster("spark://S20:7077") conf.setJars(Array("/Users/hadoop/workspace/bigdata/all_frame_intellij/Spark1.6.0-practice-maven/target/Spark1.6.0-practice-maven-1.0.jar"))

val sc = new SparkContext(conf) val textFile = sc.textFile("hdfs://S20:9000/library/wordcount/input/Data/wordCount.txt",1) /**

* textFile 在 scala 程序中不能直接打印，需要先进行处理
*/

//val first = textFile.first() val word = textFile.map(line => line.split(" ").size).reduce((a,b) => if(a>b) a else b)

//val numSpark = textFile.filter(line => line.cont) println("行数量:textFile.count:" + textFile.count()) // textFile.foreach(x => println("textFile输出的内容:"+ x)) sc.stop()

}

console.log

/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/bin/java -Didea.launcher.port=7534 "-Didea.launcher.bin.path=/Applications/IntelliJ IDEA 15.app/Contents/bin" -Dfile.encoding=UTF-8 -classpath "/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/deploy.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/cldrdata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/dnsns.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/jaccess.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/jfxrt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/localedata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/nashorn.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/sunec.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/sunjce_provider.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/sunpkcs11.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/ext/zipfs.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/javaws.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/jfxswt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/management-agent.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/plugin.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/lib/ant-javafx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/lib/dt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/lib/javafx-mx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/lib/jconsole.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/lib/packager.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/lib/sa-jdi.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_65.jdk/Contents/Home/lib/tools.jar:/Users/hadoop/workspace/bigdata/all_frame_intellij/Spark1.6.0-practice-maven/target/classes:/opt/modules/scala/scala-2.10.6-resources/scala-2.10.6/lib/scala-reflect.jar:/opt/modules/scala/scala-2.10.6-resources/scala-2.10.6/lib/scala-library.jar:/opt/modules/scala/scala-2.10.6-resources/scala-2.10.6/lib/scala-actors.jar:/opt/modules/scala/scala-2.10.6-resources/scala-2.10.6/lib/scala-actors-migration.jar:/opt/modules/scala/scala-2.10.6-resources/scala-2.10.6/lib/scala-swing.jar:/opt/modules/.m2/repository/log4j/log4j/1.2.12/log4j-1.2.12.jar:/opt/modules/.m2/repository/org/apache/spark/spark-core_2.10/1.6.0/spark-core_2.10-1.6.0.jar:/opt/modules/.m2/repository/org/apache/avro/avro-mapred/1.7.7/avro-mapred-1.7.7-hadoop2.jar:/opt/modules/.m2/repository/org/apache/avro/avro-ipc/1.7.7/avro-ipc-1.7.7.jar:/opt/modules/.m2/repository/org/apache/avro/avro-ipc/1.7.7/avro-ipc-1.7.7-tests.jar:/opt/modules/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/opt/modules/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/opt/modules/.m2/repository/com/twitter/chill_2.10/0.5.0/chill_2.10-0.5.0.jar:/opt/modules/.m2/repository/com/esotericsoftware/kryo/kryo/2.21/kryo-2.21.jar:/opt/modules/.m2/repository/com/esotericsoftware/reflectasm/reflectasm/1.07/reflectasm-1.07-shaded.jar:/opt/modules/.m2/repository/com/esotericsoftware/minlog/minlog/1.2/minlog-1.2.jar:/opt/modules/.m2/repository/org/objenesis/objenesis/1.2/objenesis-1.2.jar:/opt/modules/.m2/repository/com/twitter/chill-java/0.5.0/chill-java-0.5.0.jar:/opt/modules/.m2/repository/org/apache/xbean/xbean-asm5-shaded/4.4/xbean-asm5-shaded-4.4.jar:/opt/modules/.m2/repository/org/apache/spark/spark-launcher_2.10/1.6.0/spark-launcher_2.10-1.6.0.jar:/opt/modules/.m2/repository/org/apache/spark/spark-network-common_2.10/1.6.0/spark-network-common_2.10-1.6.0.jar:/opt/modules/.m2/repository/org/apache/spark/spark-network-shuffle_2.10/1.6.0/spark-network-shuffle_2.10-1.6.0.jar:/opt/modules/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/opt/modules/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.4.4/jackson-annotations-2.4.4.jar:/opt/modules/.m2/repository/org/apache/spark/spark-unsafe_2.10/1.6.0/spark-unsafe_2.10-1.6.0.jar:/opt/modules/.m2/repository/net/java/dev/jets3t/jets3t/0.7.1/jets3t-0.7.1.jar:/opt/modules/.m2/repository/commons-codec/commons-codec/1.3/commons-codec-1.3.jar:/opt/modules/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/opt/modules/.m2/repository/org/apache/curator/curator-recipes/2.4.0/curator-recipes-2.4.0.jar:/opt/modules/.m2/repository/org/apache/curator/curator-framework/2.4.0/curator-framework-2.4.0.jar:/opt/modules/.m2/repository/org/apache/zookeeper/zookeeper/3.4.5/zookeeper-3.4.5.jar:/opt/modules/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/opt/modules/.m2/repository/com/google/guava/guava/14.0.1/guava-14.0.1.jar:/opt/modules/.m2/repository/org/eclipse/jetty/orbit/javax.servlet/3.0.0.v201112011016/javax.servlet-3.0.0.v201112011016.jar:/opt/modules/.m2/repository/org/apache/commons/commons-lang3/3.3.2/commons-lang3-3.3.2.jar:/opt/modules/.m2/repository/org/apache/commons/commons-math3/3.4.1/commons-math3-3.4.1.jar:/opt/modules/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/opt/modules/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/opt/modules/.m2/repository/org/slf4j/jul-to-slf4j/1.7.10/jul-to-slf4j-1.7.10.jar:/opt/modules/.m2/repository/org/slf4j/jcl-over-slf4j/1.7.10/jcl-over-slf4j-1.7.10.jar:/opt/modules/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar:/opt/modules/.m2/repository/com/ning/compress-lzf/1.0.3/compress-lzf-1.0.3.jar:/opt/modules/.m2/repository/org/xerial/snappy/snappy-java/1.1.2/snappy-java-1.1.2.jar:/opt/modules/.m2/repository/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar:/opt/modules/.m2/repository/org/roaringbitmap/RoaringBitmap/0.5.11/RoaringBitmap-0.5.11.jar:/opt/modules/.m2/repository/commons-net/commons-net/2.2/commons-net-2.2.jar:/opt/modules/.m2/repository/com/typesafe/akka/akka-remote_2.10/2.3.11/akka-remote_2.10-2.3.11.jar:/opt/modules/.m2/repository/com/typesafe/akka/akka-actor_2.10/2.3.11/akka-actor_2.10-2.3.11.jar:/opt/modules/.m2/repository/com/typesafe/config/1.2.1/config-1.2.1.jar:/opt/modules/.m2/repository/io/netty/netty/3.8.0.Final/netty-3.8.0.Final.jar:/opt/modules/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/opt/modules/.m2/repository/org/uncommons/maths/uncommons-maths/1.2.2a/uncommons-maths-1.2.2a.jar:/opt/modules/.m2/repository/com/typesafe/akka/akka-slf4j_2.10/2.3.11/akka-slf4j_2.10-2.3.11.jar:/opt/modules/.m2/repository/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar:/opt/modules/.m2/repository/org/json4s/json4s-jackson_2.10/3.2.10/json4s-jackson_2.10-3.2.10.jar:/opt/modules/.m2/repository/org/json4s/json4s-core_2.10/3.2.10/json4s-core_2.10-3.2.10.jar:/opt/modules/.m2/repository/org/json4s/json4s-ast_2.10/3.2.10/json4s-ast_2.10-3.2.10.jar:/opt/modules/.m2/repository/org/scala-lang/scalap/2.10.0/scalap-2.10.0.jar:/opt/modules/.m2/repository/org/scala-lang/scala-compiler/2.10.0/scala-compiler-2.10.0.jar:/opt/modules/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/opt/modules/.m2/repository/asm/asm/3.1/asm-3.1.jar:/opt/modules/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/opt/modules/.m2/repository/org/apache/mesos/mesos/0.21.1/mesos-0.21.1-shaded-protobuf.jar:/opt/modules/.m2/repository/io/netty/netty-all/4.0.29.Final/netty-all-4.0.29.Final.jar:/opt/modules/.m2/repository/com/clearspring/analytics/stream/2.7.0/stream-2.7.0.jar:/opt/modules/.m2/repository/io/dropwizard/metrics/metrics-core/3.1.2/metrics-core-3.1.2.jar:/opt/modules/.m2/repository/io/dropwizard/metrics/metrics-jvm/3.1.2/metrics-jvm-3.1.2.jar:/opt/modules/.m2/repository/io/dropwizard/metrics/metrics-json/3.1.2/metrics-json-3.1.2.jar:/opt/modules/.m2/repository/io/dropwizard/metrics/metrics-graphite/3.1.2/metrics-graphite-3.1.2.jar:/opt/modules/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.4.4/jackson-databind-2.4.4.jar:/opt/modules/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.4.4/jackson-core-2.4.4.jar:/opt/modules/.m2/repository/com/fasterxml/jackson/module/jackson-module-scala_2.10/2.4.4/jackson-module-scala_2.10-2.4.4.jar:/opt/modules/.m2/repository/org/scala-lang/scala-reflect/2.10.4/scala-reflect-2.10.4.jar:/opt/modules/.m2/repository/com/thoughtworks/paranamer/paranamer/2.6/paranamer-2.6.jar:/opt/modules/.m2/repository/org/apache/ivy/ivy/2.4.0/ivy-2.4.0.jar:/opt/modules/.m2/repository/oro/oro/2.0.8/oro-2.0.8.jar:/opt/modules/.m2/repository/org/tachyonproject/tachyon-client/0.8.2/tachyon-client-0.8.2.jar:/opt/modules/.m2/repository/commons-lang/commons-lang/2.4/commons-lang-2.4.jar:/opt/modules/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/opt/modules/.m2/repository/org/tachyonproject/tachyon-underfs-hdfs/0.8.2/tachyon-underfs-hdfs-0.8.2.jar:/opt/modules/.m2/repository/org/tachyonproject/tachyon-underfs-s3/0.8.2/tachyon-underfs-s3-0.8.2.jar:/opt/modules/.m2/repository/org/tachyonproject/tachyon-underfs-local/0.8.2/tachyon-underfs-local-0.8.2.jar:/opt/modules/.m2/repository/net/razorvine/pyrolite/4.9/pyrolite-4.9.jar:/opt/modules/.m2/repository/net/sf/py4j/py4j/0.9/py4j-0.9.jar:/opt/modules/.m2/repository/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-client/2.6.0/hadoop-client-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-common/2.6.0/hadoop-common-2.6.0.jar:/opt/modules/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/opt/modules/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/opt/modules/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/opt/modules/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/opt/modules/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/opt/modules/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/opt/modules/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/opt/modules/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/opt/modules/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/opt/modules/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-auth/2.6.0/hadoop-auth-2.6.0.jar:/opt/modules/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/opt/modules/.m2/repository/org/apache/httpcomponents/httpcore/4.2.4/httpcore-4.2.4.jar:/opt/modules/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/opt/modules/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/opt/modules/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/opt/modules/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/opt/modules/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/opt/modules/.m2/repository/org/htrace/htrace-core/3.0.4/htrace-core-3.0.4.jar:/opt/modules/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/opt/modules/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.6.0/hadoop-hdfs-2.6.0.jar:/opt/modules/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/opt/modules/.m2/repository/xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar:/opt/modules/.m2/repository/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-app/2.6.0/hadoop-mapreduce-client-app-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/2.6.0/hadoop-mapreduce-client-common-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-yarn-client/2.6.0/hadoop-yarn-client-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-yarn-server-common/2.6.0/hadoop-yarn-server-common-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.6.0/hadoop-mapreduce-client-shuffle-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-yarn-api/2.6.0/hadoop-yarn-api-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0/hadoop-mapreduce-client-core-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-yarn-common/2.6.0/hadoop-yarn-common-2.6.0.jar:/opt/modules/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/opt/modules/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/opt/modules/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/opt/modules/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/opt/modules/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/opt/modules/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/opt/modules/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.6.0/hadoop-mapreduce-client-jobclient-2.6.0.jar:/opt/modules/.m2/repository/org/apache/hadoop/hadoop-annotations/2.6.0/hadoop-annotations-2.6.0.jar:/Applications/IntelliJ IDEA 15.app/Contents/lib/idea_rt.jar" com.intellij.rt.execution.application.AppMain net.yiup.modules.common.bigdata.spark.quickstart.QuickStart Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 16/01/15 11:43:29 INFO SparkContext: Running Spark version 1.6.0 16/01/15 11:43:29 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 16/01/15 11:43:29 WARN Utils: Your hostname, liuwen.mac resolves to a loopback address: 127.0.0.1; using 192.168.0.101 instead (on interface en0) 16/01/15 11:43:29 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 16/01/15 11:43:29 INFO SecurityManager: Changing view acls to: hadoop 16/01/15 11:43:29 INFO SecurityManager: Changing modify acls to: hadoop 16/01/15 11:43:29 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(hadoop); users with modify permissions: Set(hadoop) 16/01/15 11:43:30 INFO Utils: Successfully started service 'sparkDriver' on port 52439. 16/01/15 11:43:30 INFO Slf4jLogger: Slf4jLogger started 16/01/15 11:43:30 INFO Remoting: Starting remoting 16/01/15 11:43:30 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@192.168.0.101:52440] 16/01/15 11:43:30 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 52440. 16/01/15 11:43:30 INFO SparkEnv: Registering MapOutputTracker 16/01/15 11:43:30 INFO SparkEnv: Registering BlockManagerMaster 16/01/15 11:43:30 INFO DiskBlockManager: Created local directory at /private/var/folders/qt/2fqvdkb15zn9gfcb6h07rsqm0000gp/T/blockmgr-1f22bd24-17d5-4e96-8854-5d4f3662ff11 16/01/15 11:43:30 INFO MemoryStore: MemoryStore started with capacity 2.4 GB 16/01/15 11:43:30 INFO SparkEnv: Registering OutputCommitCoordinator 16/01/15 11:43:30 INFO Utils: Successfully started service 'SparkUI' on port 4040. 16/01/15 11:43:30 INFO SparkUI: Started SparkUI at http://192.168.0.101:4040 16/01/15 11:43:30 INFO HttpFileServer: HTTP File server directory is /private/var/folders/qt/2fqvdkb15zn9gfcb6h07rsqm0000gp/T/spark-14cc98b8-cd91-49ed-8a0e-9fa7f924a02c/httpd-d61443d2-dd47-42dd-9544-f571a60e42b5 16/01/15 11:43:30 INFO HttpServer: Starting HTTP Server 16/01/15 11:43:30 INFO Utils: Successfully started service 'HTTP file server' on port 52441. 16/01/15 11:43:30 INFO SparkContext: Added JAR /Users/hadoop/workspace/bigdata/all_frame_intellij/Spark1.6.0-practice-maven/target/Spark1.6.0-practice-maven-1.0.jar at http://192.168.0.101:52441/jars/Spark1.6.0-practice-maven-1.0.jar with timestamp 1452829410966 16/01/15 11:43:31 INFO AppClient$ClientEndpoint: Connecting to master spark://S20:7077... 16/01/15 11:43:34 INFO SparkDeploySchedulerBackend: Connected to Spark cluster with app ID app-20160114194334-0010 16/01/15 11:43:34 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52443. 16/01/15 11:43:34 INFO NettyBlockTransferService: Server created on 52443 16/01/15 11:43:34 INFO BlockManagerMaster: Trying to register BlockManager 16/01/15 11:43:34 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.0.101:52443 with 2.4 GB RAM, BlockManagerId(driver, 192.168.0.101, 52443) 16/01/15 11:43:34 INFO BlockManagerMaster: Registered BlockManager 16/01/15 11:43:34 INFO SparkDeploySchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.0 16/01/15 11:43:35 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 127.4 KB, free 127.4 KB) 16/01/15 11:43:35 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 141.3 KB) 16/01/15 11:43:35 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.0.101:52443 (size: 13.9 KB, free: 2.4 GB) 16/01/15 11:43:35 INFO SparkContext: Created broadcast 0 from textFile at QuickStart.scala:15 16/01/15 11:43:35 INFO FileInputFormat: Total input paths to process : 1 16/01/15 11:43:35 INFO SparkContext: Starting job: reduce at QuickStart.scala:20 16/01/15 11:43:35 INFO DAGScheduler: Got job 0 (reduce at QuickStart.scala:20) with 1 output partitions 16/01/15 11:43:35 INFO DAGScheduler: Final stage: ResultStage 0 (reduce at QuickStart.scala:20) 16/01/15 11:43:35 INFO DAGScheduler: Parents of final stage: List() 16/01/15 11:43:35 INFO DAGScheduler: Missing parents: List() 16/01/15 11:43:35 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[2] at map at QuickStart.scala:20), which has no missing parents 16/01/15 11:43:35 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 144.7 KB) 16/01/15 11:43:35 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2034.0 B, free 146.7 KB) 16/01/15 11:43:35 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.0.101:52443 (size: 2034.0 B, free: 2.4 GB) 16/01/15 11:43:35 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1006 16/01/15 11:43:35 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at QuickStart.scala:20) 16/01/15 11:43:35 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks 16/01/15 11:43:50 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources 16/01/15 11:44:05 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources 16/01/15 11:44:20 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources

harlixxy / x-x-x