search

databricks / spark-perf

Performance tests for Apache Spark
Apache License 2.0
379 stars 203 forks source link

Traceback in hdfs-recovery #14

Open tsailiming opened 9 years ago

tsailiming commented 9 years ago 
--------------------------------------------------------------------
Java options: -Dspark.storage.memoryFraction=0.66 -Dspark.serializer=org.apache.spark.serializer.JavaSerializer -Dspark.executor.memory=9g -Dspark.executor.extraJavaOptions= -XX:+UseConcMarkSweepGC 
Options: hdfs-recovery --total-duration=60 --hdfs-url=hdfs://numaq1-1/test --batch-duration=5000 --records-per-file=10000 --file-cleaner-delay=300
--------------------------------------------------------------------
<snip>
14/10/16 10:45:26 INFO scheduler.JobGenerator: Started JobGenerator at 1413470730000 ms
14/10/16 10:45:26 INFO scheduler.JobScheduler: Started JobScheduler
Exception in thread "main" java.lang.AbstractMethodError
        at org.apache.spark.Logging$class.log(Logging.scala:52)
        at streaming.perf.util.FileGenerator.log(FileGenerator.scala:15)
        at org.apache.spark.Logging$class.logInfo(Logging.scala:59)
        at streaming.perf.util.FileGenerator.logInfo(FileGenerator.scala:15)
        at streaming.perf.util.FileGenerator.start(FileGenerator.scala:51)
        at streaming.perf.HdfsRecoveryTest.run(HdfsRecoveryTest.scala:61)
        at streaming.perf.TestRunner$.main(TestRunner.scala:18)
        at streaming.perf.TestRunner.main(TestRunner.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:328)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:75)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Exception in thread "Thread-101" java.lang.AbstractMethodError
        at org.apache.spark.Logging$class.log(Logging.scala:52)
        at streaming.perf.util.FileGenerator.log(FileGenerator.scala:15)
        at org.apache.spark.Logging$class.logInfo(Logging.scala:59)
        at streaming.perf.util.FileGenerator.logInfo(FileGenerator.scala:15)
        at streaming.perf.util.FileGenerator.streaming$perf$util$FileGenerator$$copyFile(FileGenerator.scala:108)
        at streaming.perf.util.FileGenerator$$anonfun$streaming$perf$util$FileGenerator$$generateFiles$1$$anonfun$apply$mcVI$sp$1.apply$mcVJ$sp(FileGenerator.scala:82)
        at streaming.perf.util.FileGenerator$$anonfun$streaming$perf$util$FileGenerator$$generateFiles$1$$anonfun$apply$mcVI$sp$1.apply(FileGenerator.scala:75)
        at streaming.perf.util.FileGenerator$$anonfun$streaming$perf$util$FileGenerator$$generateFiles$1$$anonfun$apply$mcVI$sp$1.apply(FileGenerator.scala:75)
        at scala.collection.immutable.NumericRange.foreach(NumericRange.scala:74)
        at streaming.perf.util.FileGenerator$$anonfun$streaming$perf$util$FileGenerator$$generateFiles$1.apply$mcVI$sp(FileGenerator.scala:75)
        at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
        at streaming.perf.util.FileGenerator.streaming$perf$util$FileGenerator$$generateFiles(FileGenerator.scala:73)
        at streaming.perf.util.FileGenerator$$anon$1.run(FileGenerator.scala:28)
tsailiming commented 9 years ago

Also, there's a bug in the config.py.template for hdfs-recovery. Should be caused by commit: e0ec3fad1063a54feabfb9a1a8a914cd17f5bc8c

-STREAMING_TESTS += [("basic", "streaming.perf.TestRunner basic",
-    SCALE_FACTOR, STREAMING_COMMON_JAVA_OPTS, STREAMING_COMMON_OPTS + streaming_batch_duration_opts(1000))] 
+STREAMING_TESTS += [("basic", "streaming.perf.TestRunner", SCALE_FACTOR,
+    STREAMING_COMMON_JAVA_OPTS, [ConstantOption("basic")] + STREAMING_COMMON_OPTS + streaming_batch_duration_opts(1000))] 

-STREAMING_TESTS += [("state-by-key", "streaming.perf.TestRunner state-by-key",
-    SCALE_FACTOR, STREAMING_COMMON_JAVA_OPTS, STREAMING_KEY_VAL_TEST_OPTS)]
+STREAMING_TESTS += [("state-by-key", "streaming.perf.TestRunner", SCALE_FACTOR,
+    STREAMING_COMMON_JAVA_OPTS, [ConstantOption("state-by-key")] + STREAMING_KEY_VAL_TEST_OPTS)]

-STREAMING_TESTS += [("group-by-key-and-window", "streaming.perf.TestRunner group-by-key-and-window",
-    SCALE_FACTOR, STREAMING_COMMON_JAVA_OPTS, STREAMING_KEY_VAL_TEST_OPTS + streaming_window_duration_opts(10000) )]
+STREAMING_TESTS += [("group-by-key-and-window", "streaming.perf.TestRunner", SCALE_FACTOR,
+    STREAMING_COMMON_JAVA_OPTS, [ConstantOption("group-by-key-and-window")] + STREAMING_KEY_VAL_TEST_OPTS + streaming_window_duration_opts(10000) )]

-STREAMING_TESTS += [("reduce-by-key-and-window", "streaming.perf.TestRunner reduce-by-key-and-window",
-    SCALE_FACTOR, STREAMING_COMMON_JAVA_OPTS, STREAMING_KEY_VAL_TEST_OPTS + streaming_window_duration_opts(10000) )]
+STREAMING_TESTS += [("reduce-by-key-and-window", "streaming.perf.TestRunner", SCALE_FACTOR,
+    STREAMING_COMMON_JAVA_OPTS, [ConstantOption("reduce-by-key-and-window")] + STREAMING_KEY_VAL_TEST_OPTS + streaming_window_duration_opts(10000) )]

-STREAMING_TESTS += [("hdfs-recovery", "streaming.perf.TestRunner hdfs-recovery",
-    SCALE_FACTOR, STREAMING_COMMON_JAVA_OPTS, STREAMING_HDFS_RECOVERY_TEST_OPTS)]
+ STREAMING_TESTS += [("hdfs-recovery", "streaming.perf.TestRunner hdfs-recovery", SCALE_FACTOR,
+    STREAMING_COMMON_JAVA_OPTS, [ConstantOption("hdfs-recovery")] + STREAMING_HDFS_RECOVERY_TEST_OPTS)]

Shouldn't it be STREAMING_TESTS += [("hdfs-recovery", "streaming.perf.TestRunner",