search

scalar-labs / scalar-jepsen

Jepsen tests for ScalarDB and ScalarDL
45 stars 6 forks source link

Scalar DL test exceptions out #13

Closed craigpastro closed 5 years ago

craigpastro commented 5 years ago

I tried to run the scalardl tests with

lein run test --test all --time-limit 300

which resulted in the following exception. Should we include ledger.tar in resources?

...
INFO [2019-10-09 00:23:58,888] jepsen node n1 - cassandra.core n1 has stopped Cassandra
INFO [2019-10-09 00:23:58,888] jepsen node n1 - cassandra.core n1 deleting data files
INFO [2019-10-09 00:23:59,092] jepsen node n2 - cassandra.core n2 has stopped Cassandra
INFO [2019-10-09 00:23:59,093] jepsen node n2 - cassandra.core n2 deleting data files
WARN [2019-10-09 00:23:59,311] main - jepsen.core Test crashed!
java.util.concurrent.ExecutionException: java.io.FileNotFoundException: resources/ledger.tar (No such file or directory)
    at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[na:1.8.0_222]
    at java.util.concurrent.FutureTask.get(FutureTask.java:192) ~[na:1.8.0_222]
    at clojure.core$deref_future.invokeStatic(core.clj:2300) ~[clojure-1.10.0.jar:na]
    at clojure.core$future_call$reify__8439.deref(core.clj:6974) ~[clojure-1.10.0.jar:na]
    at clojure.core$deref.invokeStatic(core.clj:2320) ~[clojure-1.10.0.jar:na]
    at clojure.core$deref.invoke(core.clj:2306) ~[clojure-1.10.0.jar:na]
    at clojure.core$map$fn__5851.invoke(core.clj:2753) ~[clojure-1.10.0.jar:na]
    at clojure.lang.LazySeq.sval(LazySeq.java:42) ~[clojure-1.10.0.jar:na]
    at clojure.lang.LazySeq.seq(LazySeq.java:51) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RT.seq(RT.java:531) ~[clojure-1.10.0.jar:na]
    at clojure.core$seq__5387.invokeStatic(core.clj:137) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$seq_reduce.invokeStatic(protocols.clj:24) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$fn__8131.invokeStatic(protocols.clj:75) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$fn__8131.invoke(protocols.clj:75) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$fn__8073$G__8068__8086.invoke(protocols.clj:13) ~[clojure-1.10.0.jar:na]
    at clojure.core$reduce.invokeStatic(core.clj:6828) ~[clojure-1.10.0.jar:na]
    at clojure.core$into.invokeStatic(core.clj:6895) ~[clojure-1.10.0.jar:na]
    at clojure.core$into.invoke(core.clj:6887) ~[clojure-1.10.0.jar:na]
    at jepsen.control$on_nodes.invokeStatic(control.clj:385) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$on_nodes.invoke(control.clj:369) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$on_nodes.invokeStatic(control.clj:374) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$on_nodes.invoke(control.clj:369) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$cycle_BANG_$fn__2987.invoke(db.clj:45) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$cycle_BANG_.invokeStatic(db.clj:42) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$cycle_BANG_.invoke(db.clj:28) ~[jepsen-0.1.13.jar:na]
    at jepsen.core$run_BANG_$fn__5703.invoke(core.clj:550) ~[jepsen-0.1.13.jar:na]
    at jepsen.core$run_BANG_.invokeStatic(core.clj:538) [jepsen-0.1.13.jar:na]
    at jepsen.core$run_BANG_.invoke(core.clj:467) [jepsen-0.1.13.jar:na]
    at scalardl.runner$test_cmd$fn__415.invoke(runner.clj:81) [classes/:na]
    at jepsen.cli$run_BANG_.invokeStatic(cli.clj:299) [jepsen-0.1.13.jar:na]
    at jepsen.cli$run_BANG_.invoke(cli.clj:229) [jepsen-0.1.13.jar:na]
    at scalardl.runner$_main.invokeStatic(runner.clj:91) [classes/:na]
    at scalardl.runner$_main.doInvoke(runner.clj:89) [classes/:na]
    at clojure.lang.RestFn.invoke(RestFn.java:551) [clojure-1.10.0.jar:na]
    at clojure.lang.Var.invoke(Var.java:424) [clojure-1.10.0.jar:na]
    at user$eval140.invokeStatic(form-init5207380707746866766.clj:1) [na:na]
    at user$eval140.invoke(form-init5207380707746866766.clj:1) [na:na]
    at clojure.lang.Compiler.eval(Compiler.java:7176) [clojure-1.10.0.jar:na]
    at clojure.lang.Compiler.eval(Compiler.java:7166) [clojure-1.10.0.jar:na]
    at clojure.lang.Compiler.load(Compiler.java:7635) [clojure-1.10.0.jar:na]
    at clojure.lang.Compiler.loadFile(Compiler.java:7573) [clojure-1.10.0.jar:na]
    at clojure.main$load_script.invokeStatic(main.clj:452) [clojure-1.10.0.jar:na]
    at clojure.main$init_opt.invokeStatic(main.clj:454) [clojure-1.10.0.jar:na]
    at clojure.main$init_opt.invoke(main.clj:454) [clojure-1.10.0.jar:na]
    at clojure.main$initialize.invokeStatic(main.clj:485) [clojure-1.10.0.jar:na]
    at clojure.main$null_opt.invokeStatic(main.clj:519) [clojure-1.10.0.jar:na]
    at clojure.main$null_opt.invoke(main.clj:516) [clojure-1.10.0.jar:na]
    at clojure.main$main.invokeStatic(main.clj:598) [clojure-1.10.0.jar:na]
    at clojure.main$main.doInvoke(main.clj:561) [clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.applyTo(RestFn.java:137) [clojure-1.10.0.jar:na]
    at clojure.lang.Var.applyTo(Var.java:705) [clojure-1.10.0.jar:na]
    at clojure.main.main(main.java:37) [clojure-1.10.0.jar:na]
Caused by: java.io.FileNotFoundException: resources/ledger.tar (No such file or directory)
    at java.io.FileInputStream.open0(Native Method) ~[na:1.8.0_222]
    at java.io.FileInputStream.open(FileInputStream.java:195) ~[na:1.8.0_222]
    at java.io.FileInputStream.<init>(FileInputStream.java:138) ~[na:1.8.0_222]
    at clojure.java.io$fn__11512.invokeStatic(io.clj:346) ~[clojure-1.10.0.jar:na]
    at clojure.java.io$fn__11512.invoke(io.clj:346) ~[clojure-1.10.0.jar:na]
    at clojure.lang.MultiFn.invoke(MultiFn.java:239) ~[clojure-1.10.0.jar:na]
    at clojure.java.io$copy.invokeStatic(io.clj:406) ~[clojure-1.10.0.jar:na]
    at clojure.java.io$copy.doInvoke(io.clj:391) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:464) [clojure-1.10.0.jar:na]
    at clj_ssh.ssh$scp_copy_file.invokeStatic(ssh.clj:895) ~[jepsen-0.1.13.jar:na]
    at clj_ssh.ssh$scp_copy_file.invoke(ssh.clj:882) ~[jepsen-0.1.13.jar:na]
    at clj_ssh.ssh$scp_to.invokeStatic(ssh.clj:1037) ~[jepsen-0.1.13.jar:na]
    at clj_ssh.ssh$scp_to.doInvoke(ssh.clj:1002) ~[jepsen-0.1.13.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:445) [clojure-1.10.0.jar:na]
    at clojure.lang.AFn.applyToHelper(AFn.java:160) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.applyTo(RestFn.java:132) [clojure-1.10.0.jar:na]
    at clojure.core$apply.invokeStatic(core.clj:671) ~[clojure-1.10.0.jar:na]
    at clojure.core$apply.invoke(core.clj:660) ~[clojure-1.10.0.jar:na]
    at jepsen.control$upload$fn__2843.invoke(control.clj:200) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$upload.invokeStatic(control.clj:200) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$upload.doInvoke(control.clj:196) ~[jepsen-0.1.13.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:421) [clojure-1.10.0.jar:na]
    at scalardl.core$install_server_BANG_.invokeStatic(core.clj:112) ~[classes/:na]
    at scalardl.core$install_server_BANG_.invoke(core.clj:107) ~[classes/:na]
    at scalardl.core$db$reify__252.setup_BANG_(core.clj:136) ~[classes/:na]
    at jepsen.db$fn__2929$G__2908__2933.invoke(db.clj:8) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$fn__2929$G__2907__2938.invoke(db.clj:8) ~[jepsen-0.1.13.jar:na]
    at clojure.core$partial$fn__5824.invoke(core.clj:2625) ~[clojure-1.10.0.jar:na]
    at jepsen.control$on_nodes$fn__2893.invoke(control.clj:384) ~[jepsen-0.1.13.jar:na]
    at clojure.lang.AFn.applyToHelper(AFn.java:154) ~[clojure-1.10.0.jar:na]
    at clojure.lang.AFn.applyTo(AFn.java:144) ~[clojure-1.10.0.jar:na]
    at clojure.core$apply.invokeStatic(core.clj:665) ~[clojure-1.10.0.jar:na]
    at clojure.core$with_bindings_STAR_.invokeStatic(core.clj:1973) ~[clojure-1.10.0.jar:na]
    at clojure.core$with_bindings_STAR_.doInvoke(core.clj:1973) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.applyTo(RestFn.java:142) [clojure-1.10.0.jar:na]
    at clojure.core$apply.invokeStatic(core.clj:669) ~[clojure-1.10.0.jar:na]
    at clojure.core$bound_fn_STAR_$fn__5734.doInvoke(core.clj:2003) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:408) [clojure-1.10.0.jar:na]
    at jepsen.util$real_pmap$launcher__2044$fn__2045.invoke(util.clj:50) ~[jepsen-0.1.13.jar:na]
    at clojure.core$binding_conveyor_fn$fn__5739.invoke(core.clj:2030) ~[clojure-1.10.0.jar:na]
    at clojure.lang.AFn.call(AFn.java:18) ~[clojure-1.10.0.jar:na]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_222]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_222]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_222]
    at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_222]
ERROR [2019-10-09 00:23:59,319] main - jepsen.cli Oh jeez, I'm sorry, Jepsen broke. Here's why:
java.util.concurrent.ExecutionException: java.io.FileNotFoundException: resources/ledger.tar (No such file or directory)
    at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[na:1.8.0_222]
    at java.util.concurrent.FutureTask.get(FutureTask.java:192) ~[na:1.8.0_222]
    at clojure.core$deref_future.invokeStatic(core.clj:2300) ~[clojure-1.10.0.jar:na]
    at clojure.core$future_call$reify__8439.deref(core.clj:6974) ~[clojure-1.10.0.jar:na]
    at clojure.core$deref.invokeStatic(core.clj:2320) ~[clojure-1.10.0.jar:na]
    at clojure.core$deref.invoke(core.clj:2306) ~[clojure-1.10.0.jar:na]
    at clojure.core$map$fn__5851.invoke(core.clj:2753) ~[clojure-1.10.0.jar:na]
    at clojure.lang.LazySeq.sval(LazySeq.java:42) ~[clojure-1.10.0.jar:na]
    at clojure.lang.LazySeq.seq(LazySeq.java:51) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RT.seq(RT.java:531) ~[clojure-1.10.0.jar:na]
    at clojure.core$seq__5387.invokeStatic(core.clj:137) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$seq_reduce.invokeStatic(protocols.clj:24) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$fn__8131.invokeStatic(protocols.clj:75) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$fn__8131.invoke(protocols.clj:75) ~[clojure-1.10.0.jar:na]
    at clojure.core.protocols$fn__8073$G__8068__8086.invoke(protocols.clj:13) ~[clojure-1.10.0.jar:na]
    at clojure.core$reduce.invokeStatic(core.clj:6828) ~[clojure-1.10.0.jar:na]
    at clojure.core$into.invokeStatic(core.clj:6895) ~[clojure-1.10.0.jar:na]
    at clojure.core$into.invoke(core.clj:6887) ~[clojure-1.10.0.jar:na]
    at jepsen.control$on_nodes.invokeStatic(control.clj:385) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$on_nodes.invoke(control.clj:369) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$on_nodes.invokeStatic(control.clj:374) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$on_nodes.invoke(control.clj:369) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$cycle_BANG_$fn__2987.invoke(db.clj:45) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$cycle_BANG_.invokeStatic(db.clj:42) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$cycle_BANG_.invoke(db.clj:28) ~[jepsen-0.1.13.jar:na]
    at jepsen.core$run_BANG_$fn__5703.invoke(core.clj:550) ~[jepsen-0.1.13.jar:na]
    at jepsen.core$run_BANG_.invokeStatic(core.clj:538) ~[jepsen-0.1.13.jar:na]
    at jepsen.core$run_BANG_.invoke(core.clj:467) ~[jepsen-0.1.13.jar:na]
    at scalardl.runner$test_cmd$fn__415.invoke(runner.clj:81) ~[classes/:na]
    at jepsen.cli$run_BANG_.invokeStatic(cli.clj:299) [jepsen-0.1.13.jar:na]
    at jepsen.cli$run_BANG_.invoke(cli.clj:229) [jepsen-0.1.13.jar:na]
    at scalardl.runner$_main.invokeStatic(runner.clj:91) [classes/:na]
    at scalardl.runner$_main.doInvoke(runner.clj:89) [classes/:na]
    at clojure.lang.RestFn.invoke(RestFn.java:551) [clojure-1.10.0.jar:na]
    at clojure.lang.Var.invoke(Var.java:424) [clojure-1.10.0.jar:na]
    at user$eval140.invokeStatic(form-init5207380707746866766.clj:1) [na:na]
    at user$eval140.invoke(form-init5207380707746866766.clj:1) [na:na]
    at clojure.lang.Compiler.eval(Compiler.java:7176) [clojure-1.10.0.jar:na]
    at clojure.lang.Compiler.eval(Compiler.java:7166) [clojure-1.10.0.jar:na]
    at clojure.lang.Compiler.load(Compiler.java:7635) [clojure-1.10.0.jar:na]
    at clojure.lang.Compiler.loadFile(Compiler.java:7573) [clojure-1.10.0.jar:na]
    at clojure.main$load_script.invokeStatic(main.clj:452) [clojure-1.10.0.jar:na]
    at clojure.main$init_opt.invokeStatic(main.clj:454) [clojure-1.10.0.jar:na]
    at clojure.main$init_opt.invoke(main.clj:454) [clojure-1.10.0.jar:na]
    at clojure.main$initialize.invokeStatic(main.clj:485) [clojure-1.10.0.jar:na]
    at clojure.main$null_opt.invokeStatic(main.clj:519) [clojure-1.10.0.jar:na]
    at clojure.main$null_opt.invoke(main.clj:516) [clojure-1.10.0.jar:na]
    at clojure.main$main.invokeStatic(main.clj:598) [clojure-1.10.0.jar:na]
    at clojure.main$main.doInvoke(main.clj:561) [clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.applyTo(RestFn.java:137) [clojure-1.10.0.jar:na]
    at clojure.lang.Var.applyTo(Var.java:705) [clojure-1.10.0.jar:na]
    at clojure.main.main(main.java:37) [clojure-1.10.0.jar:na]
Caused by: java.io.FileNotFoundException: resources/ledger.tar (No such file or directory)
    at java.io.FileInputStream.open0(Native Method) ~[na:1.8.0_222]
    at java.io.FileInputStream.open(FileInputStream.java:195) ~[na:1.8.0_222]
    at java.io.FileInputStream.<init>(FileInputStream.java:138) ~[na:1.8.0_222]
    at clojure.java.io$fn__11512.invokeStatic(io.clj:346) ~[clojure-1.10.0.jar:na]
    at clojure.java.io$fn__11512.invoke(io.clj:346) ~[clojure-1.10.0.jar:na]
    at clojure.lang.MultiFn.invoke(MultiFn.java:239) ~[clojure-1.10.0.jar:na]
    at clojure.java.io$copy.invokeStatic(io.clj:406) ~[clojure-1.10.0.jar:na]
    at clojure.java.io$copy.doInvoke(io.clj:391) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:464) [clojure-1.10.0.jar:na]
    at clj_ssh.ssh$scp_copy_file.invokeStatic(ssh.clj:895) ~[jepsen-0.1.13.jar:na]
    at clj_ssh.ssh$scp_copy_file.invoke(ssh.clj:882) ~[jepsen-0.1.13.jar:na]
    at clj_ssh.ssh$scp_to.invokeStatic(ssh.clj:1037) ~[jepsen-0.1.13.jar:na]
    at clj_ssh.ssh$scp_to.doInvoke(ssh.clj:1002) ~[jepsen-0.1.13.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:445) [clojure-1.10.0.jar:na]
    at clojure.lang.AFn.applyToHelper(AFn.java:160) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.applyTo(RestFn.java:132) [clojure-1.10.0.jar:na]
    at clojure.core$apply.invokeStatic(core.clj:671) ~[clojure-1.10.0.jar:na]
    at clojure.core$apply.invoke(core.clj:660) ~[clojure-1.10.0.jar:na]
    at jepsen.control$upload$fn__2843.invoke(control.clj:200) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$upload.invokeStatic(control.clj:200) ~[jepsen-0.1.13.jar:na]
    at jepsen.control$upload.doInvoke(control.clj:196) ~[jepsen-0.1.13.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:421) [clojure-1.10.0.jar:na]
    at scalardl.core$install_server_BANG_.invokeStatic(core.clj:112) ~[classes/:na]
    at scalardl.core$install_server_BANG_.invoke(core.clj:107) ~[classes/:na]
    at scalardl.core$db$reify__252.setup_BANG_(core.clj:136) ~[classes/:na]
    at jepsen.db$fn__2929$G__2908__2933.invoke(db.clj:8) ~[jepsen-0.1.13.jar:na]
    at jepsen.db$fn__2929$G__2907__2938.invoke(db.clj:8) ~[jepsen-0.1.13.jar:na]
    at clojure.core$partial$fn__5824.invoke(core.clj:2625) ~[clojure-1.10.0.jar:na]
    at jepsen.control$on_nodes$fn__2893.invoke(control.clj:384) ~[jepsen-0.1.13.jar:na]
    at clojure.lang.AFn.applyToHelper(AFn.java:154) ~[clojure-1.10.0.jar:na]
    at clojure.lang.AFn.applyTo(AFn.java:144) ~[clojure-1.10.0.jar:na]
    at clojure.core$apply.invokeStatic(core.clj:665) ~[clojure-1.10.0.jar:na]
    at clojure.core$with_bindings_STAR_.invokeStatic(core.clj:1973) ~[clojure-1.10.0.jar:na]
    at clojure.core$with_bindings_STAR_.doInvoke(core.clj:1973) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.applyTo(RestFn.java:142) [clojure-1.10.0.jar:na]
    at clojure.core$apply.invokeStatic(core.clj:669) ~[clojure-1.10.0.jar:na]
    at clojure.core$bound_fn_STAR_$fn__5734.doInvoke(core.clj:2003) ~[clojure-1.10.0.jar:na]
    at clojure.lang.RestFn.invoke(RestFn.java:408) [clojure-1.10.0.jar:na]
    at jepsen.util$real_pmap$launcher__2044$fn__2045.invoke(util.clj:50) ~[jepsen-0.1.13.jar:na]
    at clojure.core$binding_conveyor_fn$fn__5739.invoke(core.clj:2030) ~[clojure-1.10.0.jar:na]
    at clojure.lang.AFn.call(AFn.java:18) ~[clojure-1.10.0.jar:na]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_222]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_222]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_222]
    at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_222]
craigpastro commented 5 years ago

I see that we still don't want to expose ledger.tar to the public yet, so this is not really an issue. I will close it.