search

hibari / gdss-admin

http://hibari.github.com/hibari-doc/
Other
2 stars 0 forks source link

QuickCheck squorum_eqc_tests:prop_squorum1 fails with missing key-value after crashing majority of squorum bricks #10

Open tatsuya6502 opened 9 years ago

tatsuya6502 commented 9 years ago

After resolving quorum_error, prop_squorum1 often generates the following counter example. It failed because it can't find the key value at step 5.

I believe this is a test model bug. Crashing and restarting majority of squorum bricks without repair will cause data loss. But the current expectation doesn't match to this behavior.

[{set,{var,1},
      {call,squorum_eqc_tests,set_num_bricks,
            [{var,env_bricks},{var,env_reordered}]}},
 {set,{var,2},
      {call,squorum_eqc_tests,squorum_set,
            [{var,env_bricks},
             <<"/foo/bar/0002">>,<<>>,1427205906396945024,
             [value_in_ram]]}},
 {set,{var,3},{call,squorum_eqc_tests,crash_brick,[{var,env_reordered},1]}},
 {set,{var,4},{call,squorum_eqc_tests,crash_brick,[{var,env_reordered},2]}},
 {set,{var,5},
      {call,squorum_eqc_tests,squorum_get,
            [{var,env_bricks},<<"/foo/bar/0002">>,[]]}}]
Env = [{env_reordered,[{squorum_test1,'hibariclient@127.0.0.1'},
                       {squorum_test2,'hibariclient@127.0.0.1'}]},
       {env_bricks,[{squorum_test1,'hibariclient@127.0.0.1'},
                    {squorum_test2,'hibariclient@127.0.0.1'}]}]
S = {state,6,
           [{<<"/foo/bar/0001">>,[]},
            {<<"/foo/bar/0002">>,[]},
            {<<"/foo/bar/0003">>,[]},
            {<<"/foo/bar/0004">>,[]}],
           2,
           [{squorum_test1,'hibariclient@127.0.0.1'},
            {squorum_test2,'hibariclient@127.0.0.1'}],
           [{squorum_test1,'hibariclient@127.0.0.1'},
            {squorum_test2,'hibariclient@127.0.0.1'}]}
R = {postcondition,false}
Hist = [{eqc_statem_history,
            {state,1,
                [{<<"/foo/bar/0001">>,[]},
                 {<<"/foo/bar/0002">>,[]},
                 {<<"/foo/bar/0003">>,[]},
                 {<<"/foo/bar/0004">>,[]}],
                undefined,
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'},
                 {squorum_test3,'hibariclient@127.0.0.1'},
                 {squorum_test4,'hibariclient@127.0.0.1'},
                 {squorum_test5,'hibariclient@127.0.0.1'},
                 {squorum_test6,'hibariclient@127.0.0.1'},
                 {squorum_test7,'hibariclient@127.0.0.1'}],
                undefined},
            {call,squorum_eqc_tests,set_num_bricks,
                [[{squorum_test1,'hibariclient@127.0.0.1'},
                  {squorum_test2,'hibariclient@127.0.0.1'}],
                 [{squorum_test1,'hibariclient@127.0.0.1'},
                  {squorum_test2,'hibariclient@127.0.0.1'}]]},
            [],
            {normal,ok}},
        {eqc_statem_history,
            {state,2,
                [{<<"/foo/bar/0001">>,[]},
                 {<<"/foo/bar/0002">>,[]},
                 {<<"/foo/bar/0003">>,[]},
                 {<<"/foo/bar/0004">>,[]}],
                2,
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}],
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}]},
            {call,squorum_eqc_tests,squorum_set,
                [[{squorum_test1,'hibariclient@127.0.0.1'},
                  {squorum_test2,'hibariclient@127.0.0.1'}],
                 <<"/foo/bar/0002">>,<<>>,1427205906396945024,
                 [value_in_ram]]},
            [],
            {normal,{ok,1427205912889194}}},
        {eqc_statem_history,
            {state,3,
                [{<<"/foo/bar/0001">>,[]},
                 {<<"/foo/bar/0002">>,[<<>>]},
                 {<<"/foo/bar/0003">>,[]},
                 {<<"/foo/bar/0004">>,[]}],
                2,
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}],
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}]},
            {call,squorum_eqc_tests,crash_brick,
                [[{squorum_test1,'hibariclient@127.0.0.1'},
                  {squorum_test2,'hibariclient@127.0.0.1'}],
                 1]},
            [],
            {normal,ok}},
        {eqc_statem_history,
            {state,4,
                [{<<"/foo/bar/0001">>,[]},
                 {<<"/foo/bar/0002">>,[<<>>]},
                 {<<"/foo/bar/0003">>,[]},
                 {<<"/foo/bar/0004">>,[]}],
                2,
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}],
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}]},
            {call,squorum_eqc_tests,crash_brick,
                [[{squorum_test1,'hibariclient@127.0.0.1'},
                  {squorum_test2,'hibariclient@127.0.0.1'}],
                 2]},
            [],
            {normal,ok}},
        {eqc_statem_history,
            {state,5,
                [{<<"/foo/bar/0001">>,[]},
                 {<<"/foo/bar/0002">>,[<<>>]},
                 {<<"/foo/bar/0003">>,[]},
                 {<<"/foo/bar/0004">>,[]}],
                2,
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}],
                [{squorum_test1,'hibariclient@127.0.0.1'},
                 {squorum_test2,'hibariclient@127.0.0.1'}]},
            {call,squorum_eqc_tests,squorum_get,
                [[{squorum_test1,'hibariclient@127.0.0.1'},
                  {squorum_test2,'hibariclient@127.0.0.1'}],
                 <<"/foo/bar/0002">>,[]]},
            [],
            {normal,key_not_exist}}]
false