search

dashbitco / flow

Computational parallel flows on top of GenStage
https://hexdocs.pm/flow
1.55k stars 90 forks source link

Race condition in `bounded_join` for large subscribe messages #28

Closed asonge closed 7 years ago

asonge commented 7 years ago

There's some kind of race condition within bounded_join. When the demand causes sufficiently large messages to be sent, the join direction being matched against is nil. I'm currently working around this by shrinking my demand.

a = Flow.from_enumerable(Stream.zip(1..100_000, 1..100_000))
b = Flow.from_enumerable(Stream.zip(1..100_000, -1..-100_000))
Flow.bounded_join(:left_outer, a, b,
                  &elem(&1,0), &elem(&1,0),
                  fn
                    {k,v1}, {k,v2} -> {k, {v1,v2}}
                    {k,v1}, nil -> {k, {v1,nil}}
                  end, min_demand: 10_000, max_demand: 20_000)
|> Stream.run()

Exception details:

[error] GenServer #PID<0.28590.0> terminating
** (FunctionClauseError) no function clause matching in Flow.Materialize.dispatch_join/6
    (flow) lib/flow/materialize.ex:328: Flow.Materialize.dispatch_join(
      [{98431, {98431, 98431}}, {98439, {98439, 98439}}, {98442, {98442, 98442}}, {98443, {98443, 98443}}, {98446, {98446, 98446}}, {98449, {98449, 98449}}, {98454, {98454, 98454}}, {98457, {98457, 98457}}, {98458, {98458, 98458}}, {98461, {98461, 98461}}, {98465, {98465, 98465}}, {98466, {98466, 98466}}, {98467, {98467, 98467}}, {98470, {98470, 98470}}, {98472, {98472, 98472}}, {98474, {98474, 98474}}, {98475, {98475, 98475}}, {98476, {98476, 98476}}, {98484, {98484, 98484}}, {98485, {98485, 98485}}, {98488, {98488, 98488}}, {98495, {98495, 98495}}, {98501, {98501, 98501}}, {98502, {98502, 98502}}, {98516, {98516, 98516}}, {98519, {98519, 98519}}, {98524, {98524, 98524}}, {98526, {98526, 98526}}, {98531, {98531, 98531}}, {98548, {98548, 98548}}, {98552, {98552, 98552}}, {98553, {98553, 98553}}, {98561, {98561, 98561}}, {98566, {98566, 98566}}, {98571, {98571, 98571}}, {98574, {98574, 98574}}, {98576, {98576, 98576}}, {98585, {98585, 98585}}, {98599, {98599, 98599}}, {98603, {98603, 98603}}, {98614, {98614, 98614}}, {98615, {98615, 98615}}, {98619, {98619, 98619}}, {98621, {98621, 98621}}, {98632, {98632, 98632}}, {98634, {98634, 98634}}, {98639, {98639, 98639}}, {98643, {98643, ...}}, {98645, ...}, {...}, ...],
      nil,
      %{60829 => [{60829, 60829}], 12995 => [{12995, 12995}], 89750 => [{89750, 89750}], 42408 => [{42408, 42408}], 79313 => [{79313, 79313}], 59922 => [{59922, 59922}], 51560 => [{51560, 51560}], 6653 => [{6653, 6653}], 82555 => [{82555, 82555}], 40746 => [{40746, 40746}], 33823 => [{33823, 33823}], 94037 => [{94037, 94037}], 46309 => [{46309, 46309}], 81817 => [{81817, 81817}], 15437 => [{15437, 15437}], 34319 => [{34319, 34319}], 6749 => [{6749, 6749}], 62448 => [{62448, 62448}], 79870 => [{79870, 79870}], 90980 => [{90980, 90980}], 52195 => [{52195, 52195}], 34512 => [{34512, 34512}], 19510 => [{19510, 19510}], 44436 => [{44436, 44436}], 75113 => [{75113, 75113}], 81119 => [{81119, 81119}], 8441 => [{8441, 8441}], 96376 => [{96376, 96376}], 76413 => [{76413, 76413}], 32156 => [{32156, 32156}], 43416 => [{43416, 43416}], 86876 => [{86876, 86876}], 61486 => [{61486, 61486}], 12163 => [{12163, 12163}], 40610 => [{40610, 40610}], 87629 => [{87629, 87629}], 33708 => [{33708, 33708}], 47217 => [{47217, 47217}], 77706 => [{77706, 77706}], 49309 => [{49309, 49309}], 5934 => [{5934, 5934}], 59657 => [{59657, 59657}], 61379 => [{61379, 61379}], 66854 => [{66854, 66854}], 21938 => [{21938, 21938}], 38839 => [{38839, 38839}], 76035 => [{76035, 76035}], 75873 => [{75873, 75873}], 73528 => [{73528, ...}], 4969 => [...], ...},
      %{60829 => [{60829, -60829}], 12995 => [{12995, -12995}], 42408 => [{42408, -42408}], 79313 => [{79313, -79313}], 59922 => [{59922, -59922}], 51560 => [{51560, -51560}], 6653 => [{6653, -6653}], 82555 => [{82555, -82555}], 40746 => [{40746, -40746}], 33823 => [{33823, -33823}], 46309 => [{46309, -46309}], 81817 => [{81817, -81817}], 15437 => [{15437, -15437}], 34319 => [{34319, -34319}], 6749 => [{6749, -6749}], 62448 => [{62448, -62448}], 79870 => [{79870, -79870}], 52195 => [{52195, -52195}], 34512 => [{34512, -34512}], 19510 => [{19510, -19510}], 44436 => [{44436, -44436}], 75113 => [{75113, -75113}], 81119 => [{81119, -81119}], 8441 => [{8441, -8441}], 76413 => [{76413, -76413}], 32156 => [{32156, -32156}], 43416 => [{43416, -43416}], 61486 => [{61486, -61486}], 12163 => [{12163, -12163}], 40610 => [{40610, -40610}], 33708 => [{33708, -33708}], 47217 => [{47217, -47217}], 77706 => [{77706, -77706}], 49309 => [{49309, -49309}], 5934 => [{5934, -5934}], 59657 => [{59657, -59657}], 61379 => [{61379, -61379}], 66854 => [{66854, -66854}], 21938 => [{21938, -21938}], 38839 => [{38839, -38839}], 76035 => [{76035, -76035}], 75873 => [{75873, -75873}], 73528 => [{73528, -73528}], 4969 => [{4969, -4969}], 16129 => [{16129, -16129}], 75116 => [{75116, -75116}], 45250 => [{45250, -45250}], 74081 => [{74081, -74081}], 15093 => [{15093, ...}], 45659 => [...], ...},
      #Function<7.85826293/2 in Wasatch.Jobs.JoinCatsTask.test_run/0>,
      []
    )
    (flow) lib/flow/materialize.ex:286: anonymous fn/6 in Flow.Materialize.join_ops/5
    (flow) lib/flow/map_reducer.ex:49: Flow.MapReducer.handle_events/3
    (gen_stage) lib/gen_stage.ex:2502: GenStage.consumer_dispatch/7
    (gen_stage) lib/gen_stage.ex:2618: GenStage.take_pc_events/3
Last message: {:"$gen_producer", {#PID<0.326.0>, {#Reference<0.0.2.6935>, #Reference<0.0.2.6939>}}, {:ask, 680}}
State: {%{#Reference<0.0.2.6920> => nil}, %{done?: false, producers: %{#Reference<0.0.2.6920> => #PID<0.28587.0>}, trigger: #Function<2.31322697/4 in Flow.Window.Global.materialize/5>}, {2, 4}, {%{60829 => [{60829, 60829}], 12995 => [{12995, 12995}], 89750 => [{89750, 89750}], 42408 => [{42408, 42408}], 79313 => [{79313, 79313}], 59922 => [{59922, 59922}], 51560 => [{51560, 51560}], 6653 => [{6653, 6653}], 82555 => [{82555, 82555}], 40746 => [{40746, 40746}], 33823 => [{33823, 33823}], 94037 => [{94037, 94037}], 46309 => [{46309, 46309}], 81817 => [{81817, 81817}], 15437 => [{15437, 15437}], 34319 => [{34319, 34319}], 6749 => [{6749, 6749}], 62448 => [{62448, 62448}], 79870 => [{79870, 79870}], 90980 => [{90980, 90980}], 52195 => [{52195, 52195}], 34512 => [{34512, 34512}], 19510 => [{19510, 19510}], 44436 => [{44436, 44436}], 75113 => [{75113, 75113}], 81119 => [{81119, 81119}], 8441 => [{8441, 8441}], 96376 => [{96376, 96376}], 76413 => [{76413, 76413}], 32156 => [{32156, 32156}], 43416 => [{43416, 43416}], 86876 => [{86876, 86876}], 61486 => [{61486, 61486}], 12163 => [{12163, 12163}], 40610 => [{40610, 40610}], 87629 => [{87629, 87629}], 33708 => [{33708, 33708}], 47217 => [{47217, 47217}], 77706 => [{77706, 77706}], 49309 => [{49309, 49309}], 5934 => [{5934, 5934}], 59657 => [{59657, 59657}], 61379 => [{61379, 61379}], 66854 => [{66854, ...}], 21938 => [...], ...}, %{60829 => [{60829, -60829}], 12995 => [{12995, -12995}], 42408 => [{42408, -42408}], 79313 => [{79313, -79313}], 59922 => [{59922, -59922}], 51560 => [{51560, -51560}], 6653 => [{6653, -6653}], 82555 => [{82555, -82555}], 40746 => [{40746, -40746}], 33823 => [{33823, -33823}], 46309 => [{46309, -46309}], 81817 => [{81817, -81817}], 15437 => [{15437, -15437}], 34319 => [{34319, -34319}], 6749 => [{6749, -6749}], 62448 => [{62448, -62448}], 79870 => [{79870, -79870}], 52195 => [{52195, -52195}], 34512 => [{34512, -34512}], 19510 => [{19510, -19510}], 44436 => [{44436, -44436}], 75113 => [{75113, -75113}], 81119 => [{81119, -81119}], 8441 => [{8441, -8441}], 76413 => [{76413, -76413}], 32156 => [{32156, -32156}], 43416 => [{43416, -43416}], 61486 => [{61486, -61486}], 12163 => [{12163, -12163}], 40610 => [{40610, -40610}], 33708 => [{33708, -33708}], 47217 => [{47217, -47217}], 77706 => [{77706, -77706}], 49309 => [{49309, -49309}], 5934 => [{5934, -5934}], 59657 => [{59657, -59657}], 61379 => [{61379, -61379}], 66854 => [{66854, -66854}], 21938 => [{21938, -21938}], 38839 => [{38839, -38839}], 76035 => [{76035, -76035}], 75873 => [{75873, -75873}], 73528 => [{73528, ...}], 4969 => [...], ...}, []},
#Function<21.105143239/4 in Flow.Materialize.join_ops/5>}
** (exit) exited in: GenStage.close_stream(%{#Reference<0.0.2.6937> => {:subscribed, #PID<0.28588.0>, :transient, 500, 1000, 593}, #Reference<0.0.2.6938> => {:subscribed, #PID<0.28589.0>, :transient, 500, 1000, 898}, #Reference<0.0.2.6940> => {:subscribed, #PID<0.28591.0>, :transient, 500, 1000, 572}})
    ** (EXIT) an exception was raised:
        ** (FunctionClauseError) no function clause matching in Flow.Materialize.dispatch_join/6
            (flow) lib/flow/materialize.ex:328: Flow.Materialize.dispatch_join(
              [{98431, {98431, 98431}}, {98439, {98439, 98439}}, {98442, {98442, 98442}}, {98443, {98443, 98443}}, {98446, {98446, 98446}}, {98449, {98449, 98449}}, {98454, {98454, 98454}}, {98457, {98457, 98457}}, {98458, {98458, 98458}}, {98461, {98461, 98461}}, {98465, {98465, 98465}}, {98466, {98466, 98466}}, {98467, {98467, 98467}}, {98470, {98470, 98470}}, {98472, {98472, 98472}}, {98474, {98474, 98474}}, {98475, {98475, 98475}}, {98476, {98476, 98476}}, {98484, {98484, 98484}}, {98485, {98485, 98485}}, {98488, {98488, 98488}}, {98495, {98495, 98495}}, {98501, {98501, 98501}}, {98502, {98502, 98502}}, {98516, {98516, 98516}}, {98519, {98519, 98519}}, {98524, {98524, 98524}}, {98526, {98526, 98526}}, {98531, {98531, 98531}}, {98548, {98548, 98548}}, {98552, {98552, 98552}}, {98553, {98553, 98553}}, {98561, {98561, 98561}}, {98566, {98566, 98566}}, {98571, {98571, 98571}}, {98574, {98574, 98574}}, {98576, {98576, 98576}}, {98585, {98585, 98585}}, {98599, {98599, 98599}}, {98603, {98603, 98603}}, {98614, {98614, 98614}}, {98615, {98615, 98615}}, {98619, {98619, 98619}}, {98621, {98621, 98621}}, {98632, {98632, 98632}}, {98634, {98634, 98634}}, {98639, {98639, 98639}}, {98643, {98643, ...}}, {98645, ...}, {...}, ...],
              nil,
              %{60829 => [{60829, 60829}], 12995 => [{12995, 12995}], 89750 => [{89750, 89750}], 42408 => [{42408, 42408}], 79313 => [{79313, 79313}], 59922 => [{59922, 59922}], 51560 => [{51560, 51560}], 6653 => [{6653, 6653}], 82555 => [{82555, 82555}], 40746 => [{40746, 40746}], 33823 => [{33823, 33823}], 94037 => [{94037, 94037}], 46309 => [{46309, 46309}], 81817 => [{81817, 81817}], 15437 => [{15437, 15437}], 34319 => [{34319, 34319}], 6749 => [{6749, 6749}], 62448 => [{62448, 62448}], 79870 => [{79870, 79870}], 90980 => [{90980, 90980}], 52195 => [{52195, 52195}], 34512 => [{34512, 34512}], 19510 => [{19510, 19510}], 44436 => [{44436, 44436}], 75113 => [{75113, 75113}], 81119 => [{81119, 81119}], 8441 => [{8441, 8441}], 96376 => [{96376, 96376}], 76413 => [{76413, 76413}], 32156 => [{32156, 32156}], 43416 => [{43416, 43416}], 86876 => [{86876, 86876}], 61486 => [{61486, 61486}], 12163 => [{12163, 12163}], 40610 => [{40610, 40610}], 87629 => [{87629, 87629}], 33708 => [{33708, 33708}], 47217 => [{47217, 47217}], 77706 => [{77706, 77706}], 49309 => [{49309, 49309}], 5934 => [{5934, 5934}], 59657 => [{59657, 59657}], 61379 => [{61379, 61379}], 66854 => [{66854, 66854}], 21938 => [{21938, 21938}], 38839 => [{38839, 38839}], 76035 => [{76035, 76035}], 75873 => [{75873, 75873}], 73528 => [{73528, ...}], 4969 => [...], ...},
              %{60829 => [{60829, -60829}], 12995 => [{12995, -12995}], 42408 => [{42408, -42408}], 79313 => [{79313, -79313}], 59922 => [{59922, -59922}], 51560 => [{51560, -51560}], 6653 => [{6653, -6653}], 82555 => [{82555, -82555}], 40746 => [{40746, -40746}], 33823 => [{33823, -33823}], 46309 => [{46309, -46309}], 81817 => [{81817, -81817}], 15437 => [{15437, -15437}], 34319 => [{34319, -34319}], 6749 => [{6749, -6749}], 62448 => [{62448, -62448}], 79870 => [{79870, -79870}], 52195 => [{52195, -52195}], 34512 => [{34512, -34512}], 19510 => [{19510, -19510}], 44436 => [{44436, -44436}], 75113 => [{75113, -75113}], 81119 => [{81119, -81119}], 8441 => [{8441, -8441}], 76413 => [{76413, -76413}], 32156 => [{32156, -32156}], 43416 => [{43416, -43416}], 61486 => [{61486, -61486}], 12163 => [{12163, -12163}], 40610 => [{40610, -40610}], 33708 => [{33708, -33708}], 47217 => [{47217, -47217}], 77706 => [{77706, -77706}], 49309 => [{49309, -49309}], 5934 => [{5934, -5934}], 59657 => [{59657, -59657}], 61379 => [{61379, -61379}], 66854 => [{66854, -66854}], 21938 => [{21938, -21938}], 38839 => [{38839, -38839}], 76035 => [{76035, -76035}], 75873 => [{75873, -75873}], 73528 => [{73528, -73528}], 4969 => [{4969, -4969}], 16129 => [{16129, -16129}], 75116 => [{75116, -75116}], 45250 => [{45250, -45250}], 74081 => [{74081, -74081}], 15093 => [{15093, ...}], 45659 => [...], ...},
              #Function<7.85826293/2 in Wasatch.Jobs.JoinCatsTask.test_run/0>,
              []
            )
            (flow) lib/flow/materialize.ex:286: anonymous fn/6 in Flow.Materialize.join_ops/5
            (flow) lib/flow/map_reducer.ex:49: Flow.MapReducer.handle_events/3
            (gen_stage) lib/gen_stage.ex:2502: GenStage.consumer_dispatch/7
            (gen_stage) lib/gen_stage.ex:2618: GenStage.take_pc_events/3
    (gen_stage) lib/gen_stage.ex:1705: GenStage.close_stream/1
       (elixir) lib/stream.ex:1250: Stream.do_resource/5
       (elixir) lib/stream.ex:570: Stream.run/1
josevalim commented 7 years ago

Fixed in GenStage.