Open ManApart opened 4 years ago
Persist should recover from other parts of the system failing and then recovering. Rolling the pod resolved the issue, but should not be necessary.
AC
18:48:53.545 [error] GenServer :brook_driver_kafka_persist_instance terminating ** (stop) [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}] (brod) src/brod_client.erl:554: :brod_client.ensure_metadata_connection/1 (brod) src/brod_client.erl:300: :brod_client.handle_info/2 (stdlib) gen_server.erl:637: :gen_server.try_dispatch/4 (stdlib) gen_server.erl:711: :gen_server.handle_msg/6 (stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3 Last message: :init 18:48:53.546 [warn] Elixir.Elsa.DynamicProcessManager: initializer raised exception, retrying: {[{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}], {:gen_server, :call, [#PID<0.3785.0>, {:get_metadata, "event-stream"}, :infinity]}} 18:48:53.546 [error] GenServer #PID<0.3770.0> terminating ** (stop) [{{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}] Last message: {:EXIT, #PID<0.3771.0>, [{{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]} 18:48:53.546 [error] GenServer #PID<0.3791.0> terminating ** (stop) exited in: :gen_server.call(:brook_driver_kafka_persist_instance, {:get_group_coordinator, "persist-event-stream"}, :infinity) ** (EXIT) [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}] (stdlib) gen_server.erl:223: :gen_server.call/3 (brod) src/brod_client.erl:839: :brod_client.safe_gen_call/3 (brod) src/brod_group_coordinator.erl:436: anonymous fn/2 in :brod_group_coordinator.discover_coordinator/1 (brod) src/brod_group_coordinator.erl:436: :brod_group_coordinator.discover_coordinator/1 (brod) src/brod_group_coordinator.erl:530: :brod_group_coordinator.do_stabilize/3 (brod) src/brod_group_coordinator.erl:343: :brod_group_coordinator.handle_info/2 (stdlib) gen_server.erl:637: :gen_server.try_dispatch/4 (stdlib) gen_server.erl:711: :gen_server.handle_msg/6 Last message: {:lo_cmd_stabilize, 0, :undefined} 18:48:53.547 [error] GenServer #PID<0.3756.0> terminating ** (stop) exited in: GenServer.call({:via, Elsa.Registry, {:"elsa_registry_Elixir.Writer.Kafka.Topic_#PID<0.3727.0>", :producer_process_manager}}, :ready?, 10000) ** (EXIT) time out Last message: {:EXIT, #PID<0.3727.0>, {:timeout, {GenServer, :call, [{:via, Elsa.Registry, {:"elsa_registry_Elixir.Writer.Kafka.Topic_#PID<0.3727.0>", :producer_process_manager}}, :ready?, 10000]}}} 18:48:53.551 [error] GenServer #PID<0.3779.0> terminating ** (stop) [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}] Last message: {:EXIT, #PID<0.3780.0>, [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]} 18:48:53.572 [warn] Elixir.Elsa.DynamicProcessManager: initializer raised exception, retrying: %MatchError{term: {:error, :client_down}}
Persist should recover from other parts of the system failing and then recovering. Rolling the pod resolved the issue, but should not be necessary.
AC