flyerhzm / redis-sentinel

another redis automatic master/slave failover solution for ruby by using built-in redis sentinel (deprecated)
MIT License
188 stars 67 forks source link

undefined method `values_at' for nil:NilClass #49

Closed tanema closed 7 years ago

tanema commented 9 years ago

I keep getting this error no matter what the config here is the stacktrace: Stacktrace:

redis-sentinel/client.rb:65:in `refresh_sentinels_list'
    @sentinels_options.uniq! {|h| h.values_at(:host, :port) }
  org/jruby/RubyArray.java:3209:in `uniq!'
  redis-sentinel/client.rb:65:in `refresh_sentinels_list'
    @sentinels_options.uniq! {|h| h.values_at(:host, :port) }
  redis-sentinel/client.rb:77:in `discover_master'
    refresh_sentinels_list
  redis-sentinel/client.rb:27:in `connect_with_sentinel'
    discover_master
  org/jruby/RubyProc.java:271:in `call'
  redis-sentinel/client.rb:45:in `auto_retry_with_timeout'
    block.call
  redis-sentinel/client.rb:26:in `connect_with_sentinel'
    auto_retry_with_timeout do
  redis/client.rb:304:in `ensure_connected'
    connect
  redis/client.rb:191:in `process'
    ensure_connected do
  redis/client.rb:270:in `logging'
    return yield unless @logger && @logger.debug?
  redis/client.rb:190:in `process'
    logging(commands) do
  redis/client.rb:161:in `call_pipelined'
    process(commands) do
  redis/client.rb:133:in `call_pipeline'
    pipeline.finish(call_pipelined(pipeline.commands)).tap do
  redis/client.rb:257:in `with_reconnect'
    yield
  redis/client.rb:131:in `call_pipeline'
    with_reconnect pipeline.with_reconnect? do
  redis-sentinel/client.rb:145:in `readonly_protection_with_timeout'
    send(method, *args, &block)
  redis-sentinel/client.rb:136:in `call_pipeline_with_readonly_protection'
    readonly_protection_with_timeout(:call_pipeline_without_readonly_protection, *args, &block)
  redis.rb:2094:in `multi'
    original.call_pipeline(pipeline)
  redis.rb:37:in `synchronize'
    mon_synchronize { yield(@client) }
  monitor.rb:211:in `mon_synchronize'
    yield
  monitor.rb:210:in `mon_synchronize'
    begin
  redis.rb:37:in `synchronize'
    mon_synchronize { yield(@client) }
  redis.rb:2086:in `multi'
    synchronize do |client|
  redis/namespace.rb:413:in `namespaced_block'
    redis.send(command) do |r|
  redis/namespace.rb:258:in `multi'
    namespaced_block(:multi, &block)
  sidekiq/client.rb:180:in `raw_push'
    conn.multi do
  connection_pool.rb:58:in `with'
    yield conn
  sidekiq/client.rb:179:in `raw_push'
    @redis_pool.with do |conn|
  sidekiq/client.rb:68:in `push'
    pushed = raw_push([payload]) if payload
  sidekiq/worker.rb:85:in `client_push'
    Sidekiq::Client.new(pool).push(item.stringify_keys)
  sidekiq/worker.rb:40:in `perform_async'
    client_push('class' => self, 'args' => args)
  devise/async/backend/sidekiq.rb:10:in `enqueue'
    perform_async(*args)
  devise/async/worker.rb:11:in `enqueue'
    backend_class.enqueue(method, resource_class, resource_id, *args)
  devise/async/model.rb:37:in `send_devise_notification'
    Devise::Async::Worker.enqueue(notification, self.class.name, self.id.to_s, *args)
  devise_invitable/model.rb:168:in `deliver_invitation'
    send_devise_notification(:invitation_instructions)
  devise_invitable/model.rb:132:in `invite!'
    deliver_invitation unless @skip_invitation
  devise_invitable/model.rb:240:in `_invite'
    mail = invitable.invite!
  devise_invitable/model.rb:252:in `invite!'
    invitable, mail = _invite(attributes, invited_by, &block)
  devise/invitations_controller.rb:16:in `create'
    self.resource = resource_class.invite!(resource_params, current_inviter)
  devise/invitations_controller.rb:32:in `create'
    super
  action_controller/metal/implicit_render.rb:4:in `send_action'
    ret = super
  abstract_controller/base.rb:167:in `process_action'
    send_action(method_name, *args)
  action_controller/metal/rendering.rb:10:in `process_action'
    super
  abstract_controller/callbacks.rb:18:in `process_action'
    super
  active_support/callbacks.rb:472:in `_run__1802481893__process_action__984953711__callbacks'
    # When creating or skipping callbacks, you can specify conditions that
  active_support/callbacks.rb:405:in `__run_callback'
    object.send(name, &blk)
  active_support/callbacks.rb:390:in `_run_process_action_callbacks'
    end
  active_support/callbacks.rb:81:in `run_callbacks'
    send("_run_#{kind}_callbacks", *args, &block)
  abstract_controller/callbacks.rb:17:in `process_action'
    run_callbacks(:process_action, action_name) do
  mongoid/unit_of_work.rb:39:in `unit_of_work'
    yield if block_given?
  mongoid/unit_of_work.rb:37:in `unit_of_work'
    begin
  rack/mongoid/middleware/identity_map.rb:34:in `call'
    ::Mongoid.unit_of_work { @app.call(env) }
  warden/manager.rb:35:in `call'
    @app.call(env)
  org/jruby/RubyKernel.java:1282:in `catch'
  warden/manager.rb:34:in `call'
    result = catch(:warden) do
  action_dispatch/middleware/best_standards_support.rb:17:in `call'
    status, headers, body = @app.call(env)
  rack/etag.rb:23:in `call'
    status, headers, body = @app.call(env)
  rack/conditionalget.rb:35:in `call'
    @app.call(env)
  action_dispatch/middleware/head.rb:14:in `call'
    @app.call(env)
  action_dispatch/middleware/params_parser.rb:21:in `call'
    @app.call(env)
  action_dispatch/middleware/flash.rb:242:in `call'
    @app.call(env)
  rack/session/abstract/id.rb:210:in `context'
    status, headers, body = app.call(env)
  rack/session/abstract/id.rb:205:in `call'
    context(env)
  action_dispatch/middleware/cookies.rb:341:in `call'
    status, headers, body = @app.call(env)
  action_dispatch/middleware/callbacks.rb:28:in `call'
    @app.call(env)
  active_support/callbacks.rb:408:in `_run__1990414355__call__699693046__callbacks'
    def __reset_runner(symbol)
  active_support/callbacks.rb:405:in `__run_callback'
    object.send(name, &blk)
  active_support/callbacks.rb:390:in `_run_call_callbacks'
    end
  active_support/callbacks.rb:81:in `run_callbacks'
    send("_run_#{kind}_callbacks", *args, &block)
  action_dispatch/middleware/callbacks.rb:27:in `call'
    run_callbacks :call do
  action_dispatch/middleware/remote_ip.rb:31:in `call'
    @app.call(env)
  action_dispatch/middleware/debug_exceptions.rb:16:in `call'
    response = @app.call(env)
  action_dispatch/middleware/show_exceptions.rb:56:in `call'
    response  = @app.call(env)
  rails/rack/logger.rb:32:in `call_app'
    @app.call(env)
  rails/rack/logger.rb:16:in `call'
    Rails.logger.tagged(compute_tags(request)) { call_app(request, env) }
  active_support/tagged_logging.rb:22:in `tagged'
    yield self
  rails/rack/logger.rb:16:in `call'
    Rails.logger.tagged(compute_tags(request)) { call_app(request, env) }
  action_dispatch/middleware/request_id.rb:22:in `call'
    status, headers, body = @app.call(env)
  rack/methodoverride.rb:21:in `call'
    @app.call(env)
  rack/runtime.rb:17:in `call'
    status, headers, body = @app.call(env)
  active_support/cache/strategy/local_cache.rb:72:in `call'
    @app.call(env)
  action_dispatch/middleware/static.rb:63:in `call'
    @app.call(env)
  rack/cache/context.rb:136:in `forward'
    Response.new(*backend.call(@env))
  rack/cache/context.rb:143:in `pass'
    forward
  rack/cache/context.rb:155:in `invalidate'
    pass
  rack/cache/context.rb:71:in `call!'
    invalidate
  rack/cache/context.rb:51:in `call'
    clone.call! env
  raven/rack.rb:59:in `call'
    response = @app.call(env)
  rails/engine.rb:484:in `call'
    app.call(env.merge!(env_config))
  rails/application.rb:231:in `call'
    super(env)
  rails/railtie/configurable.rb:30:in `method_missing'
    instance.send(*args, &block)
  puma/configuration.rb:68:in `call'
    @app.call(env)
  puma/server.rb:486:in `handle_request'
    status, headers, res_body = @app.call(env)
  puma/server.rb:484:in `handle_request'
    begin
  puma/server.rb:357:in `process_client'
    case handle_request(client, buffer)
  puma/server.rb:353:in `process_client'
    begin
  puma/server.rb:250:in `run'
    process_client client, buffer
  org/jruby/RubyProc.java:271:in `call'
  puma/thread_pool.rb:92:in `spawn_thread'
    block.call(work, *extra)

Here is what my config looks like:

require 'redis/namespace'

if Rails.env.production?
  # Using redis-sentinel gem to control sentinel layer
  # https://github.com/mperham/sidekiq/wiki/Advanced-Options
  options = {
    master_name: "master name",
    sentinels: [
      {host: "ip", port: "26379"}, 
      {host: "ip", port: "26379"}, 
      {host: "ip", port: "26379"} #  - arbiter
    ],
    failover_reconnect_timeout: 15, # give 15s for failover to take place
    failover_reconnect_wait: 10 # wait 10s after each failed reconnect
  }

  redis_conn = proc {
    # The following lines use a combination of redis-sentinel and
    # the redis namespace gems to give the connection pool the redis client object
    client = Redis.new(options)
    Redis::Namespace.new("bg_jobs", redis: client)
  }
  Sidekiq.configure_client do |config|
    config.redis = ConnectionPool.new(timeout: 10, size: 5, &redis_conn)
  end
  Sidekiq.configure_server do |config|
    config.redis = ConnectionPool.new(timeout: 10, size: 25, &redis_conn)
  end

else
  port = Rails.env.test? ? "9736" : "6379"
  options = {url: "redis://127.0.0.1:#{port}/12", namespace: "bg_jobs"}
  Sidekiq.configure_server do |config|
    config.redis = options
  end

  Sidekiq.configure_client do |config|
    config.redis = options
  end
end

It of course only happens in production, and I tried doing it all in the console and I get the same errors, however I can get all_clients and for some odd reason it works if I Redis.new().client.current_sentinel.

Any help explaining this would be really appreciated

tanema commented 9 years ago

still trying to figure this out, when I try and set it all up in the console by doing r = Redis.new(options) r.sentinel('sentinels', 'master') fails but r.client.current_sentinel.sentinel('sentinels', 'master') works

when I inspect them r has a wrong port of 6379 when all my sentinels have the port 26379

tanema commented 9 years ago

okay so it turns out it may be a jruby problem. I switched to jruby 1.7.10 (which the project is using) and ran your tests and got the exact same error

tanema commented 9 years ago

moved up to just jruby-1.7.11 and it works. that was not fun.

flyerhzm commented 9 years ago

@tanema I didn't try redis-sentinel in jruby, but I guess it's caused by _parse_sentinel_options, could you try the latest master?