Add memory-cap reset to Sidekiq processes (200-250 mb?)

Potential memory bloat in sync_worker.rb:

    Guild.all.each do |guild|
      # Perform basic update
      guild.update_from_battle_net(type: 'guild')
      # Perform member update
      guild.update_from_battle_net(type: 'guild-members')
    end
    # Update all characters not part of a guild
    Character.where(guild: nil) do |character|
      character.update_from_battle_net
    end

I suspect that due to the lengthy loops, the process is not able to run garbage collection properly during the perform method action.

Possible solution: Split into more sub-workers and utilize chunking where possible:

    Guild.all.each do |guild|
      # Create Worker exclusively for guild base data
      # Create another Worker for guild members
    end

and...

    Character.where(guild: nil) do |character|
      character.update_from_battle_net
    end

turns into:

character_count = Character.where(guild: nil).size
# Create worker for each 'chunk' of Characters
chunk_size = 500
limit = chunk_size
offset = 0
while (offset + limit) <= character_count
  CharactersSyncWorker.perform_async(offset: 0, limit: limit)
  offset += limit
end

class CharactersSyncWorker
  def perform(args={})
    offset = args[:offset] || 0
    limit = args[:limit]

    if limit
      Character.where(guild: nil).limit(limit).offset(offset) do |character|
        character.update_from_battle_net        
      end
    else
      Character.where(guild: nil).offset(offset) do |character|
        character.update_from_battle_net
      end
    end
  end
end

GabeStah / vcp

Add memory-cap reset to Sidekiq processes (200-250 mb?) #105