Automatically split up too big messages in smaller messages

Right now we only get an exception, so we built a splitting mechanism by ourselfes, but it would be great if your lib would adjust the message size automatically. Also sometimes we are still hitting the limits (even though we are calculating all the bytes by ourselfes), so maybe you have a better solution?

This is our solution (we are rebuilding the message on the client side afterwards):

max_message_size = 190_000  # Limit is 256kb, more than 190_000 doesn't work sometimes.. dunno why

chars = message.to_json.to_s.chars
message_chunks = []
while chars.any?

  chunks = []
  chunksize = 0
  while chars.any? && chunksize < max_message_size
    if (chunksize + chars[0].bytesize) > max_message_size
      break
    else
      next_char = chars.shift()
      chunksize += next_char.bytesize
      chunks << next_char
    end
  end
  message_chunks << chunks.join
end

client = Ably::Rest.new(ENV["ABLY_PRIVATE_KEY"])
channel = client.channel(channel_name)
message_id = SecureRandom.uuid

message_chunks.each_with_index do |chunk, index|
  data = {
    utf: "✓",
    message_id: message_id,
    message_index: index,
    message_count: message_chunks.count,
    data: chunk
  }

  channel.publish(event, data)
end

Rebuliding on client side:

let incomingMessages = {}

const handleMessageChunk = (type, messageChunk) => {

  if (messageChunk.message_id && messageChunk.message_count && (messageChunk.message_index || messageChunk.message_index === 0)) {

    if(incomingMessages[messageChunk.message_id] === undefined) {
      incomingMessages[messageChunk.message_id] = {}
    }

    incomingMessages[messageChunk.message_id][messageChunk.message_index] = messageChunk

    let isMessageComplete = Object.keys(incomingMessages[messageChunk.message_id]).length == messageChunk.message_count

    if (isMessageComplete) {
      let completeMessage = ""
      for (let i = 0; i < messageChunk.message_count; i++) {
        completeMessage += incomingMessages[messageChunk.message_id][i].d;
      }

      delete incomingMessages[messageChunk.message_id]
      console.log("message completed", JSON.parse(completeMessage), incomingMessages)
      dispatch({type: type, payload: JSON.parse(completeMessage)})
    } else {
      //console.log("No finished message", messageChunk)
    }
  }
}

┆Issue is synchronized with this Jira Bug by Unito

ably / ably-ruby

Automatically split up too big messages in smaller messages #212