How to handle partial trade messages

[spookysys]

Hi,

I am trying to interpret the aggregated trades websocket stream for CCCAGG, subscribing to several symbols, and i'm able to decode the messages fine.

I'm interested in the price and quantity for each aggregated trade, so I am looking at the fields "price" and "lastvolume", which I assume match to these. (Though I am not sure what the "last" in "lastvolume" and other fields is referring to?) I am also looking at "lasttradeid" and "lastexchange" in order to do some checking.

The problem I am facing is that a lot of the messages seem to be incomplete; they contain some but not all of the mentioned fields. I also see the same "lasttradeid" repeating.

This makes me think that single aggregated trades have been split into multiple messages that need to be re-combined?

How do I do this recombining? My first assumption would be to use "lasttradeid" as the key. However, there are messages where this field is also missing. Is it enough to combine on "symbol"?

Please clarify this aspect of the websocket API, as I expect this to be an issue for most people trying to use it. The examples do not seem to do any re-combining, they just show the data as is.

[MAkcanca]

Also having the same problem. Thumbs up

[cryptoqween]

Hey guys, use the CCC.CURRENT.unpack method in ccc-streamer-utilities.js

When you subscribing to CCCAGG, the first message is a complete message, after that we only send differences. The unpack method above will do the right mapping for you.

[slidenerd]

@spookysys if you dont mind sharing, if they are incomplete and if you are not able to use the lasttrade id , how are you aggregating the OHLCV from the prices? my ohlc more or less matches the one minute data that I tested against with the REST API but my volume calculation is off by miles

function Stream() {
    var streamMessageType = {
        'trade': '0',
        'news': '1',
        'current': '2',
        'load_complete': '3',
        'coin_pairs': '4',
        'current_agg': '5',
        'top_list': '6',
        'top_list_change': '7',
        'orderbook': '8',
        'full_orderbook': '9',
        'activation': '10',
        'full_volume': '11',
        'trade_catch_up': '100',
        'news_catch_up': '101',
        'trade_catch_up_complete': '300',
        'news_catch_up_complete': '301'
    }

    var streamKeys = {
        'type': 0x0 // hex for binary 0, it is a special case of fields that are always there
            ,
        'market': 0x0 // hex for binary 0, it is a special case of fields that are always there
            ,
        'from': 0x0 // hex for binary 0, it is a special case of fields that are always there
            ,
        'to': 0x0 // hex for binary 0, it is a special case of fields that are always there
            ,
        'flags': 0x0 // hex for binary 0, it is a special case of fields that are always there
            ,
        'price': 0x1 // hex for binary 1
            ,
        'bid': 0x2 // hex for binary 10
            ,
        'offer': 0x4 // hex for binary 100
            ,
        'last_update': 0x8 // hex for binary 1000
            ,
        'avg': 0x10 // hex for binary 10000
            ,
        'last_vol': 0x20 // hex for binary 100000
            ,
        'last_vol_to': 0x40 // hex for binary 1000000
            ,
        'last_trade_id': 0x80 // hex for binary 10000000
            ,
        'vol_24': 0x100 // hex for binary 100000000
            ,
        'vol_hour_to': 0x200 // hex for binary 1000000000
            ,
        'vol_24h': 0x400 // hex for binary 10000000000
            ,
        'vol_24h_to': 0x800 // hex for binary 100000000000
            ,
        'open_hour': 0x1000 // hex for binary 1000000000000
            ,
        'high_hour': 0x2000 // hex for binary 10000000000000
            ,
        'low_hour': 0x4000 // hex for binary 100000000000000
            ,
        'open_24h': 0x8000 // hex for binary 1000000000000000
            ,
        'high_24h': 0x10000 // hex for binary 10000000000000000
            ,
        'low_24h': 0x20000 // hex for binary 100000000000000000
            ,
        'last_market': 0x40000 // hex for binary 1000000000000000000, this is a special case and will only appear on CCCAGG messages
    }

    this.unpack = function(value) {
        var values = value.split("~")
        var mask = values[values.length - 1]
        var maskInt = parseInt(mask, 16)
        var unpackedCurrent = {}
        var currentFieldIndex = 0
        for (var property in streamKeys) {
            if (streamKeys[property] === 0) {
                unpackedCurrent[property] = values[currentFieldIndex]
                currentFieldIndex++
            } else if (maskInt & streamKeys[property]) {

                // The values which can be converted to numbers should be converted to numbers
                unpackedCurrent[property] = values[currentFieldIndex] && !isNaN(values[currentFieldIndex]) ? +values[currentFieldIndex] : values[currentFieldIndex]
                currentFieldIndex++
            }
        }

        return unpackedCurrent
    }
    socket.on("m", function(message) {
        var stream = new Stream()
        stream.handleMessage(coinData, message)
        for (var coin in coinData) {
            var currentCoin = coinData[coin]

            if (currentCoin['price'] && currentCoin['last_vol']) {

                var roundedTimestamp = (Math.floor(new Date().getTime() / 60000) * 60000) / 1000

                if (!ohlcData[coin]) {
                    ohlcData[coin] = []
                }

                var candles = ohlcData[coin]
                var lastCandle = candles[candles.length - 1]

                if (lastCandle && lastCandle['ts'] === roundedTimestamp) { //existing candle
                    lastCandle['c'] = currentCoin['price']
                    lastCandle['h'] = currentCoin['price'] > lastCandle['h'] ? currentCoin['price'] : lastCandle['h']
                    lastCandle['l'] = currentCoin['price'] < lastCandle['l'] ? currentCoin['price'] : lastCandle['l']
                    lastCandle['vf'] += currentCoin['last_vol']
                    lastCandle['vt'] += currentCoin['last_vol_to']
                } else { //add a new candle
                    ohlcData[coin].push({
                        'ts': roundedTimestamp,
                        'o': currentCoin['price'],
                        'h': currentCoin['price'],
                        'l': currentCoin['price'],
                        'c': currentCoin['price'],
                        'vf': currentCoin['last_vol'],
                        'vt': currentCoin['last_vol_to']
                    })
                }
                ohlcData[coin].splice(0, ohlcData[coin].length - 1000)
            }

            console.clear()
            console.log('PRINTING', coin, currentCoin['price'], currentCoin['last_vol'], currentCoin['last_vol_to'], ohlcData[coin].length)

            for (var i = 0; i < ohlcData[coin].length; i++) {
                console.log(
                    ohlcData[coin][i]['ts'],
                    ohlcData[coin][i]['o'],
                    ohlcData[coin][i]['h'],
                    ohlcData[coin][i]['l'],
                    ohlcData[coin][i]['c'],
                    ohlcData[coin][i]['vf'],
                    ohlcData[coin][i]['vt']
                )
            }

        }
    })