Beeks Analytics Decoder Information

A full decoder SDK example with comments

The following script decoder relates to the hypothetical protocol named "chapo". The reason we use a made up protocol in this example rather than a pre-existing one is:

So the protocol is very simple to document.
To require and thus show off many features of the script decoder.

Description of chapo protocol:

In the fictional chapo format each message begins with a header composed of:

Bell (ascii character 7)
Message Type (only byte ASCII '8' supported)
32-bit unsigned integer - length of message body, not including header.
64-bit unsigned integer - sequence id.

The unsigned integers are encoded in network-byte order. Here is an example header:
<7: 1 byte><ASCII '8': 1 byte><103: 4 bytes><1: 8 bytes>

This indicates a message with a body of size 103 and a sequence of 1.

The message body is composed of key value pairs. Pairs are seperated by the '~' character, and keys are seperated from values by the '=' character.

For example:
M=hello~T=1234

This represents two key value pairs ("M", "hello") and ("T", 1234)

Numbers are encoded as ASCII strings in the message body.

The chapo decoder script

// Load console module for writing messages to the terminal display.
var console = vu8.load('console'),
    idx     = 0,
    CHAPO_MSG_START = 7,
    CHAPO_MSG_TYPE  = 8,
    CHAPO_HEADER_SIZE  = 14, // uint64 sequence, start, type, uint32 size
    PAIR_SEP_BYTE    = '~'.charCodeAt(0),
    KEY_VAL_SEP_BYTE = '='.charCodeAt(0)
 
 
// Create 4 datafield objects used for this decoder
var dfMarketId       = new decoder.DatafieldUInt("script_v8.market_id"),
    dfSequence       = new decoder.DatafieldUInt("script_v8.sequence"),
    dfSize           = new decoder.DatafieldUInt("script_v8.size"),
    dfSymbolTicker   = new decoder.DatafieldString("script_v8.symbol_ticker")
 
try {
    // Register datafields with decoder's protocol layer.
    decoder.protocol.addDatafield(dfMarketId)
    decoder.protocol.addDatafield(dfSequence)
    decoder.protocol.addDatafield(dfSize)
    decoder.protocol.addDatafield(dfSymbolTicker)
}
catch (e) {
    // C++ exceptions propogate to JavaScript exceptions...
    console.log('chapo decoder: error adding data fields')
}
 
console.log('chapo decoder: loaded script')
 
////////////////////////////////////////////////////////////////////////////
// The next two functions are events called by the underlying stack API
 
// "on_new_decoder" is called each time a new decoder is created and must be
// defined in every script.
function on_new_decoder(dec) {
    dec.idx = ++idx
 
    // market ID is a 64-bit integer and JavaScript numbers cannot represent
    // this type so decoder.UInt64 is provided for handling this type.
    dec.marketId       = new decoder.UInt64()
    dec.sequence       = new decoder.UInt64()
 
    // dfSize is 32-bit so can use a plain JavaScript number.
    dec.dfSize         = 0
 
    // Use this to signal if a message begin marker has been found.
    dec.foundMsgBegin = false
 
    // Can associate method with decoder object for use in on_data callback.
    dec.log = function(msg) {
        console.log('chapo decoder', this.idx + ':', msg)
    }
 
    console.log('created new chapo decoder')
}
 
// "on_data" is called when a decoder receives new data and must be defined
// in every script.
// The dec.buffer attribute is a Buffer object which represents a series of
// underlying buffers as a single conceptual buffer.
// The fact that the user is dealing with multiple underlying buffers
// is hidden by the Buffer object API.
// The last buffer in the chain is not copied from the underlying
// stream until after on_data has been called (in the event that after
// such a call there is still data remaining in the buffer) allowing
// for zero-copy memory handling when possible.
function on_data(dec) {
    // Use method associated with decoder object by "on_new_decoder".
    // size is the number of bytes left in the buffer.
    dec.log('new data available, reimaning buffer length ' + dec.buffer.size())
 
    if (! dec.dfSize) {
        parse_header(dec)
        if (! dec.dfSize) return
    }
 
    while (parse_message(dec)) {
        dec.foundMsgBegin = false
        parse_header(dec)
        if (! dec.dfSize) return
    }
}
 
////////////////////////////////////////////////////////////////////////////
// helper methods from here on
 
// Parse chapo header.
function parse_header(dec) {
    if (! dec.foundMsgBegin) {
        // Skips up to and over the character with the ascii byte given..
        // returns true if the byte is found otherwise empties the buffer
        // and returns false.
        if (dec.buffer.skipToAndConsumeByte(CHAPO_MSG_START))
            dec.foundMsgBegin = true
        else
            return
    }
 
    // Doing this is faster than dec.buffer.size() < 5, there is also:
    // sizeLessThanOrEqualTo, sizeGreaterThan, sizeGreaterThanOrEqualTo
    if (dec.buffer.sizeLessThan(CHAPO_HEADER_SIZE - 1)) return
 
    // Javascript has no facility for dealing with characters, so passing a
    // number through to isByte() is better than having to deal with expensive
    // string types.
    while (! dec.buffer.isByte(CHAPO_MSG_TYPE)) {
        dec.log('unsupported message type')
        if (! dec.buffer.skipToAndConsumeByte(CHAPO_MSG_START)) {
            dec.foundMsgBegin = false
            return
        }
 
        if (dec.buffer.sizeLessThan(CHAPO_HEADER_SIZE - 1)) return
    }
 
    // This advances the current character pointer in the buffer but does not
    // reclaim memory until an entire underlying buffer within the chain has
    // been advanced over so it is efficient to use.
    dec.buffer.advance(1)
 
    // Consume a byte-encoded uint32 from the buffer using network-byte
    // order. Use consumeUInt32LE to decode the integer using little-endian
    // byte ordering. There is also consumeInt32 and consumeInt32LE for
    // signed integers. This advances the buffer past the decoded data.
    dec.dfSize = dec.buffer.consumeUInt32()
 
    // An alternative way to do the above two operations would be:
    //     dec.dfSize = dec.buffer.decodeUInt32(1)
    //     dec.buffer.advance(5)
 
    // Binary decode uint64 from head of buffer and advance buffer pointer
    // past it.
    // Can use consumeLE to decode it using little-endian byte ordering.
    dec.sequence.consume(dec.buffer)
 
    // equivalent to the above:
    //     dec.sequence.decode(dec.buffer, 0)
    //     dec.buffer.advance(8)
 
    // There are decode*() versions of all consume*() methods. Decode is like
    // consume but does not advance the buffer and takes an argument indicating
    // the offset of the data to decode.
 
    dec.log('found message header which indicates a payload of size ' + dec.dfSize)
}
 
// Parse chapo message body. Return true if current message in buffer is
// finished with. If it is finished and parsed successfully the buffer
// will be advanced past it.
function parse_message(dec) {
    if (dec.buffer.sizeLessThan(dec.dfSize)) {
        dec.log('chapo message split across multiple packets')
        return false
    }
 
    // Buffer.substring operates in the same manner as String.substring
    // in JavaScript.
    dec.log("buffer contains complete chapo message '" +
            dec.buffer.substring(0, dec.dfSize) + "'")
 
    var pairIdx, nextPairIdx = -1
    while (nextPairIdx < dec.dfSize) {
        pairIdx = nextPairIdx + 1
        var sepIdx = dec.buffer.indexOfByteFrom(KEY_VAL_SEP_BYTE, pairIdx + 1)
        nextPairIdx = dec.buffer.indexOfByteFrom(PAIR_SEP_BYTE, sepIdx + 1)
 
        var key = dec.buffer.substring(pairIdx, sepIdx)
        if ("S" == key) {
            dfSymbolTicker.set(dec.buffer.substring(sepIdx + 1, nextPairIdx))
        }
        else if ("3" == key) {
            // use + to get JavaScript to convert string into integer
            dec.marketId.fromUInt32(+dec.buffer.substring(sepIdx + 1, nextPairIdx))
        }
    }
 
    // Set these datafields from decoder.UInt64 objects
    dfMarketId.fromUInt64(dec.marketId)
    dfSequence.fromUInt64(dec.sequence)
 
    // fromUInt32 works on normal JavaScript numbers.
    dfSize.fromUInt32(dec.dfSize)
 
    // Pass through to next decoder with no data which triggers a
    // collection of the datafields. nextDecoder(length) and
    // nextDecoder(offset, length) are also provided for passing a
    // subsection of the buffer over.
    dec.nextDecoder()
 
    // advance past current message
    dec.buffer.advance(dec.dfSize)
 
    // Make both integers 0 again.
    dec.marketId.setZero()
    dec.dfSize = 0
 
    return true
}
 
// Called when decoder misses data
//   dec: object representing decoder that missed data
//   size: size of missed data.. has type decoder.UInt64()
function on_missed_data(dec, size) {
    console.log("missed data of length: " + size.toDouble())
 
    dec.foundMsgBegin = false
    dec.buffer.reset()
}