A simple chart...

Chart update code

function update(objectArray) {
    var duration = this.options('transition');
    var height = this.height()-1;
    var x = this.options('x');
    var y = this.options('y');

    xScale.domain(objectArray.map(x));
    yScale.domain([0, d3.max(objectArray, y)]);

    axisX.call(xAxisFn);
    axisY.call(yAxisFn);

    var selectedData = svg.selectAll('.bar').data(objectArray, x);

    selectedData.enter()
        .append('rect')
        .attr('class', 'bar')
        .attr('width', xScale.rangeBand())
        .attr('x', function(d) { return xScale(x(d)) })
        .attr('y', height)
        .attr('height', 0);

    selectedData.transition().duration(duration)
        .attr("x", function(d) { return xScale(x(d)) })
        .attr("y", function(d) { return yScale(y(d)) });
        .attr("height", function(d) {
            return height - yScale(y(d))
        });

    selectedData.exit().transition().duration(duration)
        .attr('x', function(d) { return xScale(x(d)) })
        .attr('y', height)
        .attr('height', 0)
        .remove();
}

Refresh code

var timerId = null;
$(mySection).on('start', function (event, args) {
    if (timerId === null) {
        timerId = setInterval(function () {
            data = refreshData()
            chart.update(data)
        },args.rate)
    }
})

$(mySection).on('stop', function () {
    if (timerId !== null) {
        clearInterval(timerId);
        timerId = null;
    }
})

... grows into a dashboard

Dashboard

function updateDashboard(addMore,max) {
    for (var i=0; i < addMore; i++) {
        var order = genOrder();
        orders.push(order);
        if (orders.length > max) orders.shift():
        var current = counts.get(order.customer) || 0;
        counts.set(
            order.customer,
            Math.round(current+order.price*order.quantity)
        );
    }
    total = d3.sum(orders,function (d) {
        return d.price*d.quantity
    });

    datatable.update(orders);
    gauge.update(total);
    customerBar.update(counts.topN(5));
}

Datatable

function update(objectArray) {
    var colnames = [];
    if (mydata != null && mydata.length > 0) {
        colnames = d3.keys(objectArray[0])
    }

    var th = thead.selectAll('th').data(colnames,identity)

    th.enter().append('th')
    th.text(identity);
    th.exit().remove();

    var tr = tbody.selectAll('tr').data(objectArray, opts.id)

    tr.enter().append('tr')
    tr.exit().transition().duration(200).remove()

    var cells = tr.selectAll('td').data(d3.values)
    cells.enter().append('td')
    cells.text(identity);
    cells.exit().transition().duration(200).remove();
}

Gauge

this._update = function(objectArray) {
    var duration = this.options('transition');

    pointer.transition().duration(duration)
    .attr('transform', 'rotate(' +angle(objectArray) +')');

    value.text(valueFormat(objectArray));

};

this._render = function (objectArray) {
    var center = 'translate('+radius +','+ radius +')';
    var colorFn = this.options('arcColorFn');

    var svg = this.svg().append('g').attr('class', 'gauge')
        .attr('width', this.width())
        .attr('height', this.height());

    var arcs = svg.append('g').attr('class', 'arc').attr('transform', center);

    arcs.selectAll('path').data(tickData)
        .enter().append('path')
        .attr('fill', function(d, i) { return colorFn(d * i); })
        .attr('d', arc);

    var labelMargin = this.options('labelMargin');

    var labels = svg.append('g').attr('class', 'label').attr('transform', center);

    labels.selectAll('text').data(ticks)
        .enter().append('text')
        .attr('transform', function(d) {
            return 'rotate(' +angle(d) +') translate(0,' +(labelMargin - radius) +')';
        })
        .text(this.options('labelFormat'));

    var vg = svg.append('g').attr('class', 'value').attr('transform', center);
    value = vg.append('text').attr('transform', 'translate(0,-'+radius/3+')')

    var pointerWidth = this.options('pointerWidth');
    var pointerTailLength = this.options('pointerTailLength');
    var lineData = [
        [pointerWidth / 2, 0],
        [0, -pointerHeadLength],
        [-(pointerWidth / 2), 0],
        [0, pointerTailLength],
        [pointerWidth / 2, 0]
    ];
    var pointerLine = d3.svg.line().interpolate('monotone');
    var pg = svg.append('g').data([lineData])
        .attr('class', 'pointer')
        .attr('transform', centerTx);

    pointer = pg.append('path')
        .attr('d', pointerLine)
        .attr('transform', 'rotate(' +minAngle +')');

    this._update(objectArray ? objectArray : 0);
};

Plumbing with XHR

D3.js natively implements XHR to consume HTTP documents and connect to REST APIs

Of course Jquery, angular, etc.. work too

XHR Limits

• Unpredictable latency
• Client controlled frequency
• May retransfer same data

Plumbing with event streams

WebSocket

    var socket = new WebSocket("ws://localhost:4242/eventbus");

    socket.onmessage = function(event) {
        data.push(event.data);
        chart.update(data);
    }
    socket.onopen = function(event) {
        alert("Web Socket opened!");
    };
    socket.onclose = function(event) {
        alert("Web Socket closed.");
    };

    function send(message) {
        if (!window.WebSocket) {
            return;
        }
        if (socket.readyState == WebSocket.OPEN) {
            socket.send(message);
        } else {
            alert("The socket is not open.");
        }
    }

Web workers

• Isolate rendering and data collection
• Worker may also implement client-side data transformation or aggregation
• Communicates with main chart with messages

Web workers - Example

• Main
• Worker

var worker = new Worker('js/data-collector.js');

worker.addEventListener('message',function(event) {
    gauge.update(event.data);
},false);

worker.postMessage('start');

var ws, queue = [], maxSize = 1000;

function countAndSend(){
    var count = queue.splice(0,queue.length)
            .reduce(function (prev,d) {
                return prev+d.price
            },0);
    postMessage(count);
}

self.addEventListener('message',function (evt) {
    if (evt.data == 'start') {
        ws = new WebSocket('ws://myserver/shopping-info')
    }
    ws.onmessage = function (evt) {
        queue.push(evt.data);
        if (queue.length > maxSize) queue.shift()
    }
    setInterval(countAndSend,1000);
},false);

Back Pressure

Detect congestion conditions and notify producer to reduce incoming events

• On hold/resume
• Rate limiting

If can't notify producer, drop events and notify UI

Back Pressure - Example

• UI Code
• Worker Code

var lastRun = Date.now(), onHold = false;

function refreshChart(timestamp) {
    window.requestAnimationFrame(refreshChart);
    if (timestamp - lastRun > 40) {
        worker.postMessage('hold');
        onHold = true;
    } else if (onHold) {
        worker.postMessage('resume');
    }
    lastRun = timestamp;
    chart.update(data);
};

state.eb.registerHandler('votes', function (message) {

    state.queue.push(message);
    state.display.push(message);

    if (state.display.length > 5)
        state.display.shift();

    if (message.answer) {
        state.counts.set(message.answer,
            (state.counts.get(message.answer) || 0) + 1);
        state.total += 1;
    }

    if (state.queue.length > config.discard) {
        state.queue.shift();
        state.lost += 1;
        state.eb.send('control','pause', function (reply) {
            state.onHold = true;
        })
    }

    if (onHold && state.queue.length < config.discard * 0.8) {
        state.eb.send('control','resume', function (reply) {
            state.onHold = false;
        })
    }
});

Stability and resource usage

• SVG and DOM are memory hogs
• Actions are CPU intensive
• May create stability issues with high velocity streams

Use D3.js with a <canvas> element

Some tools

Epoch.js: A visualization library over D3.js for realtime dashboards by Fastly

Streamdata.io: Service to convert a REST API to a streaming API