@sh1mmer

Using Node.js to Build Great

Streaming Services

Tom Hughes-Croucher@sh1mmer

@sh1mmer

@sh1mmer

@sh1mmer

Scalable Server-Side Code with JavaScript

Tom Hughes-Croucher

NodeUp and Running

http://ofps.oreilly.com/titles/9781449398583/http://shop.oreilly.com/product/0636920015956.do

@sh1mmer

A story

@sh1mmer

factory

@sh1mmer

@sh1mmer

@sh1mmer

var http = require('http');

var server = http.createServer();server.listen(8000);server.on('request', function(req,res) { var doodad = new Doodad(); res.writeHead(200,{'Content-Type':'text/doodad'}); res.end(doodad);});

@sh1mmer

Making Doodads

@sh1mmer

Screens AntennasKnobs

& buttons

@sh1mmer

@sh1mmer

@sh1mmer

@sh1mmer

@sh1mmer

@sh1mmer

var http = require('http'), fs = require('fs');

var server = http.createServer();server.listen(8000);server.on('request', function(req,res) { var screen = fs.readFileSync('/factory1/screen'); var dial = fs.readFileSync('/factory2/dial'); var doodad = new Doodad(screen, dial); res.writeHead(200,{'Content-Type':'text/doodad'}); res.end(doodad);});

@sh1mmer

Improving the factory with streaming

@sh1mmer

@sh1mmer

@sh1mmer

@sh1mmer

var http = require('http');

var server = http.createServer();server.listen(8000);server.on('request', function(req,res) { var parts = 0; http.get('factory1/screen/', function(res) { var screen = ""; res.on('data', function(d) { screen += d }); res.on('end', finish); parts++; }; http.get('factory2/dial', function(res) { var dial = "" res.on('data', function(d) { dial += d }); res.on('end', finish); parts++; } function finish() { if(parts == 2) { var doodad = new Doodad(screen, dial); res.writeHead(200,{'Content-Type':'text/doodad'}); res.end(doodad); } }});

@sh1mmer

Streams API

@sh1mmer

• Readable

• 'data' event

• 'end' event

• pause()

• resume()

• destroy()

• pipe()

Streams

• Writable

• write()

• end()

• 'drain' event

• destroy()

• destroySoon()

@sh1mmer

var stream = require('stream');

var s = new stream.Stream(); //I am an EventEmitters.readable = true; //now I implement read API

s.emit('data', "my data");

s.emit('end'); //I'm done sending data

@sh1mmer

var stream = require('stream');

var s = new stream.Stream(); //I am an EventEmitters.writable = true; //now I implement write APIs.data = "";

s.write = function(d) { s.data += d;};

s.end = function() { if(s.data) { console.log(s.data) }};

s.destroy = function() { s.writable = false;}

@sh1mmer

The too many parts problem and why

phone calls don't work

@sh1mmer

Event Loop Fail

libuv

add-ons

libev iocp

v8

node.ccTCP Conn

TCP Conn

TCP Conn

nextTicknextTick

FS Read

FS Read

FS Read

Timeout

Timeout

Interval

Run stuff

stream.pause()

Event Loop Fail

libuv

add-ons

libev iocp

v8

node.ccTCP Conn

TCP Conn

TCP Conn

nextTicknextTick

FS Read

FS Read

FS Read

Timeout

Timeout

Interval

stream.pause()

@sh1mmer

Managing Backpressure

• Manage Buffer sizes

• Use pause() / resume() to control back pressure

• Assume that 'data' event may happen after pause

• Remember node buffer -> kernel buffer

@sh1mmer

//manage Node's write Buffersif(socket.bufferSize > maxSafe) { writeStream.pause();}

//Manage RS read stream Buffersfs.createReadStream('./path/to/file', { flags: 'r', encoding: null, fd: null, mode: 0666, bufferSize: 64 * 1024});

@sh1mmer

New API

New Streams TBD• Writable streams

• Same

• Readable (https://github.com/isaacs/readable-stream)

• configurable stream buffer waterline

• 'readable' event

• read(len) -> returns data

• Pipe

• Subscribers must implement readable stream

• No more direct access with Pipe

@sh1mmer

@sh1mmer

Types of Streaming

@sh1mmer

Simplex

@sh1mmer

var fs = require('fs');

var rs = fs.createReadStream('/my/file/path');rs.on('data', function(d) { console.log(d);});

@sh1mmer

Throughput

@sh1mmer

var fs = require('fs');

var rs = fs.createReadStream('/my/file/path');rs.pipe(process.stdout);

@sh1mmer

Duplex

@sh1mmer

var bot1 = new chatterBot();var bot2 = new chatterBot();

bot1.pipe(bot2).pipe(bot1);

@sh1mmer

Libraries

@sh1mmer

JSONStream

@sh1mmer

{"total_rows":129,"offset":0,"rows":[ { "id":"change1_0.6995461115147918" , "key":"change1_0.6995461115147918" , "value":{"rev":"1-e240bae28c7bb3667f02760f6398d508"} , "doc":{ "_id": "change1_0.6995461115147918" , "_rev": "1-e240bae28c7bb3667f02760f6398d508","hello":1} }, { "id":"change2_0.6995461115147918" , "key":"change2_0.6995461115147918" , "value":{"rev":"1-13677d36b98c0c075145bb8975105153"} , "doc":{ "_id":"change2_0.6995461115147918" , "_rev":"1-13677d36b98c0c075145bb8975105153" , "hello":2 } },]}

@sh1mmer

var stream = JSONStream.parse(['rows', true, 'doc']) //rows, ANYTHING, doc

stream.on('data', function(data) { console.log('received:', data);});

stream.on('root', function(root, count) { if (!count) { console.log('no matches found:', root); }});

@sh1mmer

node-libexpat

@sh1mmer

Application

@sh1mmer

Questions?