@sh1mmer
Using Node.js to Build Great
Streaming Services
Tom Hughes-Croucher@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
Scalable Server-Side Code with JavaScript
Tom Hughes-Croucher
NodeUp and Running
http://ofps.oreilly.com/titles/9781449398583/http://shop.oreilly.com/product/0636920015956.do
@sh1mmer
A story
@sh1mmer
factory
@sh1mmer
@sh1mmer
@sh1mmer
var http = require('http');
var server = http.createServer();server.listen(8000);server.on('request', function(req,res) { var doodad = new Doodad(); res.writeHead(200,{'Content-Type':'text/doodad'}); res.end(doodad);});
@sh1mmer
Making Doodads
@sh1mmer
Screens AntennasKnobs
& buttons
@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
var http = require('http'), fs = require('fs');
var server = http.createServer();server.listen(8000);server.on('request', function(req,res) { var screen = fs.readFileSync('/factory1/screen'); var dial = fs.readFileSync('/factory2/dial'); var doodad = new Doodad(screen, dial); res.writeHead(200,{'Content-Type':'text/doodad'}); res.end(doodad);});
@sh1mmer
Improving the factory with streaming
@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
var http = require('http');
var server = http.createServer();server.listen(8000);server.on('request', function(req,res) { var parts = 0; http.get('factory1/screen/', function(res) { var screen = ""; res.on('data', function(d) { screen += d }); res.on('end', finish); parts++; }; http.get('factory2/dial', function(res) { var dial = "" res.on('data', function(d) { dial += d }); res.on('end', finish); parts++; } function finish() { if(parts == 2) { var doodad = new Doodad(screen, dial); res.writeHead(200,{'Content-Type':'text/doodad'}); res.end(doodad); } }});
@sh1mmer
Streams API
@sh1mmer
• Readable
• 'data' event
• 'end' event
• pause()
• resume()
• destroy()
• pipe()
Streams
• Writable
• write()
• end()
• 'drain' event
• destroy()
• destroySoon()
@sh1mmer
var stream = require('stream');
var s = new stream.Stream(); //I am an EventEmitters.readable = true; //now I implement read API
s.emit('data', "my data");
s.emit('end'); //I'm done sending data
@sh1mmer
var stream = require('stream');
var s = new stream.Stream(); //I am an EventEmitters.writable = true; //now I implement write APIs.data = "";
s.write = function(d) { s.data += d;};
s.end = function() { if(s.data) { console.log(s.data) }};
s.destroy = function() { s.writable = false;}
@sh1mmer
The too many parts problem and why
phone calls don't work
@sh1mmer
Event Loop Fail
libuv
add-ons
libev iocp
v8
node.ccTCP Conn
TCP Conn
TCP Conn
nextTicknextTick
FS Read
FS Read
FS Read
Timeout
Timeout
Interval
Run stuff
stream.pause()
Event Loop Fail
libuv
add-ons
libev iocp
v8
node.ccTCP Conn
TCP Conn
TCP Conn
nextTicknextTick
FS Read
FS Read
FS Read
Timeout
Timeout
Interval
stream.pause()
@sh1mmer
Managing Backpressure
• Manage Buffer sizes
• Use pause() / resume() to control back pressure
• Assume that 'data' event may happen after pause
• Remember node buffer -> kernel buffer
@sh1mmer
//manage Node's write Buffersif(socket.bufferSize > maxSafe) { writeStream.pause();}
//Manage RS read stream Buffersfs.createReadStream('./path/to/file', { flags: 'r', encoding: null, fd: null, mode: 0666, bufferSize: 64 * 1024});
@sh1mmer
New API
New Streams TBD• Writable streams
• Same
• Readable (https://github.com/isaacs/readable-stream)
• configurable stream buffer waterline
• 'readable' event
• read(len) -> returns data
• Pipe
• Subscribers must implement readable stream
• No more direct access with Pipe
@sh1mmer
@sh1mmer
Types of Streaming
@sh1mmer
Simplex
@sh1mmer
var fs = require('fs');
var rs = fs.createReadStream('/my/file/path');rs.on('data', function(d) { console.log(d);});
@sh1mmer
Throughput
@sh1mmer
var fs = require('fs');
var rs = fs.createReadStream('/my/file/path');rs.pipe(process.stdout);
@sh1mmer
Duplex
@sh1mmer
var bot1 = new chatterBot();var bot2 = new chatterBot();
bot1.pipe(bot2).pipe(bot1);
@sh1mmer
Libraries
@sh1mmer
JSONStream
@sh1mmer
{"total_rows":129,"offset":0,"rows":[ { "id":"change1_0.6995461115147918" , "key":"change1_0.6995461115147918" , "value":{"rev":"1-e240bae28c7bb3667f02760f6398d508"} , "doc":{ "_id": "change1_0.6995461115147918" , "_rev": "1-e240bae28c7bb3667f02760f6398d508","hello":1} }, { "id":"change2_0.6995461115147918" , "key":"change2_0.6995461115147918" , "value":{"rev":"1-13677d36b98c0c075145bb8975105153"} , "doc":{ "_id":"change2_0.6995461115147918" , "_rev":"1-13677d36b98c0c075145bb8975105153" , "hello":2 } },]}
@sh1mmer
var stream = JSONStream.parse(['rows', true, 'doc']) //rows, ANYTHING, doc
stream.on('data', function(data) { console.log('received:', data);});
stream.on('root', function(root, count) { if (!count) { console.log('no matches found:', root); }});
@sh1mmer
node-libexpat
@sh1mmer
Application
@sh1mmer
Questions?