Scaling A/B testing on Netflix.com with

_________Alex Liu @stinkydofu

data driven product development

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

A

B

C

D

E

F

G

Test 1 Test 2 Test 3 Test 4 Test 5 Test 6 Test 7

2,097,152 unique experiences across seven tests

hundreds of new A/B tests per year

433518929550349486086117218185493567650…72061153709996

2105 566 685templates CSS JS

2.5M unique packages every week

problem: conditional dependencies

Packaging

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

app.js

import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch';

export ...

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

685 files…?

2.5M packages…?

oldSearch

app.js

newSearch

dep1 dep2 dep3 dep4 dep5

sub-dep sub-depsub-dep sub-dep sub-dep sub-dep

problem: conditional dependencies

requestbuild

require('nf-include-when')

/* * @includewhen rule.notInNewSearch */

oldSearch.js

/* * @includewhen rule.inNewSearch */

newSearch.js

var Rule = require('nf-rule-infrastructure'), inNewSearch;

inNewSearch = new Rule('inNewSearch', function(context, cb) { var test = context.abtests.get(1534); cb(test && test.cell(1)); });

module.exports = inNewSearch;

anatomy of a rule

require('nf-asset-registry')

import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch';

export ...

app.js

newSearch.js

jquery

oldSearch.js

app.js

registry

"app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ], "fileSize": "4.41 kB", "fileSizeFull": "120.52 kB" }

"newSearch.js": { "rule": "inNewSearch", "deps": [ "jquery", "newSearchDep2.js", "newSearchDep1.js", ], "depsFull": [ "jquery", "newSearchSubDep3.js", "newSearchSubDep2.js" "newSearchSubDep1.js" "newSearchDep2.js", "newSearchDep1.js" ], "fileSize": "10.41 kB", "fileSizeFull": "40.52 kB" }

nf-include-when

require('nf-packager')

var packager = require('nf-packager'), includeWhen = require('nf-include-when'), registries = require('nf-asset-registry');

function getScriptUrl() return packager.getPackageDefinition('app.js', registries, includeWhen); }

"app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ], "fileSize": "4.41 kB", "fileSizeFull": "120.52 kB" }

Step 1: Get the full dependency tree for the requested package from the registry.

[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

Step 2: Determine which files have rules.

[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

Step 3: Run the rules. Filter out all deps that resolved false.

✓

[ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

Step 4: Filter out all extraneous sub deps.

✓

Step 5: Concatenate the files.

[ "jquery", /* no rule */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]

buildjavascript

registry

request registry

rulespackager

Bonus Round

be creative with the registry

"account/bb/models/ratingHistoryModel.js": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": { "underscore": 2, "backbone": 1, "jquery": 2, "common/requirejs-plugins.js": 4, "requirejs-text": 4, "utils/contextData.js": 1, "common/nfNamespace.js": 1 }, "hash": "dd23b163", "fileSize": "1.21 kB", "fileSizeFull": "173.04 kB" }

dependency counting

dependency pruning

file sizes

@import (reference) "/common/_nf_defs.less"; @import (reference) "/member/memberCore.less"; @import (reference) "/components/menu.less"; @import (reference) "/components/breadcrumbs.less";

@import modules

"account/containerResponsive.css": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": [...], "hash": "65a431f3", "fileSize": "709 B", "fileSizeFull": "709 B", "css": { "selectors": 8, "declarationBlocks": 6, "declarations": 17, "mediaQueries": 3 } }

css analysis

the

best part

"cache": { "account/pin.js": "define('account/pin.js', ['member/memberC…", "account/bb/models/changePlanModel.js": "define('account/b…", "account/bb/models/ratingHistoryModel.js": "define('account…", "account/bb/models/viewingActivityModel.js": "define('account…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/emailSubView.js": "define('account/bb/views…", "account/bb/views/viewingActivityView.js": "define('account…", "common/UITracking.js": "define('common/UITracking.js, ['me…", "common/UITrackingOverlay.js": "define('common/UITrackingOve…", … … …

css

mappings

javascript

templates templates

mappings

javascript

css

templates

mappings

javascript

css

UI Bundle

deploy UI bundles

anytime

never touch the file system

< 5ms package response times

Takeaways▶ leverage the server ▶ static analysis FTW ▶ divide and conquer with modules

Our Learnings

learn by doing

fail fastmove faster

“I have not failed.I’ve just found 10,000 waysthat won’t work.”

Thomas Edison

simplify

thanks! come find us!

Alex Liu @stinkydofu

Chris Saint-Amant @csaintamant

Micah Ransdell @mjr578

Kris Baxter @kristoferbaxter