@theburningmonk

tame |> (cloud <| complexity) |> with |> (fsharp >> powered >> DSLs)

Principal Engineer

image by nerovivo license : https://creativecommons.org/licenses/by-sa/2.0/

@theburningmonk

Under-Abstraction

@theburningmonk

Oversimplification

@theburningmonk

Impedance Mismatch

@theburningmonk

Amazon DynamoDB

Amazon SimpleWorkflow

Amazon CloudWatch

CASE STUDY

@theburningmonk

F# DSLs. Awesome!

@theburningmonk

Amazon DynamoDB

Amazon SimpleW

Amazon CloudWatch

CASE STUDY

@theburningmonk

@theburningmonk

managedkey-value store

@theburningmonk

redundancy9-9s guarantee

@theburningmonk

great performance

@theburningmonk

name your throughput

@theburningmonk

@theburningmonk

can be changed on-the-fly

@theburningmonk

@theburningmonk

infinitely scalable(but you still have to pay for it)

@theburningmonk

Hash Key

Range Key

@theburningmonk

Query:given a hash keyfilter on

range key, orlocal secondary index

@theburningmonk

Hash Key Range Key

Local Secondary Index

Global Secondary Index

@theburningmonk

Scan:FULL TABLE search(performance + cost concern)

@theburningmonk

Hash Key Range Key

Local Secondary Index

Global Secondary Index

Hash Key Range Key

Local Secondary Index

Who are the TOP 3 players in “Starship X” with a score of at least 1000?

Global Secondary Index

@theburningmonk

@theburningmonk

select GameTitle, UserId, TopScore from GameScores where GameTitle = “Starship X” and TopScore >= 1000 order desc limit 3 with (NoConsistentRead, Index(GameTitleIndex, true))

DynamoDB.SQL

github.com/fsprojects/DynamoDb.SQL

@theburningmonk

GOAL

Disguise complexity

@theburningmonk

GOAL

Prevent abstraction leak

@theburningmonk

GOAL

SELECT UserId, TopScore FROM GameScore WHERE GameTitle CONTAINS “Zelda” ORDER DESCLIMIT 3 WITH (NoConsistentRead)

@theburningmonk

Query

AST

Execution

F# & FParsec*

*www.quanttec.com/fparsec

External DSL via

@theburningmonk

@theburningmonk

SELECT * FROM GameScore

Abstract Syntax Tree (AST)

FParsec

@theburningmonk

SELECT * FROM GameScore

keyword keyword

* | attribute, attribute, …

table name

@theburningmonk

SELECT * FROM GameScore

type Attributes = | Asterisk | Attributes of string[]

@theburningmonk

SELECT * FROM GameScore

type Query = { Attributes : Attributes Table : string }

@theburningmonk

SELECT * FROM GameScore

Parser for “SELECT” keyword

pSelect

@theburningmonk

SELECT * FROM GameScorepSelect

let pSelect = skipStringCI "select"

@theburningmonk

SELECT * FROM GameScorepSelect

let pSelect = skipStringCI "select"matches the string “select” (Case

Insensitive) and ignores it

@theburningmonk

SELECT * FROM GameScorepFrom

let pFrom = skipStringCI "from"

@theburningmonk

SELECT * FROM GameScore

Parser for a string that represents the table name

pTableName

@theburningmonk

SELECT * FROM GameScorepTableName

let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName

@theburningmonk

SELECT * FROM GameScorepTableName

let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName

@theburningmonk

SELECT * FROM GameScorepTableName

let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName

@theburningmonk

SELECT * FROM GameScorepTableName

let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName

@theburningmonk

SELECT * FROM GameScorepTableName

let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName

parses a sequence of one or more chars that satisfies the

predicate function

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeName

UserId, GameTitle, TopScore, …

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeName

UserId, GameTitle, TopScore, …

let pAsterisk = stringCIReturn "*" Asterisk

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeName

UserId, GameTitle, TopScore, …

let pAsterisk = stringCIReturn "*" Asteriskmatches the specified string and return the given value

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeName

UserId, GameTitle, TopScore, …

let isAttributeName = isLetter <||> isDigit let pAttributeName = many1Satisfy isAttributeName

@theburningmonk

SELECT * FROM GameScore

UserId, GameTitle, TopScore, …

pAttributeName pCommapAsterisk

*

let pComma = skipStringCI ","

@theburningmonk

SELECT * FROM GameScore

UserId, GameTitle, TopScore, …

pAttributeName pCommapAsterisk

*

let pAttributeNames = sepBy1 pAttributeName pComma

@theburningmonk

SELECT * FROM GameScore

UserId, GameTitle, TopScore, …

pAttributeName pCommapAsterisk

*

let pAttributeNames = sepBy1 pAttributeName pComma

parses one or more occurrences of pAttributeName separated by pComma

@theburningmonk

SELECT * FROM GameScore

UserId, GameTitle, TopScore, …

pAttributeName pComma

sepBy1

pAttributeNames

pAsterisk

*

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeNames

UserId, GameTitle, TopScore, …

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeNames

UserId, GameTitle, TopScore, …

let pAttribute = pAsterisk <|> pAttributeNames

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeNames

UserId, GameTitle, TopScore, …

let pAttribute = pAsterisk <|> pAttributeNames

@theburningmonk

SELECT * FROM GameScore

pAsterisk

*

pAttributeNames

UserId, GameTitle, TopScore, …

choice

pAttribute

@theburningmonk

SELECT * FROM GameScorepAttribute

@theburningmonk

SELECT * FROM GameScorepAttribute pTableNamepFrompSelect

@theburningmonk

SELECT * FROM GameScorepAttribute pTableNamepFrompSelect

let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })

@theburningmonk

SELECT * FROM GameScorepAttribute pTableNamepFrompSelect

let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })

@theburningmonk

SELECT * FROM GameScorepAttribute pTableNamepFrompSelect

let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })

@theburningmonk

SELECT * FROM GameScorepAttribute pTableNamepFrompSelect

let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })

Query

@theburningmonk

SELECT * FROM GameScorepAttribute pTableNamepFrompSelect

tuple4

pQuery

@theburningmonk

@theburningmonk

< 50 lines of code

@theburningmonk

Amazing

F# + FParsec =

@theburningmonk

Recap

@theburningmonk

@theburningmonk

select GameTitle, UserId, TopScore from GameScores where GameTitle = “Starship X” and TopScore >= 1000 order desc limit 3 with (NoConsistentRead, Index(GameTitleIndex, true))

@theburningmonk

Amazon DynamoDB

Amazon SimpleWorkflow

Amazon CloudWatch

CASE STUDY

@theburningmonk

@theburningmonk

Decision Worker

@theburningmonk

Decision WorkerPoll

@theburningmonk

Decision WorkerDecision

Task

@theburningmonk

Decision WorkerDecide

@theburningmonk

Activity Worker

Decision Worker

@theburningmonk

Activity Worker

Decision Worker

Poll

@theburningmonk

Activity Worker

Decision Worker

Activity Task

@theburningmonk

Activity Worker

Decision Worker

Complete

@theburningmonk

Workers can run from anywhere

@theburningmonk

@theburningmonk

input = “Yan”

result = “Hello Yan!”

Start

Finish

Activity

@theburningmonk

image by Ryan Hageman license : https://creativecommons.org/licenses/by-sa/2.0/

SWF-based Application

API

Heartbeats

Error Handling

Polling

API

Activity Worker

Decision Worker

Heartbeats

Error Handling

Polling

API

Activity Worker

Decision Worker

Heartbeats

Error Handling

Polling

API

Boilerplate

– Kris Jordan

“Good simplicity is less with leverage, not less with less. Good simplicity is complexity

disguised, not complexity denied.”

http://bit.ly/1pOLeKl

Start

Finish

Activity?

@theburningmonk

the workflow is implied by decision

worker logic…

@theburningmonk

instead..

@theburningmonk

the workflow should drive decision worker logic

@theburningmonk

the workflow should driveautomate

decision worker logic

Amazon.SimpleWorkflow.Extensions

github.com/fsprojects/Amazon.SimpleWorkflow.Extensions

@theburningmonk

GOAL

Remove boilerplates

@theburningmonk

GOAL

Code that matches the way you think

@theburningmonk

Start

Finish

Activity

Start

Finish

Activity

@theburningmonk

Workflows can be nested

input

result

@theburningmonk

Recap

Activity Worker

Decision Worker

Heartbeats

Error Handling

Polling

API

@theburningmonk

Amazon DynamoDB

Amazon SimpleW

Amazon CloudWatch

CASE STUDY

@theburningmonk

@theburningmonk

@theburningmonk

@theburningmonk

wanna find correlations?

@theburningmonk

wanna find correlations?

you can DIY it!

;-)

@theburningmonk

“what latencies spiked at the same time as payment service?”

Amazon.CloudWatch.Selector

github.com/fsprojects/Amazon.CloudWatch.Selector

@theburningmonk

Find metrics whose 5 min average exceeded

1 second during last 12 hours

@theburningmonk

cloudWatch.Select( unitIs “milliseconds” + average (>) 1000.0 @ last 12 hours |> intervalOf 5 minutes)

@theburningmonk

cloudWatch.Select(“ unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes”)

@theburningmonk

“did any cache nodes’ CPU spike

yesterday?”

@theburningmonk

cloudWatch.Select( namespaceLike “elasticache” + nameLike “cpu” + max (>) 80.0 @ last 24 hours |> intervalOf 15 minutes)

@theburningmonk

cloudWatch.Select( namespaceLike “elasticache” + nameLike “cpu” + max (>) 80.0 @ last 24 hours |> intervalOf 15 minutes)

Regex

@theburningmonk

@theburningmonk

@theburningmonk

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

Filters

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

TimeFrame

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

Period

@theburningmonk

type Query = { Filter : Filter TimeFrame : TimeFrame Period : Period option }

@theburningmonk

Query

Internal DSL

External DSL

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

@theburningmonk

type MetricTerm = Namespace | Name

type Filter = | MetricFilter of MetricTerm * (string -> bool)

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

@theburningmonk

type MetricTerm = Namespace | Name

type Unit = | Unit

type Filter = | MetricFilter of MetricTerm * (string -> bool) | UnitFilter of Unit * (string -> bool)

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

@theburningmonk

type MetricTerm = Namespace | Name

type Unit = | Unit

type StatsTerm = | Average | Min | Max | Sum | SampleCount

type Filter = | MetricFilter of MetricTerm * (string -> bool) | UnitFilter of Unit * (string -> bool) | StatsFilter of StatsTerm * (float -> bool)

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

@theburningmonk

type MetricTerm = Namespace | Name

type Unit = | Unit

type StatsTerm = | Average | Min | Max | Sum | SampleCount

type Filter = | MetricFilter of MetricTerm * (string -> bool) | UnitFilter of Unit * (string -> bool) | StatsFilter of StatsTerm * (float -> bool) | CompositeFilter of Filter * Filter

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

@theburningmonk

type TimeFrame = | Last of TimeSpan | Since of DateTime | Between of DateTime * DateTime

@theburningmonk

namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes

@theburningmonk

type Period = | Period of TimeSpan

@theburningmonk

Active Patternsa primer on

@theburningmonk

allow patterns to be abstracted away into

named functions

@theburningmonk

Single-Case Patterns

@theburningmonk

let (|Float|) input = match Double.TryParse input with | true, n -> n | _ -> failwithf “not a float [%s]” input

@theburningmonk

let (|Float|) input = match Double.TryParse input with | true, n -> n | _ -> failwithf “not a float [%s]” input

@theburningmonk

let (|Float|) input = match Double.TryParse input with | true, n -> n | _ -> failwithf “not a float [%s]” input

Float : string -> float

@theburningmonk

match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x

@theburningmonk

let (|Float|) input = match Double.TryParse input with | true, n -> n

match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x

@theburningmonk

let (|Float|) input = match Double.TryParse input with | true, n -> n

match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x

@theburningmonk

match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x

let (|Float|) input = match Double.TryParse input with | true, n -> n

@theburningmonk

match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x

let (|Float|) input = match Double.TryParse input with | true, n -> n

@theburningmonk

match “42” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x

@theburningmonk

match “boo” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x

Error!!!

@theburningmonk

Partial Patterns

@theburningmonk

let (|Float|_|) input = match Double.TryParse input with | true, n -> Some n | _ -> None

@theburningmonk

let (|Float|_|) input = match Double.TryParse input with | true, n -> Some n | _ -> None

Float : string -> float option

@theburningmonk

match “boo” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x | _ -> “not a float”

@theburningmonk

match “boo” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x | _ -> “not a float”

@theburningmonk

Multi-Case Patterns

@theburningmonk

let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN

@theburningmonk

let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN

@theburningmonk

let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN

@theburningmonk

let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN

@theburningmonk

let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN

@theburningmonk

match someString with | Prime n -> … | NotPrime n -> … | NaN -> …

@theburningmonk

match someString with | Prime n & Float 11.0 -> … | Prime n -> … | Float 42.0 | Float 8.0 -> … | NotPrime n -> … | NaN -> …

@theburningmonk

match someString with | Prime n & Float 11.0 -> … | Prime n -> … | Float 42.0 | Float 8.0 -> … | NotPrime n -> … | NaN -> …

@theburningmonk

match someString with | Prime (IsPalindrome n) -> … | Prime (IsEven n) -> … | _ -> …

@theburningmonk

Tokenise(string -> string list)

@theburningmonk

[ “namespaceIs”; “‘JustEat’”; “and”; “nameLike”; “‘cpu’”; “and”; … ]

@theburningmonk

Visual Studio time…

@theburningmonk

Parse Filter(string list -> Filter * string list)

@theburningmonk

let rec loop acc = function | NamespaceIs (filter, tl) | NamespaceLike (filter, tl) | NameIs (filter, tl) | NameLike (filter, tl) | UnitIs (filter, tl) | Average (filter, tl) | Sum (filter, tl) | Min (filter, tl) | Max (filter, tl) | SampleCount (filter, tl) -> match tl with | And tl -> loop (filter::acc) tl | _ -> flatten (filter::acc), tl | _ -> failwith “No filters?!?!?”

@theburningmonk

let (|NamespaceIs|_|) = function | StringCI "NamespaceIs"::QuotedString ns::tl -> (eqFilter MetricFilter Namespace ns, tl) |> Some | _ -> None

let (|NamespaceLike|_|) = function | StringCI "NamespaceLike"::QuotedString pattern::tl -> (regexFilter MetricFilter Namespace pattern, tl) |> Some | _ -> None

@theburningmonk

Parse(string list -> Query)

@theburningmonk

let parse (input : string) = input |> tokenize |> parseFilter |> parseTimeFrame |> parsePeriod

@theburningmonk

Amazing

F# =

@theburningmonk

usable from anywhere you can run F# code

e.g. F# REPL, executable, ..

Internal DSL

@theburningmonk

useful for building tools e.g. CLI, …

External DSL

@theburningmonk

@theburningmonk

@theburningmonk

@theburningmonk

Recap

@theburningmonk

@theburningmonk

@theburningmonk

Amazon DynamoDB

Amazon SimpleWorkflow

Amazon CloudWatch

CASE STUDY

@theburningmonktheburningmonk.comgithub.com/theburningmonk

is hiring :-)http://tech.just-eat.com/jobs