building real time systems on mongodb using the oplog at stripe

MongoDB and the OplogEVAN BRODER @ebroder

AGENDAINTRO TO THE OPLOGEXAMPLE APPLICATIONS

INTROTO THE OPLOG

PRIMARY

SECONDARIES

APPLICATION

APPLICATION

save{_id: 1, a: 2}

THINGS I’VE DONE:

- save {_id: 1, a: 2}

APPLICATION

update where{a: 2},{$set: {a: 3}}

THINGS I’VE DONE:

- save {_id: 1, a: 2}- update {_id: 1}, {$set: {a: 3}}

THINGS I’VE DONE:

- save {_id: 1, a: 2}- update {_id: 1}, {$set: {a: 3}}- insert…- delete…- delete…- save…- update…

THINGS I’VE DONE:

save…

THINGS I’VE DONE:

THINGS I’VE DONE:

save…

THINGS I’VE DONE:

save…

TRIGGERS

GOAL:EVENT PROCESSING

GOAL:DETECT INSERTIONS

WARNINGTHIS CODE IS NOT PRODUCTION-READY

oplog = mongo_connection['local']['oplog.rs']

ns = 'eventdb.events'

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.each do |op|

puts op['o']['_id']

end

end

oplog = mongo_connection['local']['oplog.rs']

ns = 'eventdb.events'

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.each do |op|

puts op['o']['_id']

end

end

oplog = mongo_connection['local']['oplog.rs']

ns = 'eventdb.events'

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.each do |op|

puts op['o']['_id']

end

end

oplog = mongo_connection['local']['oplog.rs']

ns = 'eventdb.events'

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.each do |op|

puts op['o']['_id']

end

end

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

start_entry = oplog.find_one({},

{:sort => {'$natural' => -1}})

start = start_entry['ts']

oplog.find({'ts' => {'$gt' => start},

'op' => 'i',

'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

start_entry = oplog.find_one({},

{:sort => {'$natural' => -1}})

start = start_entry['ts']

oplog.find({'ts' => {'$gt' => start},

'op' => 'i',

'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

start_entry = oplog.find_one({},

{:sort => {'$natural' => -1}})

start = start_entry['ts']

oplog.find({'ts' => {'$gt' => start},

'op' => 'i',

'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

start_entry = oplog.find_one({},

{:sort => {'$natural' => -1}})

start = start_entry['ts']

oplog.find({'ts' => {'$gt' => start},

'op' => 'i',

'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

DATA TRANSFORMATIONS

GOAL:MONGODB TO POSTGRESQL

start_entry = oplog.find_one({},

{:sort => {'$natural' => -1}})

start = start_entry['ts']

oplog.find({'ts' => {'$gt' => start},

'op' => 'i',

'ns' => ns}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

start_entry = oplog.find_one({},

{:sort => {'$natural' => -1}})

start = start_entry['ts']

oplog.find({'ts' => {'$gt' => start}}) do |cursor|

cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)

cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)

cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)

loop do

cursor.each do |op|

puts op['o']['_id']

end

end

end

cursor.each do |op|

puts op['o']['_id']

end

cursor.each do |op|

case op['op']

when 'i'

puts op['o']['_id']

else

# ¯\_(ツ)_/¯

end

end

cursor.each do |op|

case op['op']

when 'i'

query = "INSERT INTO #{op['ns']} (" +

op['o'].keys.join(', ') +

') VALUES (' +

op['o'].values.map(&:inspect).join(', ') + ')'

else

# ¯\_(ツ)_/¯

end

end

cursor.each do |op|

case op['op']

when 'i'

query = "INSERT INTO #{op['ns']} (" +

op['o'].keys.join(', ') +

') VALUES (' +

op['o'].values.map(&:inspect).join(', ') + ')'

when 'd'

query = "DELETE FROM #{op['ns']} WHERE _id=" +

op['o']['_id'].inspect

else

# ¯\_(ツ)_/¯

end

end

query = "DELETE FROM #{op['ns']} WHERE _id=" +

op['o']['_id'].inspect

when 'u'

query = "UPDATE #{op['ns']} SET"

updates = op['o']['$set'] ? op['o']['$set'] : op['o']

updates.each do |k, v|

query += " #{k}=#{v.inspect}"

end

query += " WHERE _id="

query += op['o2']['_id'].inspect

else

# ¯\_(ツ)_/¯

end

end

cursor.each do |op|

case op['op']

when 'i'

query = "INSERT INTO #{op['ns']} (" +

op['o'].keys.join(', ') + ') VALUES (' +

op['o'].values.map(&:inspect).join(', ') + ')'

when 'd'

query = "DELETE FROM #{op['ns']} WHERE _id=" +

op['o']['_id'].inspect

when 'u'

query = "UPDATE #{op['ns']} SET"

updates = op['o']['$set'] ? op['o']['$set'] : op['o']

updates.each do |k, v|

query += " #{k}=#{v.inspect}"

end

query += " WHERE _id=" + op['o2']['_id'].inspect

else

# ¯\_(ツ)_/¯

end

end

github.com/stripe/mosql

github.com/stripe/zerowing

cursor.each do |op|

case op['op']

when 'i'

query = "INSERT INTO #{op['ns']} (" +

op['o'].keys.join(', ') + ') VALUES (' +

op['o'].values.map(&:inspect).join(', ') + ')'

when 'd'

query = "DELETE FROM #{op['ns']} WHERE _id=" +

op['o']['_id'].inspect

when 'u'

query = "UPDATE #{op['ns']} SET"

updates = op['o']['$set'] ? op['o']['$set'] : op['o']

updates.each do |k, v|

query += " #{k}=#{v.inspect}"

end

query += " WHERE _id=" + op['o2']['_id'].inspect

else

# ¯\_(ツ)_/¯

end

end

DISASTER RECOVERY

task = collection.find_one({'finished' => nil}

# do something with task…

collection.update({'_id' => task.id},

{'$set' => {'finished' => Time.now.to_i}})

loop do

collection.remove(

{'finished' => {'$lt' => Time.now.to_i - 30}})

sleep(10)

end

evan@caron:~$ mongo

MongoDB shell version: 2.4.10

connecting to: test

normal:PRIMARY> null < (Date.now() / 1000) - 30

true

THINGS I’VE DONE:

insertdelete…

THINGS I’VE DONE:

> db.getReplicationInfo()

{

"logSizeMB" : 48964.3541015625,

"usedMB" : 46116.4,

"timeDiff" : 316550,

"timeDiffHours" : 87.93,

"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",

"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",

"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"

}

> db.getReplicationInfo()

{

"logSizeMB" : 48964.3541015625,

"usedMB" : 46116.4,

"timeDiff" : 316550,

"timeDiffHours" : 87.93,

"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",

"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",

"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"

}

new_oplog.find({'ts' => {'$gt' => start}}) do |cursor| cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) cursor.each do |op| if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks' old_task = old_tasks.find_one({'_id' => op['o']['_id']}) if old_task['finished'] == nil # found one! # save old_task to a file, and we'll re-queue it later end end

old_connection['admin'].command({'applyOps' => [op]}) endend

new_oplog.find({'ts' => {'$gt' => start}}) do |cursor| cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) cursor.each do |op| if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks' old_task = old_tasks.find_one({'_id' => op['o']['_id']}) if old_task['finished'] == false # found one! # save old_task to a file, and we'll re-queue it later end end

old_connection['admin'].command({'applyOps' => [op]}) endend

new_oplog.find({'ts' => {'$gt' => start}}) do |cursor| cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) cursor.each do |op| if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks' old_task = old_tasks.find_one({'_id' => op['o']['_id']}) if old_task['finished'] == false # found one! # save old_task to a file, and we'll re-queue it later end end

old_connection['admin'].command({'applyOps' => [op]}) endend

THINGS I’VE DONE:

save…

THINGS I’VE DONE:

save…

QUESTIONS?