|
<!DOCTYPE html> |
|
<html> |
|
<head> |
|
<title>Foo</title> |
|
<meta charset="utf-8" /> |
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0" /> |
|
<style type='text/css'> |
|
body { |
|
background:#000; |
|
color:#fff; |
|
font-family:'Georgia'; |
|
margin:0; |
|
} |
|
|
|
@-webkit-keyframes blinker { |
|
from { opacity: 1.0; } |
|
to { opacity: 0.0; } |
|
} |
|
|
|
em { |
|
-webkit-animation-name: blinker; |
|
-webkit-animation-iteration-count: infinite; |
|
-webkit-animation-timing-function: cubic-bezier(1.0,0,0,1.0); |
|
-webkit-animation-duration: 800ms; |
|
} |
|
|
|
strong { |
|
font-weight:normal; |
|
color:#FFF707; |
|
} |
|
|
|
a { |
|
color:#FFF707; |
|
text-decoration:none; |
|
} |
|
|
|
</style> |
|
<script type='text/javascript'> |
|
window.onload = function() { |
|
var s = document.getElementsByTagName('div'), cur = 0; |
|
if (!s) return; |
|
function go(n) { |
|
cur = n; |
|
var i = 1e3, e = s[n]; |
|
for (var k = 0; k < s.length; k++) s[k].style.display = 'none'; |
|
e.style.display = 'inline-block'; |
|
e.style.fontSize = i + 'px'; |
|
if (e.firstChild.nodeName === 'IMG') { |
|
document.body.style.backgroundImage = 'url(' + e.firstChild.src + ')'; |
|
e.firstChild.style.display = 'none'; |
|
} else { |
|
document.body.style.backgroundImage = ''; |
|
document.body.style.backgroundColor = e.style.backgroundColor; |
|
} |
|
while ( |
|
e.offsetWidth > window.innerWidth || |
|
e.offsetHeight > window.innerHeight) { |
|
e.style.fontSize = (i -= 10) + 'px'; |
|
if (i < 0) break; |
|
} |
|
e.style.marginTop = ((window.innerHeight - e.offsetHeight) / 2) + 'px'; |
|
if (window.location.hash !== n) window.location.hash = n; |
|
document.title = e.textContent || e.innerText; |
|
} |
|
document.onclick = function() { |
|
go(++cur % (s.length)); |
|
}; |
|
document.onkeydown = function(e) { |
|
(e.which === 39) && go(Math.min(s.length - 1, ++cur)); |
|
(e.which === 37) && go(Math.max(0, --cur)); |
|
}; |
|
function parse_hash() { |
|
return Math.max(Math.min( |
|
s.length - 1, |
|
parseInt(window.location.hash.substring(1), 10)), 0); |
|
} |
|
if (window.location.hash) cur = parse_hash() || cur; |
|
window.onhashchange = function() { |
|
var c = parse_hash(); |
|
if (c !== cur) go(c); |
|
}; |
|
|
|
go(cur); |
|
}; |
|
</script></head><body> |
|
<div><h1>A/B testing at Good Eggs</h1> |
|
<p>Bob Zoller <a href="mailto:bob@goodeggs.com">bob@goodeggs.com</a></p> |
|
</div> |
|
<div><h2><a href="http://en.wikipedia.org/wiki/Multi-armed_bandit">Multi-Armed Bandit</a> is cool.</h2> |
|
<p>Simultaneously gather new data and optimize decisions based on existing data.</p> |
|
<ul> |
|
<li><a href="http://genetify.com/demo/">Genetify</a></li> |
|
<li><a href="https://github.com/maccman/abba">Abba</a></li> |
|
</ul> |
|
</div> |
|
<div><h2>Learnings</h2> |
|
<ul> |
|
<li>Genetify is a super hacky POC written in the worst kind of PHP</li> |
|
<li>Multi-armed bandit requires an extra roundtrip to the server</li> |
|
<li>Abba doesn't support unique users</li> |
|
<li>both systems exist in a vaccuum</li> |
|
</ul> |
|
</div> |
|
<div><h2>Where we landed</h2> |
|
<p>A custom setup with:</p> |
|
<ul> |
|
<li>Abba-like client API</li> |
|
<li>choices stored in LocalStorage</li> |
|
<li>asynchronously registered as a MixPanel super property</li> |
|
<li>MixPanel as a reporting interface</li> |
|
</ul> |
|
</div> |
|
<div><h2>Usage</h2> |
|
<pre><code class="lang-coffeescript">experiment = metrics.experiment(<span class="comment">'1-2-3 header')</span> |
|
.variant(<span class="comment">'product promotion', 50)</span> |
|
.variant(<span class="comment">'1-2-3', 50)</span> |
|
.start() |
|
|
|
switch experiment.chosen.name |
|
when <span class="comment">'product promotion'</span> |
|
# <span class="keyword">do</span> something |
|
when <span class="comment">'1-2-3'</span> |
|
# <span class="keyword">do</span> something <span class="keyword">else</span></code></pre> |
|
</div> |
|
<div><h2>TODO</h2> |
|
<ul> |
|
<li>server-side experiments</li> |
|
<li>multi-armed bandit support</li> |
|
</ul> |
|
</div> |
|
<div><h2>Who's next?</h2> |
|
<ul> |
|
<li>Alex: Chrome profiler</li> |
|
<li>Brian/Adam: Kale preso</li> |
|
<li>Max: DTrace</li> |
|
</ul> |
|
</div> |