Skip to content

Instantly share code, notes, and snippets.

@lukemelia

lukemelia/report_open_experiments_to_yappie_job.rb

Created April 22, 2014 17:11
Show Gist options
  • Save lukemelia/11187118 to your computer and use it in GitHub Desktop.
Save lukemelia/11187118 to your computer and use it in GitHub Desktop.
Download ZIP
Reporting active A/B tests to Slack using Split, Sidekiq, Sidetiq
Raw
report_open_experiments_to_yappie_job.rb
class ReportOpenExperimentsToYappieJob
include Sidekiq::Worker
sidekiq_options :queue => :yappie
include Sidetiq::Schedulable
recurrence { daily.hour_of_day(9) }
def perform
active_experiments = Split::Experiment.all.select{ |exp| exp.winner.nil? }
active_experiments.each do |experiment|
attachments = []
experiment.alternatives.each do |alternative|
attachments.push({
fallback: "Experiment data not shown in this client.",
pretext: "Experiment status:",
color: alternative.to_s == experiment.winner.to_s ? "good" : "#222222",
fields: [
{
title: "Alternative",
value: alternative.name,
short: true
},
{
title: "Participants",
value: alternative.participant_count,
short: true
},
{
title: "Completed",
value: alternative.completed_count,
short: true
},
{
title: "Conversion Rate",
value: conversion_rate(alternative, experiment),
short: true
},
{
title: "Confidence",
value: confidence_level(alternative.z_score),
short: true
}
]
})
end
message = "Active experiment: #{experiment.name}"
puts message
puts attachments
Slack.new.post_message(message, attachments)
end
end
def conversion_rate(alternative, experiment)
[number_to_percentage(round(alternative.conversion_rate, 3)), diff_percent(experiment, alternative)].compact.join(' ')
end
def round(number, precision = 2)
BigDecimal.new(number.to_s).round(precision).to_f
end
def number_to_percentage(number, precision = 2)
"#{round(number * 100)}%"
end
def confidence_level(z_score)
return z_score if z_score.is_a? String
z = round(z_score.to_s.to_f, 3).abs
if z == 0.0
'No Change'
elsif z < 1.645
'no confidence'
elsif z < 1.96
'95% confidence'
elsif z < 2.57
'99% confidence'
else
'99.9% confidence'
end
end
def diff_percent(experiment, alternative)
if experiment.control.conversion_rate > 0 && !alternative.control?
if alternative.conversion_rate > experiment.control.conversion_rate
"(+#{number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1)})"
elsif alternative.conversion_rate < experiment.control.conversion_rate
"(#{number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1)})"
end
end
end
end
Raw
slack.rb
class Slack
def initialize(organization = "yapp", token = ENV['SLACK_TOKEN'])
@token = token
@url = "https://#{organization}.slack.com/services/hooks/incoming-webhook?token=#{token}"
end
def post_message(message, attachments = nil)
payload = {
text: message,
channel: "#metrics",
username: "yappie",
icon_url: "https://www.yapp.us/images/yappie_48.gif"
}
if attachments
payload[:attachments] = attachments
end
conn = Faraday.new(:url => @url) do |faraday|
faraday.request :url_encoded
faraday.response :logger
faraday.adapter Faraday.default_adapter
end
if !@token
Rails.logger.warn("No token was provided for slack posting, so we will skip it. Payload was: " + payload.to_json)
else
conn.post do |req|
req.headers['Content-Type'] = 'application/json'
req.body = payload.to_json
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment