Created
April 3, 2017 19:31
-
-
Save jenseng/62465f674f8c02de09ef776f23d4dca4 to your computer and use it in GitHub Desktop.
simplecov merging race condition
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env ruby | |
# This demonstrates a race condition in simplecov:HEAD that occurs when you | |
# have several processes merging results at the same time (e.g. test-queue, | |
# parallel_test). The bug manifests itself as either missing coverage, or | |
# outright failures. | |
# | |
# Usage: | |
# ruby dumb_test.rb | |
# | |
# # with oj | |
# ENABLE_OJ=1 ruby dumb_test.rb | |
# | |
# # make the race condition worse | |
# NUM_PROCESSES=10 UNSYNCHRONIZED_CALLS=5 ruby dumb_test.rb | |
# | |
# Basically there are 3 main problems: | |
# 1. `SimpleCov.result` doesn't cache the `@result`, so the default | |
# `at_exit` behavior causes it to store and merge 3 times. | |
# 2. `SimpleCov::ResultMerger.resultset` calls `.stored_data` twice | |
# 3. `SimpleCov::ResultMerger.merged_result` doesn't synchronize or | |
# cache `.resultset`, so a concurrent `.store_result` call can | |
# causes us to read an empty file | |
# | |
# This can cause the formatter to miss out on coverage data in our | |
# formatters and/or get the wrong values for covered percentages. | |
# | |
# Furthermore, if you use OJ, `JSON.parse("") -> nil`, which means | |
# `.resultset` can be nil, causing exceptions as seen in | |
# https://github.com/colszowka/simplecov/issues/406. | |
NUM_PROCESSES = ENV.fetch("NUM_PROCESSES", "5").to_i | |
ENABLE_OJ = ENV.fetch("ENABLE_OJ", "0").to_i | |
UNSYNCHRONIZED_CALLS = ENV.fetch("UNSYNCHRONIZED_CALLS", "2").to_i | |
if ENABLE_OJ == 1 | |
require "oj" | |
require 'oj_mimic_json' | |
end | |
require "simplecov" | |
require "simplecov/result_merger" | |
File.delete("coverage/.resultset.json") if File.exist?("coverage/.resultset.json") | |
SimpleCov::ResultMerger.singleton_class.prepend(Module.new { | |
def stored_data | |
return unless File.exist?(resultset_path) | |
data = File.read(resultset_path) | |
if data.nil? || data.length < 2 | |
puts "WARNING: bad .resultset.json! #{data.inspect}" | |
return | |
end | |
data | |
end | |
}) | |
pids = NUM_PROCESSES.times.map do |i| | |
fork do | |
data = { | |
i => { | |
"coverage" => Hash[ | |
1800.times.map { |j| ["file_#{j}.rb", 100.times.map { rand < 0.5 ? nil : 1 }] } | |
], | |
"timestamp" => Time.now.to_i | |
} | |
} | |
3.times do | |
SimpleCov::ResultMerger.store_result(data) | |
UNSYNCHRONIZED_CALLS.times do | |
puts "resultset is empty!" if SimpleCov::ResultMerger.resultset.size == 0 | |
end | |
end | |
puts "done #{i}" | |
end | |
end | |
pids.each do |pid| | |
Process.wait pid | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment