Skip to content

Instantly share code, notes, and snippets.

@bwesterb
Created October 20, 2022 08:42
Show Gist options
  • Save bwesterb/db8083608aeb4161021a60eeeb84fe71 to your computer and use it in GitHub Desktop.
Save bwesterb/db8083608aeb4161021a60eeeb84fe71 to your computer and use it in GitHub Desktop.
import pprint
from math import sqrt
import scipy.stats
def parse(x):
def c(x):
try:
return int(x.split('cycles')[0].replace(',',''))
except:
return None
return [c(x) for x in x.split('median') if c(x) is not None]
def parsefile(x):
with open(x) as f:
txt = f.read()
ret = []
for bm in txt.split('Benchmarking'):
if not bm:
continue
what, rest = bm.split('make', 1)
what = what.strip()
for line in rest.split('\n'):
if 'median' not in line:
continue
action, rest2 = line.split('avg')
action = action.replace('-', '').replace('.', '').strip()
ret.append((what + ' ' + action, int(rest2.split('cycles')[0].split('median')[1].replace(',', ''))))
return ret
old = {}
new = {}
for key, cycles in parsefile(f'old'):
if key not in old:
old[key] = []
old[key].append(cycles)
for key, cycles in parsefile(f'new'):
if key not in new:
new[key] = []
new[key].append(cycles)
def avg(xs):
return sum(xs)/len(xs)
scores = []
for what in old:
old_min = min(old[what])
new_min = min(new[what])
score = abs((new_min / old_min - 1)*100)
scores.append((what, score, new_min/old_min-1,old[what], new[what]))
scores.sort(key=lambda x: -x[1])
for v in scores:
if not any(x in v[0] for x in ('Signing', 'Generating', 'Verifying')):
continue
stats = scipy.stats.mannwhitneyu(v[3], v[4])
diff = avg(v[4]) - avg(v[3])
if stats.pvalue > 0.05:
continue
print()
print(v[0])
print(f"{diff/avg(v[3])*100:.3f}% difference with p={stats.pvalue:.5f}")
print(' old:', v[3])
print(' new:', v[4])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment