Skip to content

Instantly share code, notes, and snippets.

@timfel

timfel/enaml_like_benchmark.py

Created July 7, 2022 09:27
Show Gist options
  • Save timfel/ebd12cb62f80d447248c615d9632bb10 to your computer and use it in GitHub Desktop.
Save timfel/ebd12cb62f80d447248c615d9632bb10 to your computer and use it in GitHub Desktop.
Download ZIP
Benchmark support for matplotlib and kiwi hpy ports
Raw
enaml_like_benchmark.py
# ------------------------------------------------------------------------------
# Copyright (c) 2019, Nucleic Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ------------------------------------------------------------------------------
"""Time updating an EditVariable in a set of constraints typical of enaml use.
"""
import pyperf as perf
from kiwisolver import Variable, Solver, strength
def benchmark_setup():
solver = Solver()
# Create custom strength
mmedium = strength.create(0, 1, 0, 1.25)
smedium = strength.create(0, 100, 0)
# Create the variable
left = Variable("left")
height = Variable("height")
top = Variable("top")
width = Variable("width")
contents_top = Variable("contents_top")
contents_bottom = Variable("contents_bottom")
contents_left = Variable("contents_left")
contents_right = Variable("contents_right")
midline = Variable("midline")
ctleft = Variable("ctleft")
ctheight = Variable("ctheight")
cttop = Variable("cttop")
ctwidth = Variable("ctwidth")
lb1left = Variable("lb1left")
lb1height = Variable("lb1height")
lb1top = Variable("lb1top")
lb1width = Variable("lb1width")
lb2left = Variable("lb2left")
lb2height = Variable("lb2height")
lb2top = Variable("lb2top")
lb2width = Variable("lb2width")
lb3left = Variable("lb3left")
lb3height = Variable("lb3height")
lb3top = Variable("lb3top")
lb3width = Variable("lb3width")
fl1left = Variable("fl1left")
fl1height = Variable("fl1height")
fl1top = Variable("fl1top")
fl1width = Variable("fl1width")
fl2left = Variable("fl2left")
fl2height = Variable("fl2height")
fl2top = Variable("fl2top")
fl2width = Variable("fl2width")
fl3left = Variable("fl3left")
fl3height = Variable("fl3height")
fl3top = Variable("fl3top")
fl3width = Variable("fl3width")
# Add the edit variables
solver.addEditVariable(width, 'strong')
solver.addEditVariable(height, 'strong')
# Add the constraints
for c in [(left + -0 >= 0) | "required",
(height + 0 == 0) | "medium",
(top + -0 >= 0) | "required",
(width + -0 >= 0) | "required",
(height + -0 >= 0) | "required",
(- top + contents_top + -10 == 0) | "required",
(lb3height + -16 == 0) | "strong",
(lb3height + -16 >= 0) | "strong",
(ctleft + -0 >= 0) | "required",
(cttop + -0 >= 0) | "required",
(ctwidth + -0 >= 0) | "required",
(ctheight + -0 >= 0) | "required",
(fl3left + -0 >= 0) | "required",
(ctheight + -24 >= 0) | smedium,
(ctwidth + -1.67772e+07 <= 0) | smedium,
(ctheight + -24 <= 0) | smedium,
(fl3top + -0 >= 0) | "required",
(fl3width + -0 >= 0) | "required",
(fl3height + -0 >= 0) | "required",
(lb1width + -67 == 0) | "weak",
(lb2width + -0 >= 0) | "required",
(lb2height + -0 >= 0) | "required",
(fl2height + -0 >= 0) | "required",
(lb3left + -0 >= 0) | "required",
(fl2width + -125 >= 0) | "strong",
(fl2height + -21 == 0) | "strong",
(fl2height + -21 >= 0) | "strong",
(lb3top + -0 >= 0) | "required",
(lb3width + -0 >= 0) | "required",
(fl1left + -0 >= 0) | "required",
(fl1width + -0 >= 0) | "required",
(lb1width + -67 >= 0) | "strong",
(fl2left + -0 >= 0) | "required",
(lb2width + -66 == 0) | "weak",
(lb2width + -66 >= 0) | "strong",
(lb2height + -16 == 0) | "strong",
(fl1height + -0 >= 0) | "required",
(fl1top + -0 >= 0) | "required",
(lb2top + -0 >= 0) | "required",
(- lb2top + lb3top + - lb2height + -10 == 0) | mmedium,
(- lb3top + - lb3height + fl3top + -10 >= 0) | "required",
(- lb3top + - lb3height + fl3top + -10 == 0) | mmedium,
(contents_bottom + - fl3height + - fl3top + -0 == 0) | mmedium,
(fl1top + - contents_top + 0 >= 0) | "required",
(fl1top + - contents_top + 0 == 0) | mmedium,
(contents_bottom + - fl3height + - fl3top + -0 >= 0) | "required",
(- left + - width + contents_right + 10 == 0) | "required",
(- top + - height + contents_bottom + 10 == 0) | "required",
(- left + contents_left + -10 == 0) | "required",
(lb3left + - contents_left + 0 == 0) | mmedium,
(fl1left + - midline + 0 == 0) | "strong",
(fl2left + - midline + 0 == 0) | "strong",
(ctleft + - midline + 0 == 0) | "strong",
(fl1top + 0.5 * fl1height + - lb1top + -0.5 * lb1height + 0 == 0) | "strong",
(lb1left + - contents_left + 0 >= 0) | "required",
(lb1left + - contents_left + 0 == 0) | mmedium,
(- lb1left + fl1left + - lb1width + -10 >= 0) | "required",
(- lb1left + fl1left + - lb1width + -10 == 0) | mmedium,
(- fl1left + contents_right + - fl1width + -0 >= 0) | "required",
(width + 0 == 0) | "medium",
(- fl1top + fl2top + - fl1height + -10 >= 0) | "required",
(- fl1top + fl2top + - fl1height + -10 == 0) | mmedium,
(cttop + - fl2top + - fl2height + -10 >= 0) | "required",
(- ctheight + - cttop + fl3top + -10 >= 0) | "required",
(contents_bottom + - fl3height + - fl3top + -0 >= 0) | "required",
(cttop + - fl2top + - fl2height + -10 == 0) | mmedium,
(- fl1left + contents_right + - fl1width + -0 == 0) | mmedium,
(- lb2top + -0.5 * lb2height + fl2top + 0.5 * fl2height + 0 == 0) | "strong",
(- contents_left + lb2left + 0 >= 0) | "required",
(- contents_left + lb2left + 0 == 0) | mmedium,
(fl2left + - lb2width + - lb2left + -10 >= 0) | "required",
(- ctheight + - cttop + fl3top + -10 == 0) | mmedium,
(contents_bottom + - fl3height + - fl3top + -0 == 0) | mmedium,
(lb1top + -0 >= 0) | "required",
(lb1width + -0 >= 0) | "required",
(lb1height + -0 >= 0) | "required",
(fl2left + - lb2width + - lb2left + -10 == 0) | mmedium,
(- fl2left + - fl2width + contents_right + -0 == 0) | mmedium,
(- fl2left + - fl2width + contents_right + -0 >= 0) | "required",
(lb3left + - contents_left + 0 >= 0) | "required",
(lb1left + -0 >= 0) | "required",
(0.5 * ctheight + cttop + - lb3top + -0.5 * lb3height + 0 == 0) | "strong",
(ctleft + - lb3left + - lb3width + -10 >= 0) | "required",
(- ctwidth + - ctleft + contents_right + -0 >= 0) | "required",
(ctleft + - lb3left + - lb3width + -10 == 0) | mmedium,
(fl3left + - contents_left + 0 >= 0) | "required",
(fl3left + - contents_left + 0 == 0) | mmedium,
(- ctwidth + - ctleft + contents_right + -0 == 0) | mmedium,
(- fl3left + contents_right + - fl3width + -0 == 0) | mmedium,
(- contents_top + lb1top + 0 >= 0) | "required",
(- contents_top + lb1top + 0 == 0) | mmedium,
(- fl3left + contents_right + - fl3width + -0 >= 0) | "required",
(lb2top + - lb1top + - lb1height + -10 >= 0) | "required",
(- lb2top + lb3top + - lb2height + -10 >= 0) | "required",
(lb2top + - lb1top + - lb1height + -10 == 0) | mmedium,
(fl1height + -21 == 0) | "strong",
(fl1height + -21 >= 0) | "strong",
(lb2left + -0 >= 0) | "required",
(lb2height + -16 >= 0) | "strong",
(fl2top + -0 >= 0) | "required",
(fl2width + -0 >= 0) | "required",
(lb1height + -16 >= 0) | "strong",
(lb1height + -16 == 0) | "strong",
(fl3width + -125 >= 0) | "strong",
(fl3height + -21 == 0) | "strong",
(fl3height + -21 >= 0) | "strong",
(lb3height + -0 >= 0) | "required",
(ctwidth + -119 >= 0) | smedium,
(lb3width + -24 == 0) | "weak",
(lb3width + -24 >= 0) | "strong",
(fl1width + -125 >= 0) | "strong",
]:
solver.addConstraint(c)
return solver, width, height
def bench_update_variables(solver, width, height):
"""Suggest new values and update variables.
This mimic the use of kiwi in enaml in the case of a resizing.
"""
t0 = perf.perf_counter()
for w, h in [
(400, 600),
(600, 400),
(800, 1200),
(1200, 800),
(400, 800),
(800, 400),
]:
solver.suggestValue(width, w)
solver.suggestValue(height, h)
solver.updateVariables()
return perf.perf_counter() - t0
runner = perf.Runner()
runner.timeit(
"kiwi.suggestValue",
setup="solver, width, height = setup()",
globals={"setup": benchmark_setup, "bench": bench_update_variables},
stmt="bench(solver, width, height)",
)
# for i in range(200):
# solver, width, height = benchmark_setup()
# bench_update_variables(solver, width, height)
Raw
mpl-bench-pyperf-run.py
# a simple script to be put into the root dir of https://github.com/matplotlib/mpl-bench/
# so we can run the mpl-bench benchmarks with pyperf
import sys
import os
import pyperf
import json
import argparse
import importlib
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-b', action='append')
args, rem = parser.parse_known_args()
benchmarks = []
program_args = [sys.argv[0]]
for b in args.b:
program_args += ["-b", b]
benchmarks.append(b)
runner = pyperf.Runner(program_args=program_args)
runner.parse_args(rem)
for b in benchmarks:
name = b
while name:
name = name.rpartition(".")[0]
try:
module = importlib.import_module(f"benchmarks.{name}")
except:
print(f"Got exception importing benchmarks.{name}")
else:
break
benchmark = b[len(name) + 1:]
benchmark_owner = module
while "." in benchmark:
attr, _, benchmark = benchmark.partition(".")
benchmark_owner = getattr(benchmark_owner, attr)
if type(benchmark_owner) == type:
benchmark_owner = benchmark_owner()
if benchmark == "*":
for k in benchmark_owner.__dict__:
if k.startswith("time_"):
benchmarks.append(b.replace("*", k))
continue
params = []
if hasattr(benchmark_owner, "params"):
params = benchmark_owner.params
elif hasattr(getattr(benchmark_owner, benchmark), "params"):
params = getattr(benchmark_owner, benchmark).params
if not hasattr(benchmark_owner, "setup"):
benchmark_owner.setup = lambda *a: None
if not hasattr(benchmark_owner, "teardown"):
benchmark_owner.teardown = lambda *a: None
for p in params:
runner.timeit(
f"{b}[{p!r}]",
setup="owner.setup()",
globals={
"owner": benchmark_owner,
"params": (p,),
},
teardown="owner.teardown()",
stmt=f"owner.{benchmark}(*params)"
)
if not params:
runner.timeit(
b,
setup="owner.setup()",
globals={
"owner": benchmark_owner,
},
teardown="owner.teardown()",
stmt=f"owner.{benchmark}()"
)
Raw
pyperf_plotting.py
# simple script to plot data from one or multiple pyperf json outputs in various ways for easy comparison
import argparse
import glob
import matplotlib
import matplotlib.pyplot as pyplot
import pyperf as perf
# import scipy.stats as stats
import statistics
import sys
def plot_bench(args, bench, inner_loops):
if args.avg_runs:
runs = bench.get_runs()
x = []
values = []
values_yerr = [[], []]
for run in runs:
for i,v in enumerate(run.values):
while not len(values) > i:
values.append([])
values[i].append(v * inner_loops)
for i,vs in enumerate(values):
x.append(i)
values[i] = statistics.mean(vs)
values_yerr[0].append(values[i] - min(vs))
values_yerr[1].append(max(vs) - values[i])
if args.warmups:
warmups = []
warmups_yerr = [[], []]
for run in runs:
run_values = [value for loops, value in run.warmups]
for i,v in enumerate(run_values):
while not len(warmups) > i:
warmups.append([])
warmups[i].append(v * inner_loops)
for i,vs in enumerate(reversed(warmups[:])):
x.insert(0, 0)
x = [j + 1 for j in x]
warmups[i] = statistics.mean(vs)
warmups_yerr[0].append(warmups[i] - min(vs))
warmups_yerr[1].append(max(vs) - warmups[i])
warmups.reverse()
warmups.append(values[0])
warmups_yerr.reverse()
warmups_yerr[0].append(0)
warmups_yerr[1].append(0)
plt.plot(x[:len(warmups)], warmups, color='red')
mean = statistics.mean(warmups)
plt.plot(x[:len(warmups)], [mean] * len(warmups), color='pink')
plt.plot(x[-len(values):], values, color='blue')
mean = statistics.mean(values)
plt.plot(x[-len(values):], [mean] * len(values))
elif not args.split_runs:
runs = bench.get_runs()
values = []
for run in runs:
run_values = run.values
values.extend(run_values)
plt.plot(values, label='values')
mean = statistics.mean(values)
plt.plot([mean] * len(values), label='mean')
else:
values = []
width = None
for run_index, run in enumerate(bench.get_runs()):
index = 0
x = []
y = []
run_values = run.values
for value in run_values:
x.append(index)
y.append(value)
index += 1
plt.plot(x, y, color='blue')
values.extend(run_values)
width = len(run_values)
if args.warmups:
run_values = [value for loops, value in run.warmups]
index = -len(run.warmups) + 1
x = []
y = []
for value in run_values:
x.append(index)
y.append(value)
index += 1
plt.plot(x, y, color='red')
mean = statistics.mean(values)
plt.plot([mean] * width, label='mean', color='green')
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--split-runs', action='store_true')
parser.add_argument('--avg-runs', action='store_false')
parser.add_argument('--warmups', action='store_true')
parser.add_argument('--sharey', action='store_true')
parser.add_argument('--sharex', action='store_true')
parser.add_argument('--width', default=40, type=int)
parser.add_argument('--height', default=40, type=int)
parser.add_argument('--no-hist', action='store_true')
parser.add_argument('--fontsize', default=12, type=int)
parser.add_argument('--logscale', action='store_true')
parser.add_argument('-f', action='append')
return parser.parse_args()
def display_histogram_scipy(bench, mean, bins):
values = bench.get_values()
values = sorted(values)
# fit = stats.norm.pdf(values, bench.mean(), bench.stdev())
fit = [0] * len(values)
plt.plot(values, fit, '-o')
if __name__ == "__main__":
args = parse_args()
files = [f for argf in args.f for f in glob.glob(argf)]
suites = [perf.BenchmarkSuite.load(f) for f in files]
benchmarks = list(set.intersection(*[set(b.get_name() for b in suite) for suite in suites]))
benchmarks.sort()
subplots = 1 if args.no_hist else 2
matplotlib.rcParams.update({'font.size': args.fontsize})
fig, axs = pyplot.subplots(len(benchmarks) * subplots, len(suites), sharey=('row' if args.sharey else 'none'), sharex=('row' if args.sharex else 'none'), squeeze=False, tight_layout=False, constrained_layout=True, figsize=(args.width,args.height))
for x,suite in enumerate(suites):
for bench in suite:
if bench.get_name() not in benchmarks:
continue
y = benchmarks.index(bench.get_name())
loops = [run.get_loops() for run in bench.get_runs() if run.values]
warmups = [run._warmups for run in bench.get_runs() if run.values]
values = [run.values for run in bench.get_runs() if run.values]
largest_inner_loop_count = max([run.get_inner_loops() for run in bench.get_runs() if run.values])
if len(set(loops)) != 1:
raise Exception(bench)
print(f"{files[x]}.{bench.get_name()}: loops={loops[0]} inner={largest_inner_loop_count} warmups={len(warmups[0]) if warmups[0] else 0} values={len(values[0])}")
plt = axs[y * subplots, x]
plt.set_title(f"{files[x][:-5]} {bench.get_name()}")
if args.logscale:
plt.set_yscale("log")
plot_bench(args, bench, largest_inner_loop_count)
if not args.no_hist:
plt = axs[y * 2 + 1, x]
plt.set_title(f"{files[x][:-5]} {bench.get_name()} Distribution")
display_histogram_scipy(bench, False, 25)
print(files[x], bench.get_name(), bench.mean(), bench.stdev())
pyplot.savefig("analysis.png")
@timfel
Copy link
Author

timfel commented Jul 7, 2022

I tend to plot like so, for example, to plot kiwi results from the vanilla kiwi, hpy in cpython ABI mode, and hpy in universal mode:

python3 gp_pyperf_warmup.py --warmups --sharex --sharey -f cpython-c-api-kiwi.json -f cpython-hpy-cabi-kiwi.json -f cpython-hpy-universal-kiwi.json --width 40 --height 80

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment