Last active
May 16, 2023 02:50
-
-
Save c-bata/c257feffa35ad5a35f5df2ca100e73e9 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse | |
import os | |
import subprocess | |
def run(args: argparse.Namespace) -> None: | |
kurobako_cmd = os.path.join(args.path_to_kurobako, "kurobako") | |
subprocess.run(f"{kurobako_cmd} --version", shell=True) | |
if not (os.path.exists(args.data_dir) and os.path.isdir(args.data_dir)): | |
raise ValueError(f"Data directory {args.data_dir} cannot be found.") | |
os.makedirs(args.out_dir, exist_ok=True) | |
study_json_fn = os.path.join(args.out_dir, "studies.json") | |
solvers_filename = os.path.join(args.out_dir, "solvers.json") | |
problems_filename = os.path.join(args.out_dir, "problems.json") | |
# Ensure all files are empty. | |
for filename in [study_json_fn, solvers_filename, problems_filename]: | |
with open(filename, "w"): | |
pass | |
# Create HPO bench problem. | |
datasets = [ | |
"fcnet_tabular_benchmarks/fcnet_naval_propulsion_data.hdf5", | |
"fcnet_tabular_benchmarks/fcnet_parkinsons_telemonitoring_data.hdf5", | |
"fcnet_tabular_benchmarks/fcnet_protein_structure_data.hdf5", | |
"fcnet_tabular_benchmarks/fcnet_slice_localization_data.hdf5", | |
] | |
for dataset in datasets: | |
dataset = os.path.join(args.data_dir, dataset) | |
cmd = ( | |
f'{kurobako_cmd} problem hpobench "{dataset}" | tee -a {problems_filename}' | |
) | |
subprocess.run(cmd, shell=True) | |
# Create Optuna solvers | |
for name, sampler, sampler_kwargs in [ | |
("random", "RandomSampler", r"{}"), | |
("vanilla-cmaes", "CmaEsSampler", r"{}"), | |
("cmaes-with-margin", "CmaEsSampler", r"{\"with_margin\":true}"), | |
]: | |
cmd = ( | |
f"{kurobako_cmd} solver --name {name} optuna --loglevel debug " | |
f"--sampler {sampler} --sampler-kwargs {sampler_kwargs} " | |
"--pruner NopPruner --pruner-kwargs {} " | |
f"| tee -a {solvers_filename}" | |
) | |
subprocess.run(cmd, shell=True) | |
# Create study. | |
cmd = ( | |
f"{kurobako_cmd} studies --budget {args.budget} " | |
f"--solvers $(cat {solvers_filename}) --problems $(cat {problems_filename}) " | |
f"--repeats {args.n_runs} --seed {args.seed} --concurrency {args.n_concurrency} " | |
f"> {study_json_fn}" | |
) | |
subprocess.run(cmd, shell=True, check=True) | |
result_filename = os.path.join(args.out_dir, "results.json") | |
cmd = ( | |
f"cat {study_json_fn} | {kurobako_cmd} run --parallelism {args.n_jobs} " | |
f"> {result_filename}" | |
) | |
subprocess.run(cmd, shell=True) | |
report_filename = os.path.join(args.out_dir, "report.md") | |
cmd = f"cat {result_filename} | {kurobako_cmd} report > {report_filename}" | |
subprocess.run(cmd, shell=True) | |
cmd = ( | |
f"cat {result_filename} | docker run -v $(pwd)/{args.out_dir}/images:/images/ " | |
f"--rm -i sile/kurobako plot curve --errorbar --xmin 10" | |
) | |
subprocess.run(cmd, shell=True) | |
if __name__ == "__main__": | |
parser = argparse.ArgumentParser() | |
parser.add_argument("--path-to-kurobako", type=str, default="") | |
parser.add_argument("--budget", type=int, default=2000) | |
parser.add_argument("--n-runs", type=int, default=10) | |
parser.add_argument("--n-jobs", type=int, default=10) | |
parser.add_argument("--n-concurrency", type=int, default=1) | |
parser.add_argument("--seed", type=int, default=0) | |
parser.add_argument("--data-dir", type=str, default="tmp/data") | |
parser.add_argument("--out-dir", type=str, default="tmp/benchmark_report") | |
args = parser.parse_args() | |
run(args) |
Author
c-bata
commented
May 16, 2023
•
Problem Name | Before (master) | After (PR) |
---|---|---|
Slice | ||
Protein | ||
Parkinson | ||
Naval |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment