Created
July 18, 2019 01:28
-
-
Save dylan-slack/b8c95b38d519b63cbba186d889b09330 to your computer and use it in GitHub Desktop.
A simple method to manage experiments
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """ | |
| A simple experiment manager. You can add experiments as functions with the @register decorator. Then, call the experiments you want to run | |
| using the -r flag followed by the function names of the experiments or run all the experiments with the -a flag. | |
| Experiment runs are stored in an .experiments/ folder created on the first time you run an experiment. Each run creates a folder inside | |
| .experiments/ with the name of the function and the datetime it was run. Every experiment should accept a logging_path argument. | |
| This is the path to the folder created for the specific experiment run. You can use this to store experiment specific files. | |
| """ | |
| import argparse | |
| import datetime | |
| import os | |
| experiments = {} | |
| parser = argparse.ArgumentParser(description='Run experiments.') | |
| experiment_folder = './experiments' | |
| STRIP = -7 | |
| def register(experiment): | |
| experiments[experiment.__name__] = experiment | |
| ## | |
| # Begin Experiments: | |
| # Each experiment function call should accept a logging path argument. | |
| # This is the path of the directory created for each new run of this experiment | |
| # and can be used to store files specific to the experiment. | |
| ### | |
| @register | |
| def test_exp_1(logging_path): | |
| print ("Experiment 1 here") | |
| @register | |
| def test_exp_2(logging_path): | |
| print ("Experiment 2 here") | |
| ### | |
| def setup(exp): | |
| time_of_creation = str(datetime.datetime.now()).replace(" ", ".")[:STRIP] | |
| exp_path = os.path.join(experiment_folder,f"{exp}_{time_of_creation}") | |
| if not os.path.exists(exp_path): | |
| os.makedirs(exp_path) | |
| return exp_path | |
| def main(): | |
| parser.add_argument('-s', action='store_true', default=False, | |
| help='show the current experiments avaliable to run') | |
| parser.add_argument('-r', metavar='--run', nargs='+', | |
| help='run these experiments') | |
| parser.add_argument('-a', action='store_true', default=False, | |
| help='run all the experiments avaliable') | |
| args = vars(parser.parse_args()) | |
| if args['s'] or not args['r']: | |
| print ("Avaliable Experiments:") | |
| print ("----------------------") | |
| for key in experiments: | |
| print (key) | |
| print ("----------------------") | |
| print ("Run experiments with -r flag.") | |
| exit() | |
| if args['a']: | |
| for exp in experiments: | |
| print (f"Beginning experiment {exp}") | |
| print ("-----------------------------") | |
| logging_path = setup(exp) | |
| experiments[exp](logging_path) | |
| print (f"Saving in {logging_path}") | |
| print ("-----------------------------") | |
| exit() | |
| for exp in args['r']: | |
| if exp in experiments: | |
| print (f"Beginning experiment {exp}") | |
| print ("-----------------------------") | |
| logging_path = setup(exp) | |
| experiments[exp](logging_path) | |
| print (f"Saving in {logging_path}") | |
| print ("-----------------------------") | |
| else: | |
| print(f"Experiment {exp} not registered -- skipping!.") | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment