Created
September 5, 2022 04:55
-
-
Save RodolfoFerro/08dd90ddb502a76c3016e79ecd1fa782 to your computer and use it in GitHub Desktop.
Gists for blog post about running notebooks in parallel. Ploomber, 2022.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This requires: | |
# pip install papermill | |
import papermill as pm | |
from glob import glob | |
for nb in glob('*.ipynb'): | |
pm.execute_notebook( | |
input_path=nb, | |
output_path=nb, | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This requires: | |
# pip install papermill ploomber-engine | |
import papermill as pm | |
from glob import glob | |
for nb in glob('*.ipynb'): | |
pm.execute_notebook( | |
input_path=nb, | |
output_path=nb, | |
engine_name='embedded', | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This requires: | |
# pip install ploomber | |
from ploomber import DAG | |
from ploomber.products import File | |
from ploomber.tasks import NotebookRunner | |
from ploomber.executors import Parallel | |
from pathlib import Path | |
from glob import iglob | |
dag = DAG(executor=Parallel()) | |
for path in iglob('*.ipynb'): | |
NotebookRunner(Path(path), File(path), dag=dag, papermill_params=dict(engine_name='embedded')) | |
if __name__ == '__main__': | |
dag.build(force=True) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This requires: | |
# pip install ploomber | |
from ploomber import DAG | |
from ploomber.products import File | |
from ploomber.tasks import NotebookRunner | |
from ploomber.executors import Parallel | |
from pathlib import Path | |
from glob import iglob | |
dag = DAG() | |
for path in iglob('*.ipynb'): | |
NotebookRunner(Path(path), File(path), dag=dag, papermill_params=dict(engine_name='embedded')) | |
if __name__ == '__main__': | |
dag.build(force=True) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment