Last active
September 4, 2019 21:01
-
-
Save christopherlovell/6e38495141f47ef77633da6af8737b66 to your computer and use it in GitHub Desktop.
Example Schwimmbad usage
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
A tutorial introduction to Schwimmbad | |
https://github.com/adrn/schwimmbad | |
To install the requirements: | |
> cat requirements.txt | xargs pip install | |
""" | |
import numpy as np | |
import time | |
data = np.arange(1e6) | |
def do_the_processing(x): | |
# Do something with each value in the data set! | |
# For example, here we just square each value | |
return x**4 | |
print("using a loop...\n------------------") | |
start = time.time() | |
values = [] | |
for x in data: | |
values.append(do_the_processing(x)) | |
end = time.time() | |
print(values[:10]) | |
print("Time elapsed: %.4f\n"%(end - start)) | |
print("using map...\n------------------") | |
start = time.time() | |
values = list(map(do_the_processing, data)) | |
end = time.time() | |
print(values[:10]) | |
print("Time elapsed: %.4f\n"%(end - start)) | |
print("Using schwimmbad to do this (in serial)\n------------------") | |
from schwimmbad import SerialPool | |
start = time.time() | |
pool = SerialPool() | |
values = list(pool.map(do_the_processing, data)) | |
end = time.time() | |
print(values[:10]) | |
print("Time elapsed: %.4f\n"%(end - start)) | |
pool.close() | |
print("Using schwimmbad with multiprocessing (2 threads)\n------------------") | |
from schwimmbad import MultiPool | |
N = 2 | |
start = time.time() | |
pool = MultiPool() | |
values = list(pool.map(do_the_processing, data)) | |
# with MultiPool(processes=N) as pool: | |
# """ | |
# The context manager (`with`) ensures the file gets closed once we're finished | |
# """ | |
# values = list(pool.map(do_the_processing, data)) | |
end = time.time() | |
print(values[:10]) | |
print("Time elapsed: %.4f\n"%(end - start)) | |
pool.close() | |
print("Using schwimmbad with multiprocessing in a context manager (8 threads)\n------------------") | |
from schwimmbad import MultiPool | |
N = 8 | |
start = time.time() | |
with MultiPool(processes=N) as pool: | |
""" | |
The context manager (`with`) ensures the file gets closed once we're finished | |
""" | |
values = list(pool.map(do_the_processing, data)) | |
end = time.time() | |
print(values[:10]) | |
print("Time elapsed: %.4f\n"%(end - start)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Run with: | |
> mpiexec -n 2 python mpi-demo.py | |
""" | |
import sys | |
import time | |
import numpy as np | |
from schwimmbad import MPIPool | |
def do_the_processing(x): | |
# Do something with each value in the data set! | |
# For example, here we just square each value | |
return x**2 | |
start = time.time() | |
pool = MPIPool() | |
if not pool.is_master(): | |
pool.wait() | |
sys.exit(0) | |
data = np.arange(10000) | |
values = list(pool.map(do_the_processing,data)) | |
end = time.time() | |
print("Time elapsed: %.4f\n"%(end - start)) | |
print(values[:10]) | |
pool.close() | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
six | |
numpy | |
ipython | |
mpi4py | |
schwimmbad |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment