-
-
Save raczben/1eb29568953ad5d55b716d119f200f2c to your computer and use it in GitHub Desktop.
Multiprocessing example
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import print_function | |
import multiprocessing | |
import ctypes | |
import numpy as np | |
def shared_array(shape): | |
""" | |
Form a shared memory numpy array. | |
http://stackoverflow.com/questions/5549190/is-shared-readonly-data-copied-to-different-processes-for-python-multiprocessing | |
""" | |
shared_array_base = multiprocessing.Array(ctypes.c_double, shape[0]*shape[1]) | |
shared_array = np.ctypeslib.as_array(shared_array_base.get_obj()) | |
shared_array = shared_array.reshape(*shape) | |
return shared_array | |
# Form a shared array and a lock, to protect access to shared memory. | |
array = shared_array((1000, 1000)) | |
lock = multiprocessing.Lock() | |
def parallel_function(i, def_param=(lock, array)): | |
""" | |
Function that operates on shared memory. | |
""" | |
# Make sure your not modifying data when someone else is. | |
lock.acquire() | |
array[i, :] = i | |
# Always release the lock! | |
lock.release() | |
if __name__ == '__main__': | |
""" | |
The processing pool needs to be instantiated in the main | |
thread of execution. | |
""" | |
pool = multiprocessing.Pool(processes=4) | |
# Call the parallel function with different inputs. | |
args = [(0), | |
(1), | |
(2)] | |
# Use map - blocks until all processes are done. | |
pool.map(parallel_function, args ) | |
print(array) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment