Last active
July 30, 2021 17:12
-
-
Save nfaggian/9755516 to your computer and use it in GitHub Desktop.
Multiprocessing example
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import print_function | |
import multiprocessing | |
import ctypes | |
import numpy as np | |
def shared_array(shape): | |
""" | |
Form a shared memory numpy array. | |
http://stackoverflow.com/questions/5549190/is-shared-readonly-data-copied-to-different-processes-for-python-multiprocessing | |
""" | |
shared_array_base = multiprocessing.Array(ctypes.c_double, shape[0]*shape[1]) | |
shared_array = np.ctypeslib.as_array(shared_array_base.get_obj()) | |
shared_array = shared_array.reshape(*shape) | |
return shared_array | |
# Form a shared array and a lock, to protect access to shared memory. | |
array = shared_array((1000, 1000)) | |
lock = multiprocessing.Lock() | |
def parallel_function(i, def_param=(lock, array)): | |
""" | |
Function that operates on shared memory. | |
""" | |
# Make sure your not modifying data when someone else is. | |
lock.acquire() | |
array[i, :] = i | |
# Always release the lock! | |
lock.release() | |
if __name__ == '__main__': | |
""" | |
The processing pool needs to be instantiated in the main | |
thread of execution. | |
""" | |
pool = multiprocessing.Pool(processes=4) | |
# Call the parallel function with different inputs. | |
args = [(0), | |
(1), | |
(2)] | |
# Use map - blocks until all processes are done. | |
pool.map(parallel_function, args ) | |
print(array) |
I suggest to make it more general:
def shared_array(dtype, shape):
"""
Form a shared memory numpy array.
https://stackoverflow.com/q/5549190/2506522
"""
size = sys.getsizeof(dtype())
for dim in shape:
size *= dim
shared_array_base = multiprocessing.Array(ctypes.c_byte, size)
shared_array = np.ndarray(shape, dtype=dtype, buffer=shared_array_base.get_obj())
shared_array = shared_array.reshape(*shape)
return shared_array
shared_array_base = multiprocessing.Array(ctypes.c_byte, size)
Shouldn't the first parameter be dtype here?
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This is exactly what I need. Thank you so much.