Skip to content

Instantly share code, notes, and snippets.

@tomMoral
Last active October 5, 2017 07:35
Show Gist options
  • Save tomMoral/cc27a938d669edcf0286c57516942369 to your computer and use it in GitHub Desktop.
Save tomMoral/cc27a938d669edcf0286c57516942369 to your computer and use it in GitHub Desktop.
Deadlocks with concurrent.futuress.ProcessPoolExecutor and unpicklable objects
"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to unpickling error for the result
(at least in versions 3.6 and lower).
"""
from pickle import UnpicklingError
from concurrent.futures import ProcessPoolExecutor
def return_instance(cls):
"""Function that returns a instance of cls"""
return cls()
def raise_error(Err):
"""Function that raises an Exception in process"""
raise Err()
class ObjectWithUnpickleError():
"""Triggers a RuntimeError when sending job to the workers"""
def __reduce__(self):
return raise_error, (UnpicklingError, )
if __name__ == "__main__":
with ProcessPoolExecutor() as e:
f = e.submit(return_instance, ObjectWithUnpickleError)
f.result()
"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
from concurrent.futures import ProcessPoolExecutor
class ObjectWithPickleError():
"""Triggers a RuntimeError when sending job to the workers"""
def __reduce__(self):
raise RuntimeError()
if __name__ == "__main__":
e = ProcessPoolExecutor()
f = e.submit(id, ObjectWithPickleError())
f.result() # Deadlock on get
"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
from concurrent.futures import ProcessPoolExecutor
class ObjectWithPickleError():
"""Triggers a RuntimeError when sending job to the workers"""
def __reduce__(self):
raise RuntimeError()
if __name__ == "__main__":
with ProcessPoolExecutor() as e:
f = e.submit(id, ObjectWithPickleError())
# Deadlock on shutdown
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment