Skip to content

Instantly share code, notes, and snippets.

@sanelson
Created May 1, 2016 16:58
Show Gist options
  • Save sanelson/5fd0c7d6f4eaa7f414ac48d16bae3378 to your computer and use it in GitHub Desktop.
Save sanelson/5fd0c7d6f4eaa7f414ac48d16bae3378 to your computer and use it in GitHub Desktop.
Python Multiprocessing with simple progress tracking via queues in main process
from progressbar import ProgressBar, SimpleProgress
from multiprocessing import Process, Queue
from time import sleep
import progressbar
class Worker():
def __init__(self, name, result_queue, process):
self.name = name
self.result_queue = result_queue
self.process = process
def worker_process(result_queue, min_count, max_count):
for count in range(min_count, max_count+1):
sleep(.1)
result_queue.put(count)
return
if __name__ == '__main__':
processes = 4
count = 100
tally = 0
# Set up progress bar
progress = progressbar.ProgressBar(min_val=1, max_val=count)
# Set up workers array
workers = []
for process in range(1,processes+1):
# Set up results queue
result_queue = Queue()
# Start up worker
worker = Process(target=worker_process, args=(result_queue, 1, count//processes))
worker.daemon = True
worker.start()
# Create worker tracking object
workers.append(Worker("worker%d" % process, result_queue, worker))
while tally < count:
tally = 0
for worker in workers:
tally += worker.result_queue.get()
progress.update(tally)
progress.finish()
for worker in workers:
worker.process.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment