Created
May 5, 2024 21:09
-
-
Save mrdrozdov/832cce1dd7d5830e66898486712fdea5 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Sample script to use ThreadPoolExecutor in a way that is easy to interrupt. | |
""" | |
import concurrent.futures | |
import threading | |
from tqdm import tqdm | |
# A flag indicating whether jobs should be cancelled. | |
cancel_jobs = threading.Event() | |
def api_call(item): | |
# If jobs are cancelled, then they'll all return None. | |
if cancel_jobs.is_set(): | |
return None | |
# NOTE: This is just a placeholder function. Replace this with your actual API call. | |
return send_request(item) | |
def make_async_api_calls(inputs): | |
# Function to handle user input. | |
def handle_input(executor): | |
input("Press enter to cancel all jobs...\n") | |
cancel_jobs.set() | |
print("All jobs are being cancelled...") | |
# Use ThreadPoolExecutor to make API calls in parallel. | |
with concurrent.futures.ThreadPoolExecutor() as executor: | |
# Start the thread that will listen for user input. | |
input_thread = threading.Thread(target=handle_input, args=(executor,)) | |
input_thread.start() | |
# Submit all the tasks and store the future objects. | |
futures = [executor.submit(api_call, inp) for inp in inputs] | |
# Keep track of results as they come in. | |
completed = [] | |
for future in tqdm(concurrent.futures.as_completed(futures), total=len(inputs)): | |
result = future.result() | |
if result is not None: | |
completed.append(result) | |
# NOTE: Could throw an exception here, but this way you still get partial results. | |
if cancel_jobs.is_set(): | |
print("Jobs were cancelled.") | |
return completed | |
your_results = make_async_api_calls(your_inputs) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment