Created
May 31, 2024 07:19
-
-
Save Voyz/1b6857f6d7b9ada024e3d2529bba8f4c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import datetime | |
| from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor | |
| from ratelimiter import ThrottleBarrier, CrossProcessesThrottle | |
| def log(*args, **kwargs): | |
| print(datetime.datetime.now().strftime('[%H:%M:%S]'), *args, **kwargs) | |
| def task(i, j, throttle_barrier: ThrottleBarrier): | |
| throttle_barrier.wait() # This will block until there is a free slot to make a request | |
| log(f'request: {i:2d}, {j:2d} (process, thread)') | |
| # make the request here... | |
| def worker(i, throttle_barrier: ThrottleBarrier): | |
| # example process worker, starting a bunch of threads | |
| with ThreadPoolExecutor(max_workers=5) as executor: | |
| for j in range(5): | |
| executor.submit(task, i, j, throttle_barrier) | |
| if __name__ == '__main__': | |
| cross_process_throttle = CrossProcessesThrottle(max_requests=3, per_seconds=10) | |
| throttle_barrier = cross_process_throttle.get_barrier() | |
| log('start') | |
| futures = [] | |
| with ProcessPoolExecutor(max_workers=10) as executor: | |
| # schedule 9 jobs, which should exceed our limit of 3 requests per 10 seconds | |
| for i in range(3): | |
| futures.append(executor.submit(worker, i, throttle_barrier)) | |
| while len(futures): | |
| cross_process_throttle.cycle() # calling this method carries out the rate limit calculation | |
| for future in futures: | |
| if future.done(): | |
| futures.remove(future) | |
| log('finish') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment