Created
July 20, 2012 22:21
-
-
Save ibrahima/3153647 to your computer and use it in GitHub Desktop.
A silly class that lets you use Grequests in a somewhat similar manner as the ruby gem Typhoeus
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import grequests | |
| from collections import deque | |
| class RequestQueue(object): | |
| """ | |
| This is a lame imitation of a Typhoeus Hydra using GRequests. | |
| The main thing this allows is building up a queue of requests and then | |
| executing them, and potentially adding requests to the queue in a callback | |
| so that you can build requests that depend on other requests more naturally. | |
| This shouldn't be as fast as Typhoeus in theory, but I think the interface | |
| is slightly more convenient and it should be fast enough until I feel like | |
| building a proper Typhoeus replacement using PyCurl. | |
| """ | |
| def __init__(self): | |
| self.queue = deque() | |
| self.requests = deque() | |
| self.max_concurrent = 10 | |
| self.running = False | |
| def add(self, request): | |
| if not self.running or len(self.requests) < self.max_concurrent: | |
| self.requests.append(request) | |
| else: | |
| self.queue.append(request) | |
| def run(self): | |
| """ | |
| Runs grequests.map repeatedly until the queue is depleted. | |
| """ | |
| self.running = True | |
| result = [] | |
| while self.requests: | |
| result.extend(grequests.map(self.requests)) | |
| self.requests = deque() # Clear the request queue | |
| while self.queue and len(self.requests) < self.max_concurrent: | |
| self.requests.append(self.queue.popleft()) | |
| self.running = False | |
| return result | |
| def _pop(self): | |
| """ | |
| TODO: Attempt to use grequests.imap to see if that also works and how | |
| it affects performance. | |
| """ | |
| while self.requests: | |
| yield self.requests.popleft() | |
| if self.queue: | |
| self.requests.append(self.queue.popleft()) | |
| def build_callback(self, fxn): | |
| """ | |
| This forms a closure over the queue for use in a Requests callback | |
| """ | |
| # TODO could do the queue management here somewhat like Typhoeus does | |
| def single_arg_callback(arg): | |
| fxn(arg, self) | |
| return single_arg_callback | |
| import grequests | |
| def testQueue(queue, initial=10): | |
| from random import randint | |
| initials = [randint(0,10) for x in xrange(initial)] | |
| def build_funky_callback(count): | |
| def lolback(response, q): | |
| if count > 0: | |
| q.add(grequests.get("http://httpbin.org/get", hooks={"response":build_funky_callback(count-1)}) ) | |
| else: | |
| q.add(grequests.get("http://httpbin.org/get")) | |
| return queue.build_callback(lolback) | |
| for x in initials: | |
| req = grequests.get("http://httpbin.org/get", hooks={"response":build_funky_callback(x)}) | |
| queue.add(req) | |
| ret = queue.run() | |
| return ret | |
| rq=RequestQueue() | |
| res=testQueue(rq, 100) | |
| print len(res) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment