Last active
November 29, 2016 06:23
-
-
Save ixna/46af21da853960d7ebb3d7e25a9ee995 to your computer and use it in GitHub Desktop.
Get rss parallel way
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import time | |
| import requests | |
| from multiprocessing.pool import ThreadPool | |
| from threading import Thread, current_thread | |
| RSS_LIST = [ | |
| 'http://feeds.nytimes.com/nyt/rss/HomePage', | |
| 'http://www.washingtonpost.com/rss/', | |
| 'http://hosted.ap.org/lineups/USHEADS-rss_2.0.xml?SITE=RANDOM&SECTION=HOME', | |
| 'http://www.npr.org/rss/rss.php?id=1001', | |
| 'http://feeds.reuters.com/reuters/topNews'] | |
| def some_process(): | |
| time.sleep(1) | |
| def worker(rss_address): | |
| print("Processing address %s from %s \n" % (rss_address, current_thread().getName())) | |
| response = requests.get(rss_address, timeout=3) | |
| # Some process that takes 1 second | |
| some_process() | |
| return response.content | |
| def run_parallel(): | |
| worker_pool = ThreadPool(5) | |
| result = worker_pool.map(worker, RSS_LIST) | |
| return result | |
| if __name__ == "__main__": | |
| start = time.time() | |
| rss_data = run_parallel() | |
| total_time = time.time() - start | |
| print("Selesai get rss data secara parallel %s" % (total_time,)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment