Last active
July 7, 2020 21:49
-
-
Save JesseAldridge/070839b3b06ab4674e1d4b4b56628824 to your computer and use it in GitHub Desktop.
download_symbols_polygon_loop.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import json, os, csv, time | |
| from datetime import datetime | |
| import requests | |
| import _0_load_price_by_date | |
| data_dir = os.path.expanduser('~/stock_data/polygon_prices') | |
| if not os.path.exists(data_dir): | |
| os.makedirs(data_dir) | |
| with open(os.path.expanduser('~/alpaca_config.json')) as f: | |
| text = f.read() | |
| api_key = json.loads(text)['live']['api_key'] | |
| while True: | |
| for page in range(1, 10 ** 5): | |
| print('page:', page) | |
| for attempt in range(5): | |
| try: | |
| tickers_resp = requests.get( | |
| 'https://api.polygon.io/v2/reference/tickers?apiKey={}&page={}&market={}'.format( | |
| api_key, page, 'stocks' | |
| ), timeout=5, | |
| ) | |
| except( | |
| requests.exceptions.ConnectionError, | |
| requests.exceptions.ConnectTimeout, | |
| requests.exceptions.ReadTimeout | |
| ): | |
| print('timeout loading tickers') | |
| continue | |
| break | |
| else: | |
| print('too many timeouts, giving up') | |
| continue | |
| # keys: ['page', 'perPage', 'count', 'status', 'tickers'] | |
| try: | |
| ticker_count = tickers_resp.json()['count'] | |
| except json.decoder.JSONDecodeError: | |
| print('json error:', tickers_resp.content) | |
| continue | |
| per_page = tickers_resp.json()['perPage'] | |
| tickers = tickers_resp.json()['tickers'] | |
| if not tickers: | |
| break | |
| for i_ticker, ticker_dict in enumerate(tickers): | |
| ticker = ticker_dict.get('ticker') | |
| print('{}/{}: {}'.format((page - 1) * per_page + i_ticker, ticker_count, ticker)) | |
| _0_load_price_by_date.load_price_by_date(ticker, use_filesystem_cache=False) | |
| print('completed at:', datetime.utcnow()) | |
| time.sleep(60 * 60 * 24) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment