Last active
February 11, 2024 21:25
-
-
Save thistleknot/cce164e27b5e9da5e19f39880f02c6c5 to your computer and use it in GitHub Desktop.
how to pull yahoo finance data
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def get_v1_url(symbol, period_type, crumb): | |
headers = { | |
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', | |
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', | |
'Accept-Language': 'en-US,en;q=0.5', | |
} | |
period1 = 493590046 | |
period2 = 1913180947 | |
session = requests.Session() | |
session.get('https://finance.yahoo.com', headers=headers) | |
# Use the existing crumb if provided, otherwise fetch a new one | |
if not crumb: | |
crumb_response = session.get('https://query2.finance.yahoo.com/v1/test/getcrumb', headers=headers) | |
if crumb_response.status_code == 200 and crumb_response.text: | |
crumb = crumb_response.text | |
else: | |
print("Failed to retrieve crumb") | |
base_url = 'https://query1.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/' | |
if(period_type=='Annual'): | |
annual_types = ",".join(["annual" + option for option in optionSet]) | |
url = f"{base_url}{symbol}?symbol={symbol}&type={annual_types}&period1={period1}&period2={period2}&crumb={crumb}" | |
else: | |
quarterly_types = ",".join(["quarterly" + option for option in optionSet]) | |
url = f"{base_url}{symbol}?symbol={symbol}&type={quarterly_types}&period1={period1}&period2={period2}&crumb={crumb}" | |
#print("Annual URL:", annual_url) | |
#print("Quarterly URL:", quarterly_url) | |
return(url) | |
headers = { | |
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', | |
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', | |
'Accept-Language': 'en-US,en;q=0.5', | |
} | |
session = requests.Session() | |
session.get('https://finance.yahoo.com', headers=headers) | |
crumb = False | |
# Use the existing crumb if provided, otherwise fetch a new one | |
if not crumb: | |
crumb_response = session.get('https://query2.finance.yahoo.com/v1/test/getcrumb', headers=headers) | |
if crumb_response.status_code == 200 and crumb_response.text: | |
crumb = crumb_response.text | |
symbol = 'AAPL' | |
#fundamentals | |
v7_url = f"""https://query1.finance.yahoo.com/v7/finance/quote?lang=en-US®ion=US&corsDomain=finance.yahoo.com&symbols={symbol}&crumb={crumb}""" | |
v1_annual_url = get_v1_url('AAPL','Annual',crumb) | |
v1_quarterly_url = get_v1_url('AAPL','Quarterly',crumb) | |
#print(v1_annual_url) | |
v7_response = session.get(v7_url, headers=headers) | |
v1_a_response = session.get(v1_annual_url, headers=headers) | |
v1_q_response = session.get(v1_quarterly_url, headers=headers) | |
v7_data = {} | |
if v7_response.status_code == 200: | |
v7_data = v7_response.json() | |
v7_payload = v7_data['quoteResponse']['result'][0].keys() | |
print('v7_payload') | |
print(v7_payload) | |
v1_q_data = {} | |
if v1_q_response.status_code == 200: | |
v1_q_data = v1_q_response.json() | |
v1_a_data = {} | |
if v1_a_response.status_code == 200: | |
v1_a_data = v1_a_response.json() | |
#financial statements | |
# | |
v1_q_payload = v1_q_data['timeseries']['result'] | |
v1_a_payload = v1_a_data['timeseries']['result'] | |
v1_q_payload = [p['meta']['type'] for p in v1_q_payload] | |
v1_a_payload = [p['meta']['type'] for p in v1_a_payload] | |
print('v1_q_payload') | |
print(v1_q_payload) | |
print('v1_a_payload') | |
print(v1_a_payload) | |
""" | |
[{'meta': {'symbol': ['AAPL'], | |
'annualNetIncomeContinuousOperations': [{'dataId': 20094, | |
'asOfDate': '2020-09-30', | |
'periodType': '12M', | |
'currencyCode': 'USD', | |
'reportedValue': {'raw': 57411000000.0, 'fmt': '57.41B'}}, | |
... | |
{'meta': {'symbol': ['AAPL'], 'type': ['annualTaxEffectOfUnusualItems']}, | |
'timestamp': [1601424000, 1632960000, 1664496000, 1696032000], | |
'annualTaxEffectOfUnusualItems': [{'dataId': 20419, | |
'asOfDate': '2020-09-30', | |
'periodType': '12M', | |
'currencyCode': 'USD', | |
'reportedValue': {'raw': 0.0, 'fmt': '0.00'}}, | |
""" | |
# Save the results to JSON files | |
with open('v7_data.json', 'w') as f_data: | |
json.dump(v7_data, f_data, ensure_ascii=False, indent=4) | |
# Save the results to JSON files | |
with open('v1_a_data.json', 'w') as f_data: | |
json.dump(v1_a_data, f_data, ensure_ascii=False, indent=4) | |
with open('v1_q_data.json', 'w') as f_data: | |
json.dump(v1_q_data, f_data, ensure_ascii=False, indent=4) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment