Skip to content

Instantly share code, notes, and snippets.

@scubamut

scubamut/ingest.py

Created September 7, 2024 18:05
Show Gist options
  • Save scubamut/9c2698d02f6f4168ae3c26797754d25c to your computer and use it in GitHub Desktop.
Save scubamut/9c2698d02f6f4168ae3c26797754d25c to your computer and use it in GitHub Desktop.
Download ZIP
Raw
ingest.py
# to ingest the etfs
import zipline
from zipline.data.bundles.core import register, ingest
import yfinance as yf
import pandas as pd
from datetime import datetime
import pytz
# FOR ALL ASSETS
# Change the path to where you have your data
try:
import google.colab
# for COLAB
data_home = '/content/data/daily/US_stock_data/'
except:
data_home = '/home/scubamut1/MEGAsync/10_DATA/daily/US_stock_data/'
assets = list(set(['SPY', 'MDY', 'EEM', 'IEV', 'EPP',\
'ILF', 'XOP', 'EDV', 'ZIV']))
# start = '1990-1-1'
# end = str(datetime.today().date())
start = datetime(2010, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2019, 6, 10, 0, 0, 0, 0, pytz.utc)
df = yf.download(assets, start, end)
# change to lowercase
a = df.columns.get_level_values(0).str.lower()
b = df.columns.get_level_values(1)
df.columns = pd.MultiIndex.from_arrays([a,b])
for asset in assets:
d = df.stack(0,1)[asset].unstack().ffill(limit=None).dropna()
# for funds, create volume=100k to use zipline
if d.volume[0] == 0:
d.volume = 100000
print(asset, d.index[0].date(), len(d))
inception = [(asset, d.index[0].date(), len(d))]
d.to_csv(data_home + asset + '.csv')
# sorted ascending
# sorted(inception, key=lambda x: x[1])
# FOR ETFs, STOCKS, FUNDS
register('US_stock_bundle',assets)
!zipline ingest -b US_stock_bundle
Raw
quickest.py
from google.colab import drive
drive.mount('/content/drive')
###################################################################################################
!pip install /content/drive/MyDrive/WheelFiles/TA_Lib-0.4.26-cp310-cp310-linux_x86_64.whl
!cp /content/drive/MyDrive/WheelFiles/libta_lib.so.0 /usr/lib
!pip install /content/drive/MyDrive/WheelFiles/zipline_reloaded-0.0.0-cp310-cp310-linux_x86_64.whl
####################################################################################################
!mkdir /root/.zipline
####################################################################################################
%%writefile /root/.zipline/extension.py
# US_stock_bundle
###############
from zipline.data.bundles import register, US_stock_bundle
register('US_stock_bundle', US_stock_bundle.US_stock_data, calendar_name='NYSE')
#####################################################################################################
# CSV DATA IS STORED HERE
!cd /content
!mkdir data/
!mkdir data/daily/
!mkdir data/daily/US_stock_data
!cp "/content/drive/MyDrive/00 HOT!/00 IMPORTANT!/US_stock_bundle.py" /usr/local/lib/python3.10/dist-packages/zipline/data/bundles
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment