Skip to content

Instantly share code, notes, and snippets.

@nistrup
Last active August 20, 2024 09:16
Show Gist options
  • Save nistrup/1e724d6e450fd1da09a0782e6bfcd41a to your computer and use it in GitHub Desktop.
Save nistrup/1e724d6e450fd1da09a0782e6bfcd41a to your computer and use it in GitHub Desktop.
# IMPORTS
import pandas as pd
import math
import os.path
import time
from bitmex import bitmex
from binance.client import Client
from datetime import timedelta, datetime
from dateutil import parser
from tqdm import tqdm_notebook #(Optional, used for progress-bars)
### API
bitmex_api_key = '[REDACTED]' #Enter your own API-key here
bitmex_api_secret = '[REDACTED]' #Enter your own API-secret here
binance_api_key = '[REDACTED]' #Enter your own API-key here
binance_api_secret = '[REDACTED]' #Enter your own API-secret here
### CONSTANTS
binsizes = {"1m": 1, "5m": 5, "1h": 60, "1d": 1440}
batch_size = 750
bitmex_client = bitmex(test=False, api_key=bitmex_api_key, api_secret=bitmex_api_secret)
binance_client = Client(api_key=binance_api_key, api_secret=binance_api_secret)
### FUNCTIONS
def minutes_of_new_data(symbol, kline_size, data, source):
if len(data) > 0: old = parser.parse(data["timestamp"].iloc[-1])
elif source == "binance": old = datetime.strptime('1 Jan 2017', '%d %b %Y')
elif source == "bitmex": old = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=1, reverse=False).result()[0][0]['timestamp']
if source == "binance": new = pd.to_datetime(binance_client.get_klines(symbol=symbol, interval=kline_size)[-1][0], unit='ms')
if source == "bitmex": new = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=1, reverse=True).result()[0][0]['timestamp']
return old, new
def get_all_binance(symbol, kline_size, save = False):
filename = '%s-%s-data.csv' % (symbol, kline_size)
if os.path.isfile(filename): data_df = pd.read_csv(filename)
else: data_df = pd.DataFrame()
oldest_point, newest_point = minutes_of_new_data(symbol, kline_size, data_df, source = "binance")
delta_min = (newest_point - oldest_point).total_seconds()/60
available_data = math.ceil(delta_min/binsizes[kline_size])
if oldest_point == datetime.strptime('1 Jan 2017', '%d %b %Y'): print('Downloading all available %s data for %s. Be patient..!' % (kline_size, symbol))
else: print('Downloading %d minutes of new data available for %s, i.e. %d instances of %s data.' % (delta_min, symbol, available_data, kline_size))
klines = binance_client.get_historical_klines(symbol, kline_size, oldest_point.strftime("%d %b %Y %H:%M:%S"), newest_point.strftime("%d %b %Y %H:%M:%S"))
data = pd.DataFrame(klines, columns = ['timestamp', 'open', 'high', 'low', 'close', 'volume', 'close_time', 'quote_av', 'trades', 'tb_base_av', 'tb_quote_av', 'ignore' ])
data['timestamp'] = pd.to_datetime(data['timestamp'], unit='ms')
if len(data_df) > 0:
temp_df = pd.DataFrame(data)
data_df = data_df.append(temp_df)
else: data_df = data
data_df.set_index('timestamp', inplace=True)
if save: data_df.to_csv(filename)
print('All caught up..!')
return data_df
def get_all_bitmex(symbol, kline_size, save = False):
filename = '%s-%s-data.csv' % (symbol, kline_size)
if os.path.isfile(filename): data_df = pd.read_csv(filename)
else: data_df = pd.DataFrame()
oldest_point, newest_point = minutes_of_new_data(symbol, kline_size, data_df, source = "bitmex")
delta_min = (newest_point - oldest_point).total_seconds()/60
available_data = math.ceil(delta_min/binsizes[kline_size])
rounds = math.ceil(available_data / batch_size)
if rounds > 0:
print('Downloading %d minutes of new data available for %s, i.e. %d instances of %s data in %d rounds.' % (delta_min, symbol, available_data, kline_size, rounds))
for round_num in tqdm_notebook(range(rounds)):
time.sleep(1)
new_time = (oldest_point + timedelta(minutes = round_num * batch_size * binsizes[kline_size]))
data = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=batch_size, startTime = new_time).result()[0]
temp_df = pd.DataFrame(data)
data_df = data_df.append(temp_df)
data_df.set_index('timestamp', inplace=True)
if save and rounds > 0: data_df.to_csv(filename)
print('All caught up..!')
return data_df
@TurtleJelly
Copy link

thank you my friend. This is very useful, avoid to read tons of strange API docs from BITMEX.

@matiasmansilla
Copy link

matiasmansilla commented Jan 29, 2020

hello bro: i get the error:

ModuleNotFoundError: No module named 'bitmex'

@Arpiaz-FTX
Copy link

Arpiaz-FTX commented Mar 24, 2020

hello bro: i get the error:

ModuleNotFoundError: No module named 'bitmex'

You need to install the Bitmex module first, which you can do by using pip install bitmex in the terminal

@Arpiaz-FTX
Copy link

Thanks a lot for this! Could this be expanded to the FTX exchange as well?

@farbodsaeedian
Copy link

hi, thanx for your great effort
i have these error

C:\Users\Rayan tech\Desktop\new bot\bitmex - binance data\bitmex_and_binance.py:65: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use tqdm.notebook.tqdm instead of tqdm.tqdm_notebook
for round_num in tqdm_notebook(range(rounds)):
Traceback (most recent call last):
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 96, in status_printer
pbar = IProgress(min=0, max=total)
NameError: name 'IProgress' is not defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "c:\Users\Rayan tech\Desktop\new bot\bitmex - binance data\bitmex_and_binance.py", line 76, in
data = get_all_bitmex("XBTUSD", "5m", save = True)
File "c:\Users\Rayan tech\Desktop\new bot\bitmex - binance data\bitmex_and_binance.py", line 65, in get_all_bitmex
for round_num in tqdm_notebook(range(rounds)):
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm_init_.py", line 28, in tqdm_notebook
return _tqdm_notebook(*args, **kwargs)
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 208, in init
self.container = self.status_printer(
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 103, in status_printer
raise ImportError(
ImportError: FloatProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
Exception ignored in: <function tqdm.del at 0x0B065970>
Traceback (most recent call last):
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\std.py", line 1077, in del
self.close()
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 240, in close
self.sp(bar_style='danger')
AttributeError: 'tqdm_notebook' object has no attribute 'sp'

@shubhang12
Copy link

Hi nistrup, this is an amazing code. It has saved me so much time. If you find the time, please add a function for binance futures data too. It would be a life saver

@DIBMS
Copy link

DIBMS commented Jan 20, 2021

trying to run thru visual code, adde API keys, dind save anything, runned in terminal couple times

@vanvanni
Copy link

Hi I would like to try this within Node.JS, which package has the same features? Are you using the data from Futures?

@DaniloS99
Copy link

Hello, I am getting this error, can you help me to solve this? Thank you

... get_all_binance("BTCUSDT", "5m", False) File "<stdin>", line 21 get_all_binance("BTCUSDT", "5m", False) ^ SyntaxError: invalid syntax

@0xagf0x
Copy link

0xagf0x commented Jun 26, 2021

Hello, I am getting this error, can you help me to solve this? Thank you

... get_all_binance("BTCUSDT", "5m", False) File "<stdin>", line 21 get_all_binance("BTCUSDT", "5m", False) ^ SyntaxError: invalid syntax

try setting the save attr like this:

data = get_all_binance("BTCUSDT", "5m", save=False)

@phoenix1of1
Copy link

phoenix1of1 commented Jul 5, 2021

For future reference - the prerequisite package: simplejson, which is needed by the Bitmex API wrapper, requires Python 3.7 to function.
simplejson is not compatible with Python 3.9 at this time.
In addendum, the wrapper used for the Binance connection is python-binance by Sam McHardy so be sure to install this wrapper.

@asdsadsaasd12312132
Copy link

asdsadsaasd12312132 commented Jul 4, 2022

The Bitmex function doesn't seem to work. Binance works though.

> data = get_all_bitmex("BTCUSDT", "5m", save=True)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "bitmex_and_binance_data_downloader.py", line 65, in get_all_bitmex
    if os.path.isfile(filename): data_df = pd.read_csv(filename)
  File "bitmex_and_binance_data_downloader.py", line 37, in minutes_of_new_data
    elif source == "bitmex": old = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=1, reverse=False).result()[0][0]['timestamp']
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 271, in result
    swagger_result = self._get_swagger_result(incoming_response)
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 124, in wrapper
    return func(self, *args, **kwargs)
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 303, in _get_swagger_result
    self.request_config.response_callbacks,
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 353, in unmarshal_response
    raise_on_expected(incoming_response)
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 422, in raise_on_expected
    swagger_result=http_response.swagger_result)
bravado.exception.HTTPBadRequest: 400 Bad Request: {'error': {'message': 'This request has expired - `expires` is in the past. Current time: 1656901326', 'name': 'HTTPError'}}

@nkipa
Copy link

nkipa commented Aug 17, 2022

Getting this annoying message

C:\Users\Paul\anaconda3\lib\site-packages\dateparser\date_parser.py:35: PytzUsageWarning: The localize method is no longer necessary, as this time zone supports the fold attribute (PEP 495). For more details on migrating to a PEP 495-compliant implementation, see https://pytz-deprecation-shim.readthedocs.io/en/latest/migration.html
date_obj = stz.localize(date_obj)

@MiguelAngelMikeMeza
Copy link

Thanks Bro, its amazing

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment