Last active
March 2, 2019 14:13
-
-
Save yukoba/988ba3fe8c9e1e15752d79faacd66de1 to your computer and use it in GitHub Desktop.
三菱UFJ銀行公表の対顧客外国為替相場のダウンロード
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
三菱UFJ銀行公表の対顧客外国為替相場を http://www.murc-kawasesouba.jp/fx/ からダウンロードします。 | |
conda install lxml が必要です。 | |
""" | |
from datetime import date | |
from time import sleep | |
from typing import Optional | |
import pandas as pd | |
import requests | |
from lxml import html | |
def safe_strip(s): | |
"""エラーにならないように strip()""" | |
if type(s) == str: | |
return s.strip() | |
else: | |
return s | |
def download_1day(download_date: date) -> Optional[pd.DataFrame]: | |
"""1日分のデータをダウンロード""" | |
download_day = "{:02}{:02}{:02}".format(download_date.year % 100, download_date.month, download_date.day) | |
page = requests.get("http://www.murc-kawasesouba.jp/fx/past/index.php?id={}".format(download_day)) | |
tree = html.fromstring(page.content) | |
ths = [safe_strip(th.text) for th in tree.xpath("//table[@class='data-table7'][1]/tr/th")] | |
if len(ths) == 0: | |
return None | |
tds = [safe_strip(td.text) for td in tree.xpath("//table[@class='data-table7'][1]/tr/td")] | |
assert len(tds) % len(ths) == 0 | |
df = pd.DataFrame(dict([(ths[i], tds[i::len(ths)]) for i in range(len(ths))])) | |
df.insert(0, "date", [download_date] * len(df)) | |
return df | |
def download_1year(year: int) -> pd.DataFrame: | |
"""1年分のデータをダウンロード""" | |
dfs = [] | |
for month in range(1, 12 + 1): | |
for day in range(1, 31 + 1): | |
try: | |
download_date = date(year, month, day) | |
if download_date.weekday() <= 4: | |
print("download_date = {}".format(download_date)) | |
df = download_1day(download_date) | |
if df is not None: | |
dfs.append(df) | |
except ValueError: | |
pass | |
sleep(0.1) | |
df = pd.concat(dfs) | |
df = df.set_index([df.columns[0], df.columns[1]]) | |
return df | |
def write_one_currency_csv(df: pd.DataFrame, currency: str, path: str): | |
"""特定の為替だけをCSVに保存""" | |
df.xs(currency, level=1, drop_level=False).to_csv(path) | |
if __name__ == '__main__': | |
# print(download_1day(date(2019, 2, 19))) | |
df_1year = download_1year(2018) | |
write_one_currency_csv(df_1year, "US Dollar", "/tmp/usd.csv") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment