Created
February 23, 2019 13:08
-
-
Save njanakiev/242681ecb3f52aa8e730db1a1420015b to your computer and use it in GitHub Desktop.
Load and merge data from Rescuetime
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests | |
| import datetime | |
| import pandas as pd | |
| import json | |
| import time | |
| # Read rescuetime key | |
| with open('key.txt', 'r') as f: | |
| key = f.read() | |
| # Get current date | |
| d = datetime.datetime.now() | |
| url = 'https://www.rescuetime.com/anapi/data' | |
| # https://www.rescuetime.com/anapi/setup/documentation#analytic-api-reference | |
| resolution_times = ['day', 'hour', 'minute'] | |
| restrict_kinds = ['document', 'overview', 'efficiency'] | |
| perspectives = ['interval', 'rank'] | |
| # Function for cleaning up strings | |
| #convert = lambda s: s.encode('ascii', 'ignore').replace('\n', '') | |
| convert = lambda s: s.replace('\n', '') | |
| start, end = '', '' | |
| for perspective in perspectives: | |
| for restrict_kind in restrict_kinds: | |
| for resolution_time in resolution_times: | |
| print(perspective, restrict_kind, resolution_time) | |
| try: | |
| response = requests.get(url, params={ | |
| 'key': key, | |
| 'format': 'json', | |
| 'perspective': perspective, | |
| 'interval': resolution_time, | |
| 'restrict_kind': restrict_kind, | |
| 'restrict_begin': '2017-01-01', | |
| 'restrict_end': d.strftime('%Y-%m-%d') | |
| }) | |
| print("Status Code : {}".format(response.status_code)) | |
| data = response.json() | |
| except Exception as e: | |
| print(response.text) | |
| print(e) | |
| exit() | |
| df = pd.DataFrame(data['rows'], columns=data['row_headers']) | |
| if restrict_kind == 'document': | |
| df['Document'] = df['Document'].apply(convert) | |
| if perspective == 'interval': | |
| start = data['rows'][ 0][0].split('T')[0] | |
| end = data['rows'][-1][0].split('T')[0] | |
| output = 'data/rt_{}_{}_{}_{}_{}.csv'.format( | |
| start, end, resolution_time, perspective, restrict_kind) | |
| print(output) | |
| df.to_csv(output, index=False) | |
| time.sleep(5) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os | |
| import pandas as pd | |
| import numpy as np | |
| resolution_times = ['day', 'hour', 'minute'] | |
| perspective = 'interval' | |
| restrict_kind = 'document' | |
| files = [os.path.join('data', f) for f in os.listdir('data') | |
| if f.endswith('.csv') and (restrict_kind in f) and (perspective in f)] | |
| for resolution_time in resolution_times: | |
| print(resolution_time) | |
| df_list = [] | |
| for f in files: | |
| if resolution_time in f: | |
| df = pd.read_csv(f) | |
| print(f, df.shape) | |
| df_list.append(df) | |
| df = pd.concat(df_list) | |
| df['Date'] = pd.to_datetime(df['Date']) | |
| df = df.sort_values(by='Date') | |
| df.drop_duplicates(inplace=True) | |
| print(df.shape) | |
| filepath = 'rt_{}_{}_{}.csv'.format( | |
| resolution_time, perspective, restrict_kind) | |
| df.to_csv(filepath, index=False) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment