Created
March 31, 2021 03:36
-
-
Save wholmgren/3300f475c13382c90ac84f956c842d2a to your computer and use it in GitHub Desktop.
Insert RTC Cocoa Beach weather data provided by Manajit Sengupta (NREL) into Solar Forecast Arbiter reference database
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"""Insert RTC Cocoa Beach weather data provided by Manajit Sengupta (NREL).""" | |
import sys | |
from pathlib import Path | |
import os | |
import numpy as np | |
import pandas as pd | |
from solarforecastarbiter.io.api import APISession, request_cli_access_token | |
from solarforecastarbiter.io.reference_observations.common import ( | |
_prepare_data_to_post | |
) | |
print(sys.path) | |
path = Path('fsec_rtc') | |
files = [f for f in path.iterdir()] | |
print(files) | |
rtc_to_sfa = { | |
'Direct_Wm2_Avg': 'dni', | |
'Global_Wm2_Avg': 'ghi', | |
'Diffuse_Wm2_Avg': 'dhi', | |
'Temp_C_Avg': 'air_temperature', | |
'WS_ms_Mean': 'wind_speed', | |
'RH_pct_Avg': 'relative_humidity' | |
} | |
username = "reference@solarforecastarbiter.org" | |
# assumes username/password is stored in default os x keychain | |
cmd = f"security find-internet-password -a '{username}' -w" | |
with os.popen(cmd) as p: | |
password = p.read().rstrip('\n') | |
token = request_cli_access_token(username, password) | |
session = APISession(token) | |
sites = session.list_sites() | |
cocoa = list(filter( | |
lambda x: 'DOE RTC Cocoa FL' == x.name and 'Reference' == x.provider, | |
sites))[0] | |
obs = session.list_observations() | |
cocoa_obs = list(filter(lambda x: x.site == cocoa, obs)) | |
print(*cocoa_obs, sep='\n\n') | |
def read_data_file(file): | |
df = pd.read_csv(file, index_col='Time', parse_dates=True) | |
df = df.tz_localize('Etc/GMT+5').rename(columns=rtc_to_sfa) | |
df = fix_st_to_dst(df) | |
return df | |
def fix_st_to_dst(df): | |
# assume duplicated only occurs at ST to DST transition | |
duplicated = df.index.duplicated() | |
num_duplicates = duplicated.sum() | |
first_duplicate_index = np.argmax(duplicated) | |
first_shift_point = first_duplicate_index - num_duplicates | |
new_hour = \ | |
df.index[first_shift_point:first_duplicate_index] - pd.Timedelta('1h') | |
new_hour_df = df.iloc[first_shift_point:first_duplicate_index].copy() | |
new_hour_df.index = new_hour | |
df_fixed = pd.concat([ | |
df.iloc[:first_shift_point], | |
new_hour_df, | |
df.iloc[first_duplicate_index:] | |
]) | |
return df_fixed | |
def post_data(session, data): | |
for obs in cocoa_obs: | |
try: | |
observation_df = _prepare_data_to_post( | |
data, obs.variable, obs, data.index[0], data.index[-1]) | |
except KeyError: | |
print(f'no {obs.variable} in data. continuing') | |
continue | |
print(f'posting {obs.variable}') | |
for split in np.array_split(observation_df, 12): | |
session.post_observation_values(obs.observation_id, split) | |
for file in files: | |
df = read_data_file(file) | |
post_data(session, df) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment