This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import streamlit as st | |
import pandas as pd | |
import altair as alt | |
import pydeck as pdk | |
import os | |
from dateutil.parser import parse | |
try: | |
from dotenv import load_dotenv, find_dotenv |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from h3 import h3 | |
from pygfl.easy import solve_gfl | |
def build_neighbor_edges(hexids): | |
# Hash the hexid to the position so we can easily look | |
# up where the original hexid position is in the array. | |
hexid_to_position = {h:ii for ii,h in enumerate(hexids)} | |
edges = [] | |
for h in hexids: | |
for n in h3.k_ring(h,1): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
data_frame.withColumn( | |
"prediction", | |
predict_pandas_udf(col("feature1"), col("feature2"), ...) | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
from pyspark.sql.functions import pandas_udf | |
from pyspark.sql.types import DoubleType | |
@pandas_udf(returnType=DoubleType()) | |
def predict_pandas_udf(*features): | |
""" Executes the prediction using numpy arrays. | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
my_data.rdd.mapPartitions(predict_partition).toDF() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
# We'll need this handy list more than once. It enforces the | |
# column order required by the model. | |
FEATURES = ["feature1", "feature2", "feature3", ...] | |
def predict_partition(rows): | |
""" Calls a vectorized prediction by loading the partition into memory. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
my_df.withColumn( | |
"predicted_score", | |
predict_udf(col("feature1"), col("feature2"), ...) | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pyspark.sql.functions import udf | |
from pyspark.sql.types import DoubleType | |
predict_udf = udf(predict, DoubleType()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import numpy as np | |
def predict(*features): | |
""" Performs a prediction on the features. | |
Parameters | |
---------- | |
features : List[float] | |
The feature values the model needs to make a prediction. | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def average_agreement(list1, list2, max_depth): | |
# Empty lists evaluate to false. | |
if (not list1) or (not list2): | |
return 0.0 | |
### NEW CODE ### | |
# Truncate the depth | |
max_list_len = max(len(list1), len(list2)) | |
max_depth = min(max_depth, max_list_len) |
NewerOlder