Skip to content

Instantly share code, notes, and snippets.

View skrawcz's full-sized avatar

Stefan Krawczyk skrawcz

57 followers · 23 following
View GitHub Profile
@skrawcz
skrawcz / ray_burr_option1.py
Created August 12, 2024 23:55
Shows how to wrap a burr application for delegation to Ray. This is one possible strategy to make things run on Ray.
import copy
from IPython.display import Image, display
from IPython.core.display import HTML
import openai
from burr.core import ApplicationBuilder, State, default, graph, when
from burr.core.action import action
from burr.tracking import LocalTrackingClient
@skrawcz
skrawcz / run.py
Created July 19, 2024 18:07
Gist for guest post on blog
# python 3.12
"""
Hamilton demo. Runs the Hamilton code.
"""
import sys
import pprint
@skrawcz
skrawcz / parameterizing_config.py
Last active July 19, 2024 17:16
Shows how to potentially get around boilerplate config loading code by parameterizing it
# %%cell_to_module config_plate --display
from hamilton.function_modifiers import parameterize, source, value
class TablePaths:
pass # your code here
table_names = ["vendor", "predictions"] # update this
@parameterize(
@skrawcz
skrawcz / parameterize_sources_example.py
Last active July 19, 2024 17:25
@parameterize_sources example. This will create
# functions.py - declare and link your transformations as functions....
import pandas as pd
from hamilton.function_modifiers import parameterize_sources
def a(input: pd.Series) -> pd.Series:
return input % 7
def b(a: pd.Series) -> pd.Series:
return a * 2
@skrawcz
skrawcz / ai_response.py
Created June 19, 2024 22:56
Gist for the Hamilton, Burr, FalkorDB blog post
@action(
reads=["chat_history"],
writes=["chat_history"],
)
def AI_generate_response(state: State, client: openai.Client) -> tuple[dict, State]:
"""AI step to generate the response."""
messages = state["chat_history"]
response = client.chat.completions.create(
model="gpt-4-turbo-preview",
messages=messages,
@skrawcz
skrawcz / telephone-burr-application.py
Last active April 6, 2024 22:47
"""
This module demonstrates a telephone application
using Burr that:
- captions an image
- creates caption embeddings (for analysis)
- creates a new image based on the created caption
"""
import os
import uuid
@skrawcz
skrawcz / functions.py
Created January 21, 2024 05:23
Example using parallelism with Hamilton
# functions.py - declare and link your transformations as functions....
import pandas as pd
from hamilton.htypes import Parallelizable, Collect
def motor(motor_list: list[int]) -> Parallelizable[int]:
for _motor in motor_list:
yield _motor
def _is_motor_on(motor: int ) -> bool:
@skrawcz
skrawcz / hamilton_airflow_node.py
Created June 29, 2023 18:02
Usage pattern for Hamilton within an Airflow DAG task
from airflow.decorators import dag, task
from airflow.operators.python import get_current_context
# set the Airflow DAG parameters. This will appear in the Airflow UI.
DEFAULT_DAG_PARAMS = dict(
label="absenteeism_time_in_hours",
feature_set=[
"age_zero_mean_unit_variance",
"has_children",
"has_pet",
@skrawcz
skrawcz / async_funcs.py
Created June 13, 2023 18:42
Shows how to do async based functions in hamilton -- and then a suggestion for another way to parallelize
from hamilton.function_modifiers import extract_columns
import pandas as pd
async def _run_query(col_name: str, query_string: str) -> pd.DataFrame:
# this would go to the database -- ideally the client is passed in as a parameter
# the assumption here is that the database driver is asyncio based, else there's no
# value in doing this :)
return pd.DataFrame({col_name: [query_string]})
# async Hamilton func for query #1
@skrawcz
skrawcz / embed_this_code.py
Created February 7, 2023 21:50
Hamilton streaming pseudo code example
from hamilton import driver
import transforms
# load the client
kafka_client = KafkaClient() # or whatever
config = {...}
dr = driver.Driver(config, transforms, adapter=...)