This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from langchain.chains.openai_functions import create_structured_output_runnable | |
from langchain.chat_models import ChatOpenAI | |
from langchain.prompts import ChatPromptTemplate | |
from langchain.pydantic_v1 import BaseModel, Field | |
class Insight(BaseModel): | |
insight: str = Field(description="""insight""") | |
chat_model = ChatOpenAI(model_name="gpt-4-1106-preview") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from typing import Sequence, TypeVar, Generic | |
from pydantic import BaseModel, TypeAdapter | |
from prefect import flow, task | |
ResultType = TypeVar("ResultType") | |
class Result(BaseModel, Generic[ResultType]): | |
data: Sequence[ResultType] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import socket | |
class Netcat: | |
""" Python 'netcat like' module """ | |
def __init__(self, ip, port): | |
self.buff = "" | |
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import numpy | |
from pandas import DataFrame | |
from sklearn.feature_extraction.text import CountVectorizer | |
from sklearn.naive_bayes import MultinomialNB | |
from sklearn.pipeline import Pipeline | |
from sklearn.cross_validation import KFold | |
from sklearn.metrics import confusion_matrix, f1_score | |
NEWLINE = '\n' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
WARNING: dont use loguru, use structlog | |
https://gist.github.com/nkhitrov/38adbb314f0d35371eba4ffb8f27078f | |
Configure handlers and formats for application loggers. | |
""" | |
import logging | |
import sys | |
from pprint import pformat |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"""Provide high-level UDP endpoints for asyncio. | |
Example: | |
async def main(): | |
# Create a local UDP enpoint | |
local = await open_local_endpoint('localhost', 8888) | |
# Create a remote UDP enpoint, pointing to the first one |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import openai | |
import boto3 | |
import json | |
import time | |
from typing import Dict, List | |
openai.api_key = '### SET YOUR OPENAPI API KEY HERE ###' | |
session = boto3.session.Session() | |
client = session.client('iam') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
import argparse | |
import logging | |
import subprocess | |
import os | |
import tempfile | |
from tempfile import mkstemp | |
import configparser | |
import gzip |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This is a modified version of TRL's `SFTTrainer` example (https://github.com/huggingface/trl/blob/main/examples/scripts/sft_trainer.py), | |
# adapted to run with DeepSpeed ZeRO-3 and Mistral-7B-V1.0. The settings below were run on 1 node of 8 x A100 (80GB) GPUs. | |
# | |
# Usage: | |
# - Install the latest transformers & accelerate versions: `pip install -U transformers accelerate` | |
# - Install deepspeed: `pip install deepspeed==0.9.5` | |
# - Install TRL from main: pip install git+https://github.com/huggingface/trl.git | |
# - Clone the repo: git clone github.com/huggingface/trl.git | |
# - Copy this Gist into trl/examples/scripts | |
# - Run from root of trl repo with: accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --gradient_accumulation_steps 8 examples/scripts/sft_trainer.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# coding=UTF-8 | |
from __future__ import division | |
import nltk | |
from collections import Counter | |
# This is a simple tool for adding automatic hashtags into an article title | |
# Created by Shlomi Babluki | |
# Sep, 2013 |
NewerOlder