Mark mvandermeulen

Places for things to live: Openflow & OvS, K8s, Ansible, ML with Python & FastAPI

mvandermeulen / runnable.py

Created September 20, 2024 00:09 — forked from hwchase17/runnable.py

	from langchain.chains.openai_functions import create_structured_output_runnable
	from langchain.chat_models import ChatOpenAI
	from langchain.prompts import ChatPromptTemplate
	from langchain.pydantic_v1 import BaseModel, Field

	class Insight(BaseModel):
	insight: str = Field(description="""insight""")

	chat_model = ChatOpenAI(model_name="gpt-4-1106-preview")

mvandermeulen / prefect_and_generics.py

Created September 20, 2024 00:09 — forked from zzstoatzz/prefect_and_generics.py

generics in prefect

	from typing import Sequence, TypeVar, Generic

	from pydantic import BaseModel, TypeAdapter
	from prefect import flow, task


	ResultType = TypeVar("ResultType")

	class Result(BaseModel, Generic[ResultType]):
	data: Sequence[ResultType]

mvandermeulen / netcat.py

Created September 19, 2024 23:56 — forked from leonjza/netcat.py

Python Netcat

	import socket

	class Netcat:

	""" Python 'netcat like' module """

	def __init__(self, ip, port):

	self.buff = ""
	self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

mvandermeulen / classifier.py

Created September 19, 2024 23:56 — forked from zacstewart/classifier.py

Document Classification with scikit-learn

	import os
	import numpy
	from pandas import DataFrame
	from sklearn.feature_extraction.text import CountVectorizer
	from sklearn.naive_bayes import MultinomialNB
	from sklearn.pipeline import Pipeline
	from sklearn.cross_validation import KFold
	from sklearn.metrics import confusion_matrix, f1_score

	NEWLINE = '\n'

mvandermeulen / logger.py

Created September 19, 2024 23:55 — forked from nkhitrov/logger.py

Configure uvicorn logs with loguru for FastAPI

	"""
	WARNING: dont use loguru, use structlog
	https://gist.github.com/nkhitrov/38adbb314f0d35371eba4ffb8f27078f


	Configure handlers and formats for application loggers.
	"""
	import logging
	import sys
	from pprint import pformat

mvandermeulen / aioudp.py

Created September 19, 2024 23:55 — forked from vxgmichel/aioudp.py

High-level UDP endpoints for asyncio

	"""Provide high-level UDP endpoints for asyncio.

	Example:

	async def main():

	# Create a local UDP enpoint
	local = await open_local_endpoint('localhost', 8888)

	# Create a remote UDP enpoint, pointing to the first one

mvandermeulen / gpt.py

Created September 19, 2024 23:55 — forked from ustayready/gpt.py

CloudGPT - Use ChatGPT to analyze AWS policies for vulnerabilities

	import openai
	import boto3
	import json
	import time
	from typing import Dict, List

	openai.api_key = '### SET YOUR OPENAPI API KEY HERE ###'
	session = boto3.session.Session()
	client = session.client('iam')

mvandermeulen / postgres_manager.py

Created September 19, 2024 23:55 — forked from valferon/postgres_manager.py

Python script to take care of postgres backup and restore of data

	#!/usr/bin/python3
	import argparse
	import logging
	import subprocess
	import os
	import tempfile
	from tempfile import mkstemp

	import configparser
	import gzip

mvandermeulen / sft_trainer.py

Created September 19, 2024 23:54 — forked from lewtun/sft_trainer.py

Fine-tuning Mistral 7B with TRL & DeepSpeed ZeRO-3

	# This is a modified version of TRL's `SFTTrainer` example (https://github.com/huggingface/trl/blob/main/examples/scripts/sft_trainer.py),
	# adapted to run with DeepSpeed ZeRO-3 and Mistral-7B-V1.0. The settings below were run on 1 node of 8 x A100 (80GB) GPUs.
	#
	# Usage:
	# - Install the latest transformers & accelerate versions: `pip install -U transformers accelerate`
	# - Install deepspeed: `pip install deepspeed==0.9.5`
	# - Install TRL from main: pip install git+https://github.com/huggingface/trl.git
	# - Clone the repo: git clone github.com/huggingface/trl.git
	# - Copy this Gist into trl/examples/scripts
	# - Run from root of trl repo with: accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --gradient_accumulation_steps 8 examples/scripts/sft_trainer.py

mvandermeulen / hashtagify.py

Created September 19, 2024 23:54 — forked from shlomibabluki/hashtagify.py

	# coding=UTF-8
	from __future__ import division
	import nltk
	from collections import Counter

	# This is a simple tool for adding automatic hashtags into an article title
	# Created by Shlomi Babluki
	# Sep, 2013