Last active
January 5, 2024 06:35
-
-
Save luoyjx/6850ac26efdf039061daded308242ef3 to your computer and use it in GitHub Desktop.
streaming parse openAI completions, source from https://github.com/Nutlope/twitterbio/blob/main/utils/OpenAIStream.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { | |
createParser, | |
ParsedEvent, | |
ReconnectInterval, | |
} from "eventsource-parser"; | |
export interface OpenAIStreamPayload { | |
model: string; | |
prompt: string; | |
temperature: number; | |
top_p: number; | |
frequency_penalty: number; | |
presence_penalty: number; | |
max_tokens: number; | |
stream: boolean; | |
n: number; | |
} | |
export async function OpenAIStream(payload: OpenAIStreamPayload) { | |
const encoder = new TextEncoder(); | |
const decoder = new TextDecoder(); | |
let counter = 0; | |
const res = await fetch("https://api.openai.com/v1/completions", { | |
headers: { | |
"Content-Type": "application/json", | |
Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ""}`, | |
}, | |
method: "POST", | |
body: JSON.stringify(payload), | |
}); | |
const stream = new ReadableStream({ | |
async start(controller) { | |
// callback | |
function onParse(event: ParsedEvent | ReconnectInterval) { | |
if (event.type === "event") { | |
const data = event.data; | |
// https://beta.openai.com/docs/api-reference/completions/create#completions/create-stream | |
if (data === "[DONE]") { | |
controller.close(); | |
return; | |
} | |
try { | |
const json = JSON.parse(data); | |
const text = json.choices[0].text; | |
if (counter < 2 && (text.match(/\n/) || []).length) { | |
// this is a prefix character (i.e., "\n\n"), do nothing | |
return; | |
} | |
const queue = encoder.encode(text); | |
controller.enqueue(queue); | |
counter++; | |
} catch (e) { | |
// maybe parse error | |
controller.error(e); | |
} | |
} | |
} | |
// stream response (SSE) from OpenAI may be fragmented into multiple chunks | |
// this ensures we properly read chunks and invoke an event for each SSE event stream | |
const parser = createParser(onParse); | |
// https://web.dev/streams/#asynchronous-iteration | |
for await (const chunk of res.body as any) { | |
parser.feed(decoder.decode(chunk)); | |
} | |
}, | |
}); | |
return stream; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const generateBio = async (e: any) => { | |
e.preventDefault(); | |
setGeneratedBios(""); | |
setLoading(true); | |
const response = await fetch("/api/generate", { | |
method: "POST", | |
headers: { | |
"Content-Type": "application/json", | |
}, | |
body: JSON.stringify({ | |
prompt, | |
}), | |
}); | |
console.log("Edge function returned."); | |
if (!response.ok) { | |
throw new Error(response.statusText); | |
} | |
// This data is a ReadableStream | |
const data = response.body; | |
if (!data) { | |
return; | |
} | |
const reader = data.getReader(); | |
const decoder = new TextDecoder(); | |
let done = false; | |
while (!done) { | |
const { value, done: doneReading } = await reader.read(); | |
done = doneReading; | |
const chunkValue = decoder.decode(value); | |
setGeneratedBios((prev) => prev + chunkValue); | |
} | |
setLoading(false); | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment