Skip to content

Instantly share code, notes, and snippets.

@alberduris
Last active July 9, 2023 12:39
Show Gist options
  • Save alberduris/32e4ad5827cb01c28022ded982bfd8bc to your computer and use it in GitHub Desktop.
Save alberduris/32e4ad5827cb01c28022ded982bfd8bc to your computer and use it in GitHub Desktop.
STREAMING FROM CF WORKER ACCOMPLISHED!!
import { ChatOpenAI } from "langchain/chat_models";
import { HumanChatMessage } from "langchain/schema";
import { CallbackManager } from "langchain/callbacks";
interface Env {
OPENAI_API_KEY: string;
}
export default {
async fetch(
_request: Request,
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const { readable, writable } = new TransformStream();
const writer = writable.getWriter();
ctx.waitUntil(getResponse(writer, env));
return new Response(readable, {
status: 200,
headers: {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache, no-transform",
Connection: "keep-alive",
},
});
},
};
async function getResponse(
writer: WritableStreamDefaultWriter,
env: Env
): Promise<void> {
const encoder = new TextEncoder();
writer.write(encoder.encode("[START]\n"));
const chatStreaming = new ChatOpenAI({
openAIApiKey: env.OPENAI_API_KEY,
streaming: true,
callbackManager: CallbackManager.fromHandlers({
async handleLLMNewToken(token: string) {
// Send the token back to the client.
await writer.write(encoder.encode(token));
},
}),
});
await chatStreaming.call([
new HumanChatMessage(
"Write a song about sparkling water"
),
]);
writer.write(encoder.encode("\n[DONE]"));
return writer.close();
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment