Created
June 26, 2024 14:34
-
-
Save flexchar/fc83f9d0ffc04d16e49b0aebe8814451 to your computer and use it in GitHub Desktop.
easy way to use variety of (providers) models with open ai client // typescrit
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| export type Provider = | |
| | 'anyscale' | |
| | 'deepinfra' | |
| | 'deepinfra-helicone' | |
| | 'mistral' | |
| | 'openai-helicone' | |
| | 'openai' | |
| | 'anthropic' | |
| | 'groq' | |
| | 'google-ai-studio' | |
| | 'vertex-ai'; | |
| const PORTKEY_GATEWAY = 'https://url/v1'; | |
| function getPortKeyGatewayConfig( | |
| provider: string, | |
| api_key: string, | |
| ): ClientOptions { | |
| return { | |
| baseURL: PORTKEY_GATEWAY, | |
| apiKey: api_key, | |
| defaultHeaders: { | |
| 'x-portkey-provider': provider, | |
| }, | |
| }; | |
| } | |
| export function getOpenAiConfig( | |
| c: Context, | |
| provider: Provider, | |
| extraHeliconeMeta: Record<string, string> = {}, | |
| ): ClientOptions { | |
| const env = c.env as Bindings; | |
| const deepInfraConfig: ClientOptions = { | |
| apiKey: env.DEEPINFRA_API_KEY, | |
| baseURL: 'https://api.deepinfra.com/v1/openai', | |
| }; | |
| const mistralConfig: ClientOptions = { | |
| apiKey: env.MISTRAL_API_KEY, | |
| baseURL: 'https://api.mistral.ai/v1', | |
| }; | |
| const anyscaleConfig: ClientOptions = { | |
| apiKey: env.ANYSCALE_API_KEY, | |
| baseURL: 'https://api.anyscale.com/1/openai', | |
| }; | |
| const groqConfig: ClientOptions = { | |
| apiKey: env.GROQ_API_KEY, | |
| baseURL: 'https://api.groq.com/openai/v1', | |
| }; | |
| const openAiConfig: ClientOptions = { | |
| compatibility: 'strict', // strict mode, enable when using the OpenAI API | |
| baseURL: 'https://api.openai.com/v1', | |
| // baseURL: 'http://localhost:1234/v1', // LM Studio localhost testing | |
| apiKey: env.OPENAI_API_KEY, | |
| organization: env.OPENAI_ORGANIZATION, | |
| }; | |
| const heliconeHeaders = { | |
| 'Helicone-Auth': `Bearer ${c.env.HELICONE_API_KEY}`, | |
| ...extraHeliconeMeta, | |
| }; | |
| const heliconeOpenAiConfig: ClientOptions = { | |
| compatibility: 'strict', // strict mode, enable when using the OpenAI API | |
| apiKey: env.OPENAI_API_KEY, | |
| organization: env.OPENAI_ORGANIZATION, | |
| baseURL: 'https://oai.helicone.ai/v1', | |
| defaultHeaders: heliconeHeaders, | |
| headers: heliconeHeaders, | |
| }; | |
| const heliconeDeepInfraConfig: ClientOptions = { | |
| compatibility: 'compatible', // strict mode, enable when using the OpenAI API | |
| apiKey: env.DEEPINFRA_API_KEY, | |
| baseURL: 'https://deepinfra.helicone.ai/v1/openai', | |
| defaultHeaders: heliconeHeaders, | |
| headers: heliconeHeaders, | |
| }; | |
| // Claude supported thanks to Portkey Gateway | |
| // https://github.com/Portkey-AI/gateway?tab=readme-ov-file#supported-providers | |
| const anthropicConfig: ClientOptions = getPortKeyGatewayConfig( | |
| 'anthropic', | |
| env.ANTHROPIC_KEY, | |
| ); | |
| // Google AI Studio supported thanks to Portkey Gateway | |
| // Note that AI Studio is not the same as VertexAI | |
| const googleStudioConfig: ClientOptions = getPortKeyGatewayConfig( | |
| 'google', | |
| env.GOOGLE_GEMINI_API_KEY, | |
| ); | |
| switch (provider) { | |
| case 'anyscale': | |
| return anyscaleConfig; | |
| case 'deepinfra': | |
| return deepInfraConfig; | |
| case 'deepinfra-helicone': | |
| return heliconeDeepInfraConfig; | |
| case 'mistral': | |
| return mistralConfig; | |
| case 'openai': | |
| return openAiConfig; | |
| case 'openai-helicone': | |
| return heliconeOpenAiConfig; | |
| case 'anthropic': | |
| return anthropicConfig; | |
| case 'groq': | |
| return groqConfig; | |
| case 'google-ai-studio': | |
| return googleStudioConfig; | |
| default: | |
| throw new Error(`Unknown OpenAi config provider: ${provider}`); | |
| } | |
| } | |
| export async function getVertexAiConfig( | |
| c: Context, | |
| region?: string, | |
| ): Promise<ClientOptions> { | |
| const res = await callPhp(c, '/cloudflare/vertex-ai-config'); | |
| // ToDo: cache somewhere for until the token expires | |
| const config: { | |
| project_id: string; | |
| api_key: string; | |
| ttl: number; // always 3599 | |
| created: number; // unix timestamp | |
| } = await res.json(); | |
| // prefer europe-west3 to match Frankfurt | |
| // https://www.economize.cloud/resources/gcp/regions-zones-map/ | |
| // Could be cool to match the region with the user's location | |
| const isEu = c.req.raw.cf?.isEUCountry; | |
| const vertex_region = region ?? (isEu ? 'europe-west3' : 'us-central1'); | |
| const portkeyConfig: ClientOptions = { | |
| baseURL: PORTKEY_GATEWAY, | |
| apiKey: '', | |
| defaultHeaders: { | |
| 'x-portkey-config': JSON.stringify({ | |
| provider: 'vertex-ai', | |
| vertex_region, | |
| vertex_project_id: config.project_id, | |
| api_key: config.api_key, | |
| }), | |
| }, | |
| }; | |
| // console.log({ portkeyConfig, config }); | |
| return portkeyConfig; | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment