From 46728b5cb19a84e65afbf25f0c2b59dcae3d09b7 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Thu, 21 Nov 2024 17:08:15 +0530 Subject: [PATCH 01/35] Updating proxy handler to use tryTargetsRecursively function itself --- src/handlers/handlerUtils.ts | 54 ++++- src/handlers/proxyHandler.ts | 265 ++------------------- src/handlers/responseHandlers.ts | 2 +- src/services/transformToProviderRequest.ts | 5 + 4 files changed, 81 insertions(+), 245 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index afdb5f03..ca98086e 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -14,6 +14,8 @@ import { CONTENT_TYPES, HUGGING_FACE, STABILITY_AI, + OLLAMA, + TRITON, } from '../globals'; import Providers from '../providers'; import { ProviderAPIConfig, endpointStrings } from '../providers/types'; @@ -90,6 +92,41 @@ export function constructRequest( return fetchOptions; } +function getProxyPath( + requestURL: string, + proxyProvider: string, + proxyEndpointPath: string, + customHost: string +) { + let reqURL = new URL(requestURL); + let reqPath = reqURL.pathname; + const reqQuery = reqURL.search; + reqPath = reqPath.replace(proxyEndpointPath, ''); + + if (customHost) { + return `${customHost}${reqPath}${reqQuery}`; + } + + const providerBasePath = Providers[proxyProvider].api.getBaseURL({ + providerOptions: {}, + }); + if (proxyProvider === AZURE_OPEN_AI) { + return `https:/${reqPath}${reqQuery}`; + } + + if (proxyProvider === OLLAMA || proxyProvider === TRITON) { + return `https:/${reqPath}`; + } + let proxyPath = `${providerBasePath}${reqPath}${reqQuery}`; + + // Fix specific for Anthropic SDK calls. Is this needed? - Yes + if (proxyProvider === ANTHROPIC) { + proxyPath = proxyPath.replace('/v1/v1/', '/v1/'); + } + + return proxyPath; +} + /** * Selects a provider based on their assigned weights. * The weight is used to determine the probability of each provider being chosen. @@ -456,7 +493,8 @@ export async function tryPost( inputParams: Params | FormData, requestHeaders: Record, fn: endpointStrings, - currentIndex: number | string + currentIndex: number | string, + method: string = 'POST' ): Promise { const overrideParams = providerOption?.overrideParams || {}; const params: Params = { ...inputParams, ...overrideParams }; @@ -518,7 +556,14 @@ export async function tryPost( fn, gatewayRequestBody: params, }); - const url = `${baseUrl}${endpoint}`; + + let url: string; + if (fn=="proxy") { + let proxyPath = c.req.url.indexOf('/v1/proxy') > -1 ? '/v1/proxy' : '/v1'; + url = getProxyPath(c.req.url, provider, proxyPath, customHost); + } else { + url = `${baseUrl}${endpoint}`; + } const headers = await apiConfig.headers({ c, @@ -533,7 +578,7 @@ export async function tryPost( const fetchOptions = constructRequest( headers, provider, - 'POST', + method, forwardHeaders, requestHeaders ); @@ -926,7 +971,8 @@ export async function tryTargetsRecursively( request, requestHeaders, fn, - currentJsonPath + currentJsonPath, + method ); } catch (error: any) { // tryPost always returns a Response. diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index 2dcc147d..b8d99338 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -15,12 +15,15 @@ import Providers from '../providers'; import { Config, ShortConfig } from '../types/requestBody'; import { convertKeysToCamelCase, getStreamingMode } from '../utils'; import { + constructConfigFromRequestHeaders, fetchProviderOptionsFromConfig, tryProvidersInSequence, + tryTargetsRecursively, updateResponseHeaders, } from './handlerUtils'; import { retryRequest } from './retryHandler'; import { responseHandler } from './responseHandlers'; +import { RouterError } from '../errors/RouterError'; // Find the proxy provider function proxyProvider(proxyModeHeader: string, providerHeader: string) { const proxyProvider = proxyModeHeader?.split(' ')[1] ?? providerHeader; @@ -122,256 +125,38 @@ function headersToSend( export async function proxyHandler(c: Context): Promise { try { - const requestHeaders = Object.fromEntries(c.req.raw.headers); - const requestContentType = requestHeaders['content-type']?.split(';')[0]; - const { requestJSON, requestFormData, requestBinary } = - await getRequestData(c.req.raw, requestContentType); - const store: Record = { - proxyProvider: proxyProvider( - requestHeaders[HEADER_KEYS.MODE], - requestHeaders[`x-${POWERED_BY}-provider`] - ), - reqBody: requestJSON, - requestFormData: requestFormData, - customHeadersToAvoid: env(c).CUSTOM_HEADERS_TO_IGNORE ?? [], - proxyPath: c.req.url.indexOf('/v1/proxy') > -1 ? '/v1/proxy' : '/v1', - }; - - let requestConfig: Config | ShortConfig | null = null; - if (requestHeaders[`x-${POWERED_BY}-config`]) { - requestConfig = JSON.parse(requestHeaders[`x-${POWERED_BY}-config`]); - if (requestConfig && 'provider' in requestConfig) { - store.proxyProvider = requestConfig.provider; - } - } - - const customHost = - requestHeaders[HEADER_KEYS.CUSTOM_HOST] || - requestConfig?.customHost || - ''; - let urlToFetch = getProxyPath( - c.req.url, - store.proxyProvider, - store.proxyPath, - customHost - ); - store.isStreamingMode = getStreamingMode( - store.reqBody, - store.proxyProvider, - urlToFetch + let request = await c.req.json(); + let requestHeaders = Object.fromEntries(c.req.raw.headers); + const camelCaseConfig = constructConfigFromRequestHeaders(requestHeaders); + + const tryTargetsResponse = await tryTargetsRecursively( + c, + camelCaseConfig, + request, + requestHeaders, + 'proxy', + c.req.method, + 'config' ); - if ( - requestConfig && - (('options' in requestConfig && requestConfig.options) || - ('targets' in requestConfig && requestConfig.targets) || - ('provider' in requestConfig && requestConfig.provider)) - ) { - let providerOptions = fetchProviderOptionsFromConfig(requestConfig); - - if (!providerOptions) { - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Could not find a provider option.', - }), - { - status: 400, - headers: { - 'content-type': 'application/json', - }, - } - ); - } - - providerOptions = providerOptions.map((po) => ({ - ...po, - urlToFetch, - })); - - try { - return await tryProvidersInSequence( - c, - providerOptions, - store.reqBody, - requestHeaders, - 'proxy' - ); - } catch (error: any) { - const errorArray = JSON.parse(error.message); - return new Response(errorArray[errorArray.length - 1].errorObj, { - status: errorArray[errorArray.length - 1].status, - headers: { - 'content-type': 'application/json', - }, - }); - } - } - - if (requestConfig) { - requestConfig = convertKeysToCamelCase( - requestConfig as Record, - ['override_params', 'params', 'metadata'] - ) as Config | ShortConfig; - } - - let body; - if (requestContentType.startsWith(CONTENT_TYPES.GENERIC_AUDIO_PATTERN)) { - body = requestBinary; - } else if (requestContentType === CONTENT_TYPES.MULTIPART_FORM_DATA) { - body = store.requestFormData; - } else { - body = JSON.stringify(store.reqBody); - } - - let fetchOptions = { - headers: headersToSend(requestHeaders, store.customHeadersToAvoid), - method: c.req.method, - body: body, - }; - - let retryCount = 0; - let retryStatusCodes = RETRY_STATUS_CODES; - if (requestHeaders[HEADER_KEYS.RETRIES]) { - retryCount = parseInt(requestHeaders[HEADER_KEYS.RETRIES]); - } else if ( - requestConfig?.retry && - typeof requestConfig.retry === 'object' - ) { - (retryCount = requestConfig.retry?.attempts ?? 1), - (retryStatusCodes = - requestConfig.retry?.onStatusCodes ?? RETRY_STATUS_CODES); - } - - retryCount = Math.min(retryCount, MAX_RETRIES); - - const getFromCacheFunction = c.get('getFromCache'); - const cacheIdentifier = c.get('cacheIdentifier'); - - let cacheResponse, cacheKey, cacheMaxAge; - let cacheStatus = 'DISABLED'; - let cacheMode = requestHeaders[HEADER_KEYS.CACHE]; - - if ( - requestConfig?.cache && - typeof requestConfig.cache === 'object' && - requestConfig.cache.mode - ) { - cacheMode = requestConfig.cache.mode; - cacheMaxAge = requestConfig.cache.maxAge; - } else if ( - requestConfig?.cache && - typeof requestConfig.cache === 'string' - ) { - cacheMode = requestConfig.cache; - } + return tryTargetsResponse; + } catch (err: any) { + console.log('proxy error', err.message); + let statusCode = 500; + let errorMessage = `Proxy error: ${err.message}`; - if (getFromCacheFunction && cacheMode) { - [cacheResponse, cacheStatus, cacheKey] = await getFromCacheFunction( - env(c), - { ...requestHeaders, ...fetchOptions.headers }, - store.reqBody, - urlToFetch, - cacheIdentifier, - cacheMode - ); - if (cacheResponse) { - const { response: cacheMappedResponse } = await responseHandler( - new Response(cacheResponse, { - headers: { - 'content-type': 'application/json', - }, - }), - false, - store.proxyProvider, - undefined, - urlToFetch, - false, - store.reqBody, - false - ); - c.set('requestOptions', [ - { - providerOptions: { - ...store.reqBody, - provider: store.proxyProvider, - requestURL: urlToFetch, - rubeusURL: 'proxy', - }, - requestParams: store.reqBody, - response: cacheMappedResponse.clone(), - cacheStatus: cacheStatus, - cacheKey: cacheKey, - cacheMode: cacheMode, - cacheMaxAge: cacheMaxAge, - }, - ]); - updateResponseHeaders( - cacheMappedResponse, - 0, - store.reqBody, - cacheStatus, - 0, - requestHeaders[HEADER_KEYS.TRACE_ID] ?? '' - ); - return cacheMappedResponse; - } + if (err instanceof RouterError) { + statusCode = 400; + errorMessage = err.message; } - // Make the API call to the provider - let [lastResponse, lastAttempt] = await retryRequest( - urlToFetch, - fetchOptions, - retryCount, - retryStatusCodes, - null - ); - const { response: mappedResponse } = await responseHandler( - lastResponse, - store.isStreamingMode, - store.proxyProvider, - undefined, - urlToFetch, - false, - store.reqBody, - false - ); - updateResponseHeaders( - mappedResponse, - 0, - store.reqBody, - cacheStatus, - lastAttempt ?? 0, - requestHeaders[HEADER_KEYS.TRACE_ID] ?? '' - ); - - c.set('requestOptions', [ - { - providerOptions: { - ...store.reqBody, - provider: store.proxyProvider, - requestURL: urlToFetch, - rubeusURL: 'proxy', - }, - requestParams: store.reqBody, - response: mappedResponse.clone(), - cacheStatus: cacheStatus, - cacheKey: cacheKey, - cacheMode: cacheMode, - cacheMaxAge: cacheMaxAge, - }, - ]); - - return mappedResponse; - } catch (err: any) { - console.log('proxy error', err.message); return new Response( JSON.stringify({ status: 'failure', - message: 'Something went wrong', + message: errorMessage, }), { - status: 500, + status: statusCode, headers: { 'content-type': 'application/json', }, diff --git a/src/handlers/responseHandlers.ts b/src/handlers/responseHandlers.ts index 15db82bc..d8b35865 100644 --- a/src/handlers/responseHandlers.ts +++ b/src/handlers/responseHandlers.ts @@ -92,7 +92,7 @@ export async function responseHandler( return { response: streamingResponse, responseJson: null }; } - if (streamingMode && response.status === 200) { + if (streamingMode && response.status === 200 && responseTransformerFunction) { return { response: handleStreamingMode( response, diff --git a/src/services/transformToProviderRequest.ts b/src/services/transformToProviderRequest.ts index 08da34cf..fa4a39fd 100644 --- a/src/services/transformToProviderRequest.ts +++ b/src/services/transformToProviderRequest.ts @@ -188,6 +188,11 @@ export const transformToProviderRequest = ( fn: endpointStrings ) => { if (MULTIPART_FORM_DATA_ENDPOINTS.includes(fn)) return inputParams; + + if (fn === "proxy") { + return params; + } + const providerAPIConfig = ProviderConfigs[provider].api; if ( providerAPIConfig.transformToFormData && From 6aa30459a37b3fa635de7d60b26f6436f3862922 Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 21 Nov 2024 17:51:50 +0530 Subject: [PATCH 02/35] chore: remove redundant response transformer check from stream handling --- src/handlers/responseHandlers.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/handlers/responseHandlers.ts b/src/handlers/responseHandlers.ts index d8b35865..1c5ef572 100644 --- a/src/handlers/responseHandlers.ts +++ b/src/handlers/responseHandlers.ts @@ -91,8 +91,7 @@ export async function responseHandler( ); return { response: streamingResponse, responseJson: null }; } - - if (streamingMode && response.status === 200 && responseTransformerFunction) { + if (streamingMode && response.status === 200) { return { response: handleStreamingMode( response, From e3362bcbe3c6fbd12826a70e48359fddc21f709e Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Thu, 21 Nov 2024 19:22:41 +0530 Subject: [PATCH 03/35] fixes for proxy requests using multi-part form data --- src/handlers/handlerUtils.ts | 23 +++++++++++---- src/handlers/proxyHandler.ts | 34 ++++++++-------------- src/services/transformToProviderRequest.ts | 2 +- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index ca98086e..e947479b 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -50,7 +50,8 @@ export function constructRequest( provider: string, method: string, forwardHeaders: string[], - requestHeaders: Record + requestHeaders: Record, + fn: endpointStrings ) { let baseHeaders: any = { 'content-type': 'application/json', @@ -72,13 +73,18 @@ export function constructRequest( }); // Add any headers that the model might need - headers = { ...baseHeaders, ...headers, ...forwardHeadersMap }; + headers = { + ...baseHeaders, + ...headers, + ...forwardHeadersMap, + ...(fn === 'proxy' ? requestHeaders : {}), + }; let fetchOptions: RequestInit = { method, headers, }; - const contentType = headers['content-type']; + const contentType = headers['content-type']?.split(';')[0]; const isGetMethod = method === 'GET'; const isMultipartFormData = contentType === CONTENT_TYPES.MULTIPART_FORM_DATA; const shouldDeleteContentTypeHeader = @@ -318,7 +324,8 @@ export async function tryPostProxy( provider, method, forwardHeaders, - requestHeaders + requestHeaders, + fn ); if (method === 'POST') { @@ -580,11 +587,15 @@ export async function tryPost( provider, method, forwardHeaders, - requestHeaders + requestHeaders, + fn ); + const headerContentType = headers[HEADER_KEYS.CONTENT_TYPE]; + const requestContentType = requestHeaders[HEADER_KEYS.CONTENT_TYPE.toLowerCase()]?.split(';')[0]; + fetchOptions.body = - headers[HEADER_KEYS.CONTENT_TYPE] === CONTENT_TYPES.MULTIPART_FORM_DATA + (headerContentType === CONTENT_TYPES.MULTIPART_FORM_DATA || (fn=="proxy" && requestContentType === CONTENT_TYPES.MULTIPART_FORM_DATA)) ? (transformedRequestBody as FormData) : JSON.stringify(transformedRequestBody); diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index b8d99338..5c188bb2 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -66,27 +66,20 @@ function getProxyPath( } async function getRequestData(request: Request, contentType: string) { - let requestJSON: Record = {}; - let requestFormData; - let requestBody = ''; - let requestBinary: ArrayBuffer = new ArrayBuffer(0); - + let finalRequest: any; if (contentType == CONTENT_TYPES.APPLICATION_JSON) { if (['GET', 'DELETE'].includes(request.method)) { - return { requestJSON, requestFormData }; + finalRequest = {}; + } else { + finalRequest = await request.json(); } - requestBody = await request.text(); - requestJSON = JSON.parse(requestBody); } else if (contentType == CONTENT_TYPES.MULTIPART_FORM_DATA) { - requestFormData = await request.formData(); - requestFormData.forEach(function (value, key) { - requestJSON[key] = value; - }); + finalRequest = await request.formData(); } else if (contentType.startsWith(CONTENT_TYPES.GENERIC_AUDIO_PATTERN)) { - requestBinary = await request.arrayBuffer(); + finalRequest = await request.arrayBuffer(); } - return { requestJSON, requestFormData, requestBinary }; + return finalRequest; } function headersToSend( @@ -100,12 +93,6 @@ function headersToSend( ...customHeadersToIgnore, ...headersToAvoidForCloudflare, ]; - if ( - headersObj['content-type']?.split(';')[0] === - CONTENT_TYPES.MULTIPART_FORM_DATA - ) { - headersToAvoid.push('content-type'); - } headersToAvoid.push('content-length'); Object.keys(headersObj).forEach((key: string) => { if ( @@ -125,15 +112,18 @@ function headersToSend( export async function proxyHandler(c: Context): Promise { try { - let request = await c.req.json(); let requestHeaders = Object.fromEntries(c.req.raw.headers); + const requestContentType = requestHeaders['content-type'].split(';')[0]; + + const request = await getRequestData(c.req.raw, requestContentType); + const camelCaseConfig = constructConfigFromRequestHeaders(requestHeaders); const tryTargetsResponse = await tryTargetsRecursively( c, camelCaseConfig, request, - requestHeaders, + headersToSend(requestHeaders, []), 'proxy', c.req.method, 'config' diff --git a/src/services/transformToProviderRequest.ts b/src/services/transformToProviderRequest.ts index fa4a39fd..894ed84f 100644 --- a/src/services/transformToProviderRequest.ts +++ b/src/services/transformToProviderRequest.ts @@ -187,7 +187,7 @@ export const transformToProviderRequest = ( inputParams: Params | FormData, fn: endpointStrings ) => { - if (MULTIPART_FORM_DATA_ENDPOINTS.includes(fn)) return inputParams; + if (inputParams instanceof FormData) return inputParams; if (fn === "proxy") { return params; From 3ccfea95c0b09bc3ebba5fc9efdd99a363eda03b Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Thu, 21 Nov 2024 19:34:51 +0530 Subject: [PATCH 04/35] Removed deprecated handlers --- src/handlers/chatCompleteHandler.ts | 101 ----------------- src/handlers/completeHandler.ts | 89 --------------- src/handlers/embedHandler.ts | 82 -------------- src/handlers/proxyGetHandler.ts | 163 ---------------------------- src/index.ts | 46 +++----- 5 files changed, 13 insertions(+), 468 deletions(-) delete mode 100644 src/handlers/chatCompleteHandler.ts delete mode 100644 src/handlers/completeHandler.ts delete mode 100644 src/handlers/embedHandler.ts delete mode 100644 src/handlers/proxyGetHandler.ts diff --git a/src/handlers/chatCompleteHandler.ts b/src/handlers/chatCompleteHandler.ts deleted file mode 100644 index 6544f3d1..00000000 --- a/src/handlers/chatCompleteHandler.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Targets } from '../types/requestBody'; -import { - fetchProviderOptionsFromConfig, - tryProvidersInSequence, -} from './handlerUtils'; -import { Context } from 'hono'; - -/** - * @deprecated - * Handles the 'chatComplete' API request by selecting the appropriate provider(s) and making the request to them. - * - * The environment variables (`env`) should be the cloudflare environment variables. - * - * The `request` parameter is an object that includes: - * - `config`: An object that specifies how the request should be handled. It can either be a `ShortConfig` object with `provider` and `apiKeyName` fields, or a `Config` object with `mode` and `options` fields. - * The `mode` can be "single" (uses the first provider), "loadbalance" (selects one provider based on weights), or "fallback" (uses all providers in the given order). - * - `params`: An object that specifies the parameters of the request, such as `model`, `prompt`, `messages`, etc. - * - * If a provider is specified in the request config, that provider is used. Otherwise, the provider options are determined based on the mode in the request config. - * If no provider options can be determined, an error is thrown. If the request to the provider(s) fails, an error is also thrown. - * - * This function returns a `CResponse` object which includes `id`, `object`, `created`, `model`, `choices`, and `usage` fields. - * - * @param {any} env - The cloudflare environment variables. - * @param {RequestBody} request - The request body, which includes the config for the request (provider, mode, etc.). - * @returns {Promise} - The response from the provider. - * @throws Will throw an error if no provider options can be determined or if the request to the provider(s) fails. - */ -export async function chatCompleteHandler(c: Context): Promise { - try { - const request = await c.req.json(); - const requestHeaders = Object.fromEntries(c.req.raw.headers); - if ( - request.config?.targets && - request.config?.targets?.filter((t: Targets) => t.targets).length > 0 - ) { - return new Response( - JSON.stringify({ - status: 'failure', - message: - 'Please use the latest routes or SDK to use this version of config.', - }), - { - status: 400, - headers: { - 'content-type': 'application/json', - }, - } - ); - } - - const providerOptions = fetchProviderOptionsFromConfig(request.config); - - if (!providerOptions) { - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Could not find a provider option.', - }), - { - status: 400, - headers: { - 'content-type': 'application/json', - }, - } - ); - } - - try { - return await tryProvidersInSequence( - c, - providerOptions, - request.params, - requestHeaders, - 'chatComplete' - ); - } catch (error: any) { - const errorArray = JSON.parse(error.message); - return new Response(errorArray[errorArray.length - 1].errorObj, { - status: errorArray[errorArray.length - 1].status, - headers: { - 'content-type': 'application/json', - }, - }); - } - } catch (err: any) { - console.log('chatComplete error', err.message); - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Something went wrong', - }), - { - status: 500, - headers: { - 'content-type': 'application/json', - }, - } - ); - } -} diff --git a/src/handlers/completeHandler.ts b/src/handlers/completeHandler.ts deleted file mode 100644 index ffffbda1..00000000 --- a/src/handlers/completeHandler.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Targets } from '../types/requestBody'; -import { - fetchProviderOptionsFromConfig, - tryProvidersInSequence, -} from './handlerUtils'; -import { Context } from 'hono'; - -/** - * @deprecated - * Handles the 'complete' API request by selecting the appropriate provider(s) and making the request to them. - * If a provider is specified in the request config, that provider is used. Otherwise, the provider options are determined based on the mode in the request config. - * If no provider options can be determined, an error is thrown. If the request to the provider(s) fails, an error is also thrown. - * - * @returns {Promise} - The response from the provider. - * @throws Will throw an error if no provider options can be determined or if the request to the provider(s) fails. - */ -export async function completeHandler(c: Context): Promise { - try { - const request = await c.req.json(); - const requestHeaders = Object.fromEntries(c.req.raw.headers); - if ( - request.config?.targets && - request.config?.targets?.filter((t: Targets) => t.targets).length > 0 - ) { - return new Response( - JSON.stringify({ - status: 'failure', - message: - 'Please use the latest routes or SDK to use this version of config.', - }), - { - status: 400, - headers: { - 'content-type': 'application/json', - }, - } - ); - } - - const providerOptions = fetchProviderOptionsFromConfig(request.config); - - if (!providerOptions) { - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Could not find a provider option.', - }), - { - status: 400, - headers: { - 'content-type': 'application/json', - }, - } - ); - } - - try { - return await tryProvidersInSequence( - c, - providerOptions, - request.params, - requestHeaders, - 'complete' - ); - } catch (error: any) { - const errorArray = JSON.parse(error.message); - return new Response(errorArray[errorArray.length - 1].errorObj, { - status: errorArray[errorArray.length - 1].status, - headers: { - 'content-type': 'application/json', - }, - }); - } - } catch (err: any) { - console.log('complete error', err.message); - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Something went wrong', - }), - { - status: 500, - headers: { - 'content-type': 'application/json', - }, - } - ); - } -} diff --git a/src/handlers/embedHandler.ts b/src/handlers/embedHandler.ts deleted file mode 100644 index 5b9b2770..00000000 --- a/src/handlers/embedHandler.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { Context } from 'hono'; -import { - fetchProviderOptionsFromConfig, - tryProvidersInSequence, -} from './handlerUtils'; -import { Targets } from '../types/requestBody'; - -/** - * @deprecated - */ -export async function embedHandler(c: Context): Promise { - try { - const request = await c.req.json(); - const requestHeaders = Object.fromEntries(c.req.raw.headers); - if ( - request.config?.targets && - request.config?.targets?.filter((t: Targets) => t.targets).length > 0 - ) { - return new Response( - JSON.stringify({ - status: 'failure', - message: - 'Please use the latest routes or SDK to use this version of config.', - }), - { - status: 400, - headers: { - 'content-type': 'application/json', - }, - } - ); - } - - let providerOptions = fetchProviderOptionsFromConfig(request.config); - if (!providerOptions) { - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Could not find a provider option.', - }), - { - status: 400, - headers: { - 'content-type': 'application/json', - }, - } - ); - } - - try { - return await tryProvidersInSequence( - c, - providerOptions, - request.params, - requestHeaders, - 'embed' - ); - } catch (error: any) { - console.error(`embed error: ${error.message}`); - const errorArray = JSON.parse(error.message); - return new Response(errorArray[errorArray.length - 1].errorObj, { - status: errorArray[errorArray.length - 1].status, - headers: { - 'content-type': 'application/json', - }, - }); - } - } catch (err: any) { - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Something went wrong', - }), - { - status: 500, - headers: { - 'content-type': 'application/json', - }, - } - ); - } -} diff --git a/src/handlers/proxyGetHandler.ts b/src/handlers/proxyGetHandler.ts deleted file mode 100644 index 736f71af..00000000 --- a/src/handlers/proxyGetHandler.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { Context } from 'hono'; -import { retryRequest } from './retryHandler'; -import Providers from '../providers'; -import { - ANTHROPIC, - MAX_RETRIES, - HEADER_KEYS, - RETRY_STATUS_CODES, - POWERED_BY, - AZURE_OPEN_AI, -} from '../globals'; -import { updateResponseHeaders } from './handlerUtils'; -import { env } from 'hono/adapter'; -import { responseHandler } from './responseHandlers'; -// Find the proxy provider -function proxyProvider(proxyModeHeader: string, providerHeader: string) { - const proxyProvider = proxyModeHeader?.split(' ')[1] ?? providerHeader; - return proxyProvider; -} - -function getProxyPath( - requestURL: string, - proxyProvider: string, - proxyEndpointPath: string, - customHost: string -) { - let reqURL = new URL(requestURL); - let reqPath = reqURL.pathname; - const reqQuery = reqURL.search; - reqPath = reqPath.replace(proxyEndpointPath, ''); - - if (customHost) { - return `${customHost}${reqPath}${reqQuery}`; - } - - const providerBasePath = Providers[proxyProvider].api.getBaseURL({ - providerOptions: {}, - }); - if (proxyProvider === AZURE_OPEN_AI) { - return `https:/${reqPath}${reqQuery}`; - } - let proxyPath = `${providerBasePath}${reqPath}${reqQuery}`; - - // Fix specific for Anthropic SDK calls. Is this needed? - Yes - if (proxyProvider === ANTHROPIC) { - proxyPath = proxyPath.replace('/v1/v1/', '/v1/'); - } - - return proxyPath; -} - -function headersToSend( - headersObj: Record, - customHeadersToIgnore: Array -): Record { - let final: Record = {}; - const poweredByHeadersPattern = `x-${POWERED_BY}-`; - const headersToAvoid = [...customHeadersToIgnore]; - headersToAvoid.push('content-length'); - Object.keys(headersObj).forEach((key: string) => { - if ( - !headersToAvoid.includes(key) && - !key.startsWith(poweredByHeadersPattern) - ) { - final[key] = headersObj[key]; - } - }); - final['accept-encoding'] = 'gzip, deflate'; - return final; -} - -export async function proxyGetHandler(c: Context): Promise { - try { - const requestHeaders = Object.fromEntries(c.req.raw.headers); - delete requestHeaders['content-type']; - const store: Record = { - proxyProvider: proxyProvider( - requestHeaders[HEADER_KEYS.MODE], - requestHeaders[HEADER_KEYS.PROVIDER] - ), - customHeadersToAvoid: env(c).CUSTOM_HEADERS_TO_IGNORE ?? [], - reqBody: {}, - proxyPath: c.req.url.indexOf('/v1/proxy') > -1 ? '/v1/proxy' : '/v1', - }; - - const customHost = requestHeaders[HEADER_KEYS.CUSTOM_HOST] || ''; - - let urlToFetch = getProxyPath( - c.req.url, - store.proxyProvider, - store.proxyPath, - customHost - ); - - let fetchOptions = { - headers: headersToSend(requestHeaders, store.customHeadersToAvoid), - method: c.req.method, - }; - - let retryCount = Math.min( - parseInt(requestHeaders[HEADER_KEYS.RETRIES]) || 1, - MAX_RETRIES - ); - - let [lastResponse, lastAttempt] = await retryRequest( - urlToFetch, - fetchOptions, - retryCount, - RETRY_STATUS_CODES, - null - ); - - const { response: mappedResponse } = await responseHandler( - lastResponse, - store.isStreamingMode, - store.proxyProvider, - undefined, - urlToFetch, - false, - store.reqBody, - false - ); - updateResponseHeaders( - mappedResponse, - 0, - store.reqBody, - 'DISABLED', - lastAttempt ?? 0, - requestHeaders[HEADER_KEYS.TRACE_ID] ?? '' - ); - - c.set('requestOptions', [ - { - providerOptions: { - ...store.reqBody, - provider: store.proxyProvider, - requestURL: urlToFetch, - rubeusURL: 'proxy', - }, - requestParams: store.reqBody, - response: mappedResponse.clone(), - cacheStatus: 'DISABLED', - cacheKey: undefined, - }, - ]); - - return mappedResponse; - } catch (err: any) { - console.log('proxyGet error', err.message); - return new Response( - JSON.stringify({ - status: 'failure', - message: 'Something went wrong', - }), - { - status: 500, - headers: { - 'content-type': 'application/json', - }, - } - ); - } -} diff --git a/src/index.ts b/src/index.ts index ecc5c672..6e3a2349 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,28 +7,29 @@ import { Hono } from 'hono'; import { prettyJSON } from 'hono/pretty-json'; import { HTTPException } from 'hono/http-exception'; +import { compress } from 'hono/compress'; +import { getRuntimeKey } from 'hono/adapter'; // import { env } from 'hono/adapter' // Have to set this up for multi-environment deployment -import { completeHandler } from './handlers/completeHandler'; -import { chatCompleteHandler } from './handlers/chatCompleteHandler'; -import { embedHandler } from './handlers/embedHandler'; +// Middlewares +import { requestValidator } from './middlewares/requestValidator'; +import { hooks } from './middlewares/hooks'; +import { memoryCache } from './middlewares/cache'; + +// Handlers import { proxyHandler } from './handlers/proxyHandler'; -import { proxyGetHandler } from './handlers/proxyGetHandler'; import { chatCompletionsHandler } from './handlers/chatCompletionsHandler'; import { completionsHandler } from './handlers/completionsHandler'; import { embeddingsHandler } from './handlers/embeddingsHandler'; -import { requestValidator } from './middlewares/requestValidator'; -import { hooks } from './middlewares/hooks'; -import { compress } from 'hono/compress'; -import { getRuntimeKey } from 'hono/adapter'; import { imageGenerationsHandler } from './handlers/imageGenerationsHandler'; -import { memoryCache } from './middlewares/cache'; import { createSpeechHandler } from './handlers/createSpeechHandler'; -import conf from '../conf.json'; import { createTranscriptionHandler } from './handlers/createTranscriptionHandler'; import { createTranslationHandler } from './handlers/createTranslationHandler'; import { modelsHandler, providersHandler } from './handlers/modelsHandler'; +// Config +import conf from '../conf.json'; + // Create a new Hono server instance const app = new Hono(); @@ -82,27 +83,6 @@ app.onError((err, c) => { return c.json({ status: 'failure', message: err.message }); }); -/** - * @deprecated - * POST route for '/v1/complete'. - * Handles requests by passing them to the completeHandler. - */ -app.post('/v1/complete', completeHandler); - -/** - * @deprecated - * POST route for '/v1/chatComplete'. - * Handles requests by passing them to the chatCompleteHandler. - */ -app.post('/v1/chatComplete', chatCompleteHandler); - -/** - * @deprecated - * POST route for '/v1/embed'. - * Handles requests by passing them to the embedHandler. - */ -app.post('/v1/embed', embedHandler); - /** * POST route for '/v1/chat/completions'. * Handles requests by passing them to the chatCompletionsHandler. @@ -179,9 +159,9 @@ app.post('/v1/proxy/*', proxyHandler); app.post('/v1/*', requestValidator, proxyHandler); // Support the /v1 proxy endpoint after all defined endpoints so this does not interfere. -app.get('/v1/*', requestValidator, proxyGetHandler); +app.get('/v1/*', requestValidator, proxyHandler); -app.delete('/v1/*', requestValidator, proxyGetHandler); +app.delete('/v1/*', requestValidator, proxyHandler); // Export the app export default app; From afecf6bbbfc2a3aec199c97d7ad1bc578db8aabb Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Thu, 21 Nov 2024 19:39:52 +0530 Subject: [PATCH 05/35] Removed deprecated functions --- src/handlers/handlerUtils.ts | 373 ----------------------------------- src/handlers/proxyHandler.ts | 56 ------ 2 files changed, 429 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index e947479b..036c4c8c 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -21,10 +21,8 @@ import Providers from '../providers'; import { ProviderAPIConfig, endpointStrings } from '../providers/types'; import transformToProviderRequest from '../services/transformToProviderRequest'; import { - Config, Options, Params, - ShortConfig, StrategyModes, Targets, } from '../types/requestBody'; @@ -171,318 +169,6 @@ export function selectProviderByWeight(providers: Options[]): Options { throw new Error('No provider selected, please check the weights'); } -/** - * @deprecated - * Gets the provider options based on the specified mode. - * Modes can be "single" (uses the first provider), "loadbalance" (selects one provider based on weights), - * or "fallback" (uses all providers in the given order). If the mode does not match these options, null is returned. - * - * @param {string} mode - The mode for selecting providers. - * @param {any} config - The configuration for the providers. - * @returns {(Options[]|null)} - The selected provider options. - */ -export function getProviderOptionsByMode( - mode: string, - config: any -): Options[] | null { - if (config.targets) { - config.options = config.targets; - } - - if (config.options) { - // Inherit cache and retry from top level if not present on option level - config.options.forEach((configOption: any) => { - if (config.cache && !configOption.cache) { - configOption.cache = config.cache; - } - if (config.retry && !configOption.retry) { - configOption.retry = config.retry; - } - }); - } - - switch (mode) { - case 'single': - return [config.options[0]]; - case 'loadbalance': - return [selectProviderByWeight(config.options)]; - case 'fallback': - return config.options; - default: - return null; - } -} - -/** - * @deprecated - */ -export const fetchProviderOptionsFromConfig = ( - config: Config | ShortConfig -): Options[] | null => { - let providerOptions: Options[] | null = null; - let mode: string; - const camelCaseConfig = convertKeysToCamelCase(config, [ - 'override_params', - 'params', - 'metadata', - ]) as Config | ShortConfig; - - if ('provider' in camelCaseConfig) { - providerOptions = [ - { - provider: camelCaseConfig.provider, - virtualKey: camelCaseConfig.virtualKey, - apiKey: camelCaseConfig.apiKey, - cache: camelCaseConfig.cache, - retry: camelCaseConfig.retry, - customHost: camelCaseConfig.customHost, - }, - ]; - if (camelCaseConfig.resourceName) - providerOptions[0].resourceName = camelCaseConfig.resourceName; - if (camelCaseConfig.deploymentId) - providerOptions[0].deploymentId = camelCaseConfig.deploymentId; - if (camelCaseConfig.apiVersion) - providerOptions[0].apiVersion = camelCaseConfig.apiVersion; - if (camelCaseConfig.azureModelName) - providerOptions[0].azureModelName = camelCaseConfig.azureModelName; - if (camelCaseConfig.apiVersion) - providerOptions[0].vertexProjectId = camelCaseConfig.vertexProjectId; - if (camelCaseConfig.apiVersion) - providerOptions[0].vertexRegion = camelCaseConfig.vertexRegion; - if (camelCaseConfig.workersAiAccountId) - providerOptions[0].workersAiAccountId = - camelCaseConfig.workersAiAccountId; - mode = 'single'; - } else { - if (camelCaseConfig.strategy && camelCaseConfig.strategy.mode) { - mode = camelCaseConfig.strategy.mode; - } else { - mode = camelCaseConfig.mode; - } - providerOptions = getProviderOptionsByMode(mode, camelCaseConfig); - } - return providerOptions; -}; - -/** - * @deprecated - * Makes a request (GET or POST) to a provider and returns the response. - * The request is constructed using the provider, apiKey, and requestBody parameters. - * The fn parameter is the type of request being made (e.g., "complete", "chatComplete"). - * - * @param {Options} providerOption - The provider options. This object follows the Options interface and may contain a RetrySettings object for retry configuration. - * @param {RequestBody} requestBody - The request body. - * @param {string} fn - The function for the request. - * @param {string} method - The method for the request (GET, POST). - * @returns {Promise} - The response from the request. - * @throws Will throw an error if the response is not ok or if all retry attempts fail. - */ -export async function tryPostProxy( - c: Context, - providerOption: Options, - inputParams: Params, - requestHeaders: Record, - fn: endpointStrings, - currentIndex: number, - method: string = 'POST' -): Promise { - const overrideParams = providerOption?.overrideParams || {}; - const params: Params = { ...inputParams, ...overrideParams }; - const isStreamingMode = params.stream ? true : false; - - const provider: string = providerOption.provider ?? ''; - - // Mapping providers to corresponding URLs - const apiConfig: ProviderAPIConfig = Providers[provider].api; - - const forwardHeaders: string[] = []; - const customHost = - requestHeaders[HEADER_KEYS.CUSTOM_HOST] || providerOption.customHost || ''; - const baseUrl = - customHost || apiConfig.getBaseURL({ providerOptions: providerOption }); - const endpoint = apiConfig.getEndpoint({ - providerOptions: providerOption, - fn, - gatewayRequestBody: params, - }); - - const url = endpoint - ? `${baseUrl}${endpoint}` - : (providerOption.urlToFetch as string); - - const headers = await apiConfig.headers({ - c, - providerOptions: providerOption, - fn, - transformedRequestBody: params, - transformedRequestUrl: url, - }); - - const fetchOptions = constructRequest( - headers, - provider, - method, - forwardHeaders, - requestHeaders, - fn - ); - - if (method === 'POST') { - fetchOptions.body = JSON.stringify(params); - } - - let response: Response; - let retryCount: number | undefined; - - if (providerOption.retry && typeof providerOption.retry === 'object') { - providerOption.retry = { - attempts: providerOption.retry?.attempts ?? 0, - onStatusCodes: providerOption.retry?.onStatusCodes ?? RETRY_STATUS_CODES, - }; - } else if (typeof providerOption.retry === 'number') { - providerOption.retry = { - attempts: providerOption.retry, - onStatusCodes: RETRY_STATUS_CODES, - }; - } else { - providerOption.retry = { - attempts: 1, - onStatusCodes: [], - }; - } - - const getFromCacheFunction = c.get('getFromCache'); - const cacheIdentifier = c.get('cacheIdentifier'); - const requestOptions = c.get('requestOptions') ?? []; - - let cacheResponse, cacheKey, cacheMode, cacheMaxAge; - let cacheStatus = 'DISABLED'; - - if (requestHeaders[HEADER_KEYS.CACHE]) { - cacheMode = requestHeaders[HEADER_KEYS.CACHE]; - } else if ( - providerOption?.cache && - typeof providerOption.cache === 'object' && - providerOption.cache.mode - ) { - cacheMode = providerOption.cache.mode; - cacheMaxAge = providerOption.cache.maxAge; - } else if ( - providerOption?.cache && - typeof providerOption.cache === 'string' - ) { - cacheMode = providerOption.cache; - } - - if (getFromCacheFunction && cacheMode) { - [cacheResponse, cacheStatus, cacheKey] = await getFromCacheFunction( - env(c), - { ...requestHeaders, ...fetchOptions.headers }, - params, - url, - cacheIdentifier, - cacheMode, - cacheMaxAge - ); - if (cacheResponse) { - ({ response } = await responseHandler( - new Response(cacheResponse, { - headers: { - 'content-type': 'application/json', - }, - }), - false, - provider, - undefined, - url, - false, - params, - false - )); - - c.set('requestOptions', [ - ...requestOptions, - { - providerOptions: { - ...providerOption, - requestURL: url, - rubeusURL: fn, - }, - requestParams: params, - response: response.clone(), - cacheStatus: cacheStatus, - lastUsedOptionIndex: currentIndex, - cacheKey: cacheKey, - cacheMode: cacheMode, - cacheMaxAge: cacheMaxAge, - }, - ]); - updateResponseHeaders( - response, - currentIndex, - params, - cacheStatus, - 0, - requestHeaders[HEADER_KEYS.TRACE_ID] ?? '' - ); - return response; - } - } - - [response, retryCount] = await retryRequest( - url, - fetchOptions, - providerOption.retry.attempts, - providerOption.retry.onStatusCodes, - null - ); - const mappedResponse = await responseHandler( - response, - isStreamingMode, - provider, - undefined, - url, - false, - params, - false - ); - updateResponseHeaders( - mappedResponse.response, - currentIndex, - params, - cacheStatus, - retryCount ?? 0, - requestHeaders[HEADER_KEYS.TRACE_ID] ?? '' - ); - - c.set('requestOptions', [ - ...requestOptions, - { - providerOptions: { - ...providerOption, - requestURL: url, - rubeusURL: fn, - }, - requestParams: params, - response: mappedResponse.response.clone(), - cacheStatus: cacheStatus, - lastUsedOptionIndex: currentIndex, - cacheKey: cacheKey, - cacheMode: cacheMode, - }, - ]); - // If the response was not ok, throw an error - if (!response.ok) { - // Check if this request needs to be retried - const errorObj: any = new Error(await mappedResponse.response.text()); - errorObj.status = mappedResponse.response.status; - throw errorObj; - } - - return mappedResponse.response; -} - /** * Makes a POST request to a provider and returns the response. * The POST request is constructed using the provider, apiKey, and requestBody parameters. @@ -723,65 +409,6 @@ export async function tryPost( return createResponse(mappedResponse, undefined, false, true); } -/** - * @deprecated - * Tries providers in sequence until a successful response is received. - * The providers are attempted in the order they are given in the providers parameter. - * If all providers fail, an error is thrown with the details of the errors from each provider. - * - * @param {Options[]} providers - The providers to try. Each object in the array follows the Options interface and may contain a RetrySettings object for retry configuration. - * @param {RequestBody} request - The request body. - * @param {endpointStrings} fn - The function for the request. - * @param {String} method - The method to be used (GET, POST) for the request. - * @returns {Promise} - The response from the first successful provider. - * @throws Will throw an error if all providers fail. - */ -export async function tryProvidersInSequence( - c: Context, - providers: Options[], - params: Params, - requestHeaders: Record, - fn: endpointStrings, - method: string = 'POST' -): Promise { - let errors: any[] = []; - for (let [index, providerOption] of providers.entries()) { - try { - const loadbalanceIndex = !isNaN(Number(providerOption.index)) - ? Number(providerOption.index) - : null; - if (fn === 'proxy') { - return await tryPostProxy( - c, - providerOption, - params, - requestHeaders, - fn, - loadbalanceIndex ?? index, - method - ); - } - return await tryPost( - c, - providerOption, - params, - requestHeaders, - fn, - loadbalanceIndex ?? index - ); - } catch (error: any) { - // Log and store the error - errors.push({ - provider: providerOption.provider, - errorObj: error.message, - status: error.status, - }); - } - } - // If we're here, all providers failed. Throw an error with the details. - throw new Error(JSON.stringify(errors)); -} - export async function tryTargetsRecursively( c: Context, targetGroup: Targets, diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index 5c188bb2..00671fb9 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -1,69 +1,13 @@ import { Context } from 'hono'; -import { env } from 'hono/adapter'; import { - ANTHROPIC, - AZURE_OPEN_AI, CONTENT_TYPES, - HEADER_KEYS, - MAX_RETRIES, - OLLAMA, POWERED_BY, - RETRY_STATUS_CODES, - TRITON, } from '../globals'; -import Providers from '../providers'; -import { Config, ShortConfig } from '../types/requestBody'; -import { convertKeysToCamelCase, getStreamingMode } from '../utils'; import { constructConfigFromRequestHeaders, - fetchProviderOptionsFromConfig, - tryProvidersInSequence, tryTargetsRecursively, - updateResponseHeaders, } from './handlerUtils'; -import { retryRequest } from './retryHandler'; -import { responseHandler } from './responseHandlers'; import { RouterError } from '../errors/RouterError'; -// Find the proxy provider -function proxyProvider(proxyModeHeader: string, providerHeader: string) { - const proxyProvider = proxyModeHeader?.split(' ')[1] ?? providerHeader; - return proxyProvider; -} - -function getProxyPath( - requestURL: string, - proxyProvider: string, - proxyEndpointPath: string, - customHost: string -) { - let reqURL = new URL(requestURL); - let reqPath = reqURL.pathname; - const reqQuery = reqURL.search; - reqPath = reqPath.replace(proxyEndpointPath, ''); - - if (customHost) { - return `${customHost}${reqPath}${reqQuery}`; - } - - const providerBasePath = Providers[proxyProvider].api.getBaseURL({ - providerOptions: {}, - }); - if (proxyProvider === AZURE_OPEN_AI) { - return `https:/${reqPath}${reqQuery}`; - } - - if (proxyProvider === OLLAMA || proxyProvider === TRITON) { - return `https:/${reqPath}`; - } - let proxyPath = `${providerBasePath}${reqPath}${reqQuery}`; - - // Fix specific for Anthropic SDK calls. Is this needed? - Yes - if (proxyProvider === ANTHROPIC) { - proxyPath = proxyPath.replace('/v1/v1/', '/v1/'); - } - - return proxyPath; -} async function getRequestData(request: Request, contentType: string) { let finalRequest: any; From f2f109cbf6485bafdd81e2d96143c96eb6288fbf Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Thu, 21 Nov 2024 19:56:57 +0530 Subject: [PATCH 06/35] fix --- src/handlers/proxyHandler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index 00671fb9..99109047 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -19,7 +19,7 @@ async function getRequestData(request: Request, contentType: string) { } } else if (contentType == CONTENT_TYPES.MULTIPART_FORM_DATA) { finalRequest = await request.formData(); - } else if (contentType.startsWith(CONTENT_TYPES.GENERIC_AUDIO_PATTERN)) { + } else if (contentType?.startsWith(CONTENT_TYPES.GENERIC_AUDIO_PATTERN)) { finalRequest = await request.arrayBuffer(); } @@ -57,7 +57,7 @@ function headersToSend( export async function proxyHandler(c: Context): Promise { try { let requestHeaders = Object.fromEntries(c.req.raw.headers); - const requestContentType = requestHeaders['content-type'].split(';')[0]; + const requestContentType = requestHeaders['content-type']?.split(';')[0]; const request = await getRequestData(c.req.raw, requestContentType); From 63eb06ded2b690cd654b559b6414ffce6b756e9b Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Thu, 21 Nov 2024 20:20:42 +0530 Subject: [PATCH 07/35] format --- src/handlers/handlerUtils.ts | 15 ++++++--------- src/handlers/proxyHandler.ts | 5 +---- src/services/transformToProviderRequest.ts | 2 +- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 036c4c8c..0ab4a364 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -20,12 +20,7 @@ import { import Providers from '../providers'; import { ProviderAPIConfig, endpointStrings } from '../providers/types'; import transformToProviderRequest from '../services/transformToProviderRequest'; -import { - Options, - Params, - StrategyModes, - Targets, -} from '../types/requestBody'; +import { Options, Params, StrategyModes, Targets } from '../types/requestBody'; import { convertKeysToCamelCase } from '../utils'; import { retryRequest } from './retryHandler'; import { env, getRuntimeKey } from 'hono/adapter'; @@ -251,7 +246,7 @@ export async function tryPost( }); let url: string; - if (fn=="proxy") { + if (fn == 'proxy') { let proxyPath = c.req.url.indexOf('/v1/proxy') > -1 ? '/v1/proxy' : '/v1'; url = getProxyPath(c.req.url, provider, proxyPath, customHost); } else { @@ -278,10 +273,12 @@ export async function tryPost( ); const headerContentType = headers[HEADER_KEYS.CONTENT_TYPE]; - const requestContentType = requestHeaders[HEADER_KEYS.CONTENT_TYPE.toLowerCase()]?.split(';')[0]; + const requestContentType = + requestHeaders[HEADER_KEYS.CONTENT_TYPE.toLowerCase()]?.split(';')[0]; fetchOptions.body = - (headerContentType === CONTENT_TYPES.MULTIPART_FORM_DATA || (fn=="proxy" && requestContentType === CONTENT_TYPES.MULTIPART_FORM_DATA)) + headerContentType === CONTENT_TYPES.MULTIPART_FORM_DATA || + (fn == 'proxy' && requestContentType === CONTENT_TYPES.MULTIPART_FORM_DATA) ? (transformedRequestBody as FormData) : JSON.stringify(transformedRequestBody); diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index 99109047..35e7ec9a 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -1,8 +1,5 @@ import { Context } from 'hono'; -import { - CONTENT_TYPES, - POWERED_BY, -} from '../globals'; +import { CONTENT_TYPES, POWERED_BY } from '../globals'; import { constructConfigFromRequestHeaders, tryTargetsRecursively, diff --git a/src/services/transformToProviderRequest.ts b/src/services/transformToProviderRequest.ts index 894ed84f..218db7fd 100644 --- a/src/services/transformToProviderRequest.ts +++ b/src/services/transformToProviderRequest.ts @@ -189,7 +189,7 @@ export const transformToProviderRequest = ( ) => { if (inputParams instanceof FormData) return inputParams; - if (fn === "proxy") { + if (fn === 'proxy') { return params; } From 24963f0a7828ee64c777baf34ee3078b9cb5d53d Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 21 Nov 2024 21:02:27 +0530 Subject: [PATCH 08/35] fix: delete fetch options body for get and delete requests --- src/handlers/handlerUtils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 0ab4a364..da68ee75 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -282,6 +282,10 @@ export async function tryPost( ? (transformedRequestBody as FormData) : JSON.stringify(transformedRequestBody); + if (['GET', 'DELETE'].includes(method)) { + delete fetchOptions.body; + } + providerOption.retry = { attempts: providerOption.retry?.attempts ?? 0, onStatusCodes: providerOption.retry?.onStatusCodes ?? RETRY_STATUS_CODES, From 98dc073e38f24fd45587f98414f8943d2ca830ea Mon Sep 17 00:00:00 2001 From: visargD Date: Wed, 27 Nov 2024 17:11:12 +0530 Subject: [PATCH 09/35] feat: add getProxyEndpoint method in provider api config --- src/providers/azure-openai/api.ts | 13 +++++++++++++ src/providers/types.ts | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/src/providers/azure-openai/api.ts b/src/providers/azure-openai/api.ts index 19ae9e3e..7f4ef679 100644 --- a/src/providers/azure-openai/api.ts +++ b/src/providers/azure-openai/api.ts @@ -100,6 +100,19 @@ const AzureOpenAIAPIConfig: ProviderAPIConfig = { return ''; } }, + getProxyEndpoint: ({ reqPath, reqQuery, providerOptions }) => { + const { apiVersion } = providerOptions; + if (!reqQuery.includes('api-version')) { + let _reqQuery = reqQuery; + if (!reqQuery) { + _reqQuery = `?api-version=${apiVersion}`; + } else { + _reqQuery += `&api-version=${apiVersion}`; + } + return `${reqPath}${_reqQuery}`; + } + return `${reqPath}${reqQuery}`; + }, }; export default AzureOpenAIAPIConfig; diff --git a/src/providers/types.ts b/src/providers/types.ts index cb9c93cc..56187ded 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -54,6 +54,11 @@ export interface ProviderAPIConfig { }) => string; /** A function to determine if the request body should be transformed to form data */ transformToFormData?: (args: { gatewayRequestBody: Params }) => boolean; + getProxyEndpoint?: (args: { + providerOptions: Options; + reqPath: string; + reqQuery: string; + }) => string; } export type endpointStrings = From 738dfcd4011deec6d18e62dd43470665796bb628 Mon Sep 17 00:00:00 2001 From: visargD Date: Wed, 27 Nov 2024 17:12:10 +0530 Subject: [PATCH 10/35] chore: refactor get proxy path function --- src/handlers/handlerUtils.ts | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index f1e338c3..364541cf 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -95,28 +95,19 @@ function getProxyPath( requestURL: string, proxyProvider: string, proxyEndpointPath: string, - customHost: string + baseURL: string, + providerOptions: Options ) { let reqURL = new URL(requestURL); let reqPath = reqURL.pathname; const reqQuery = reqURL.search; reqPath = reqPath.replace(proxyEndpointPath, ''); - if (customHost) { - return `${customHost}${reqPath}${reqQuery}`; + if (Providers[proxyProvider]?.api?.getProxyEndpoint) { + return `${baseURL}${Providers[proxyProvider].api.getProxyEndpoint({ reqPath, reqQuery, providerOptions })}`; } - const providerBasePath = Providers[proxyProvider].api.getBaseURL({ - providerOptions: {}, - }); - if (proxyProvider === AZURE_OPEN_AI) { - return `https:/${reqPath}${reqQuery}`; - } - - if (proxyProvider === OLLAMA || proxyProvider === TRITON) { - return `https:/${reqPath}`; - } - let proxyPath = `${providerBasePath}${reqPath}${reqQuery}`; + let proxyPath = `${baseURL}${reqPath}${reqQuery}`; // Fix specific for Anthropic SDK calls. Is this needed? - Yes if (proxyProvider === ANTHROPIC) { @@ -249,7 +240,7 @@ export async function tryPost( let url: string; if (fn == 'proxy') { let proxyPath = c.req.url.indexOf('/v1/proxy') > -1 ? '/v1/proxy' : '/v1'; - url = getProxyPath(c.req.url, provider, proxyPath, customHost); + url = getProxyPath(c.req.url, provider, proxyPath, baseUrl, providerOption); } else { url = `${baseUrl}${endpoint}`; } From 3cec080a9bc148e2802c26e91fe19c7e173cb1e5 Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 03:16:04 +0530 Subject: [PATCH 11/35] chore: handle 204 no content response --- src/handlers/responseHandlers.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/handlers/responseHandlers.ts b/src/handlers/responseHandlers.ts index 1c5ef572..9ce45666 100644 --- a/src/handlers/responseHandlers.ts +++ b/src/handlers/responseHandlers.ts @@ -130,6 +130,13 @@ export async function responseHandler( return { response: textResponse, responseJson: null }; } + if (!responseContentType && response.status === 204) { + return { + response: new Response(response.body, response), + responseJson: null, + }; + } + const nonStreamingResponse = await handleNonStreamingMode( response, responseTransformerFunction, From ae19708bc0927d2c6c44e710713c52bbe1ff6b7a Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 18:44:38 +0530 Subject: [PATCH 12/35] chore: revert removal of custom headers to ignore env in proxy handler --- src/handlers/proxyHandler.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index 35e7ec9a..e34f22c4 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -5,6 +5,7 @@ import { tryTargetsRecursively, } from './handlerUtils'; import { RouterError } from '../errors/RouterError'; +import { env } from 'hono/adapter'; async function getRequestData(request: Request, contentType: string) { let finalRequest: any; @@ -64,7 +65,7 @@ export async function proxyHandler(c: Context): Promise { c, camelCaseConfig, request, - headersToSend(requestHeaders, []), + headersToSend(requestHeaders, env(c).CUSTOM_HEADERS_TO_IGNORE ?? []), 'proxy', c.req.method, 'config' From 34b546b85efdfe3432ea83d49c719e15b807a5f0 Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 19:44:02 +0530 Subject: [PATCH 13/35] chore: remove unused imports --- src/services/transformToProviderRequest.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/services/transformToProviderRequest.ts b/src/services/transformToProviderRequest.ts index 218db7fd..6abb3609 100644 --- a/src/services/transformToProviderRequest.ts +++ b/src/services/transformToProviderRequest.ts @@ -1,8 +1,7 @@ import { GatewayError } from '../errors/GatewayError'; -import { MULTIPART_FORM_DATA_ENDPOINTS } from '../globals'; import ProviderConfigs from '../providers'; import { endpointStrings } from '../providers/types'; -import { Options, Params, Targets } from '../types/requestBody'; +import { Params } from '../types/requestBody'; /** * Helper function to set a nested property in an object. From 463f0a49fb4baecbec500e5420cd1cce5027af27 Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 20:39:55 +0530 Subject: [PATCH 14/35] fix: retry count last attempt logic --- src/handlers/handlerUtils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 364541cf..ee7b02ab 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -946,7 +946,12 @@ export async function recursiveAfterRequestHookHandler( ); } - return [arhResponse, retryAttemptsMade]; + let lastAttempt = (retryCount || 0) + retryAttemptsMade; + if (lastAttempt === (retry?.attempts || 0)) { + lastAttempt = -1; // All retry attempts exhausted without success. + } + + return [arhResponse, lastAttempt]; } /** From 96b3cc1079b0c6598abf4635501769ddcbbb746a Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 20:44:36 +0530 Subject: [PATCH 15/35] fix: retry count last attempt logic --- src/handlers/handlerUtils.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index ee7b02ab..4397e708 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -927,10 +927,11 @@ export async function recursiveAfterRequestHookHandler( const remainingRetryCount = (retry?.attempts || 0) - (retryCount || 0) - retryAttemptsMade; - if ( - remainingRetryCount > 0 && - retry?.onStatusCodes?.includes(arhResponse.status) - ) { + const isRetriableStatusCode = retry?.onStatusCodes?.includes( + arhResponse.status + ); + + if (remainingRetryCount > 0 && isRetriableStatusCode) { return recursiveAfterRequestHookHandler( c, url, @@ -947,7 +948,7 @@ export async function recursiveAfterRequestHookHandler( } let lastAttempt = (retryCount || 0) + retryAttemptsMade; - if (lastAttempt === (retry?.attempts || 0)) { + if (lastAttempt === (retry?.attempts || 0) && isRetriableStatusCode) { lastAttempt = -1; // All retry attempts exhausted without success. } From d92614bd57881d6d6865ec0d1d3342d7ddc9e552 Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 20:59:14 +0530 Subject: [PATCH 16/35] chore: remove unused imports --- src/handlers/handlerUtils.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 4397e708..955d3a0d 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -14,8 +14,6 @@ import { CONTENT_TYPES, HUGGING_FACE, STABILITY_AI, - OLLAMA, - TRITON, } from '../globals'; import Providers from '../providers'; import { ProviderAPIConfig, endpointStrings } from '../providers/types'; From 8efd48fca71a5a54e6c230085fa50a5be9edd6e9 Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 21:27:48 +0530 Subject: [PATCH 17/35] chore: keep the deprecated azure proxy url construction logic --- src/handlers/handlerUtils.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 955d3a0d..21cdf5bd 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -101,6 +101,15 @@ function getProxyPath( const reqQuery = reqURL.search; reqPath = reqPath.replace(proxyEndpointPath, ''); + // NOTE: temporary support for the deprecated way of making azure requests + // where the endpoint was sent in request path of the incoming gateway url + if ( + proxyProvider === AZURE_OPEN_AI && + reqPath.includes('.openai.azure.com') + ) { + return `https:/${reqPath}${reqQuery}`; + } + if (Providers[proxyProvider]?.api?.getProxyEndpoint) { return `${baseURL}${Providers[proxyProvider].api.getProxyEndpoint({ reqPath, reqQuery, providerOptions })}`; } From 6062c73ff312cc99446ff181c956375162de1e14 Mon Sep 17 00:00:00 2001 From: visargD Date: Thu, 28 Nov 2024 21:28:48 +0530 Subject: [PATCH 18/35] chore: minor cleanup in azure provider config --- src/providers/azure-openai/api.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/providers/azure-openai/api.ts b/src/providers/azure-openai/api.ts index 7f4ef679..afd2a98f 100644 --- a/src/providers/azure-openai/api.ts +++ b/src/providers/azure-openai/api.ts @@ -102,7 +102,8 @@ const AzureOpenAIAPIConfig: ProviderAPIConfig = { }, getProxyEndpoint: ({ reqPath, reqQuery, providerOptions }) => { const { apiVersion } = providerOptions; - if (!reqQuery.includes('api-version')) { + if (!apiVersion) return `${reqPath}${reqQuery}`; + if (!reqQuery?.includes('api-version')) { let _reqQuery = reqQuery; if (!reqQuery) { _reqQuery = `?api-version=${apiVersion}`; From cfde8c392c554605776c4b7c4fa1475a0642461a Mon Sep 17 00:00:00 2001 From: Narendranath Gogineni Date: Thu, 28 Nov 2024 21:35:16 +0530 Subject: [PATCH 19/35] support sagemaker as provider --- src/globals.ts | 2 + src/handlers/handlerUtils.ts | 45 ++++++++++-- src/providers/index.ts | 2 + src/providers/sagemaker/api.ts | 113 +++++++++++++++++++++++++++++++ src/providers/sagemaker/index.ts | 8 +++ src/types/requestBody.ts | 12 +++- 6 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 src/providers/sagemaker/api.ts create mode 100644 src/providers/sagemaker/index.ts diff --git a/src/globals.ts b/src/globals.ts index 75d55968..c7932f5c 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -75,6 +75,7 @@ export const UPSTAGE: string = 'upstage'; export const LAMBDA: string = 'lambda'; export const DASHSCOPE: string = 'dashscope'; export const X_AI: string = 'x-ai'; +export const SAGEMAKER: string = 'sagemaker'; export const VALID_PROVIDERS = [ ANTHROPIC, @@ -123,6 +124,7 @@ export const VALID_PROVIDERS = [ LAMBDA, DASHSCOPE, X_AI, + SAGEMAKER, ]; export const CONTENT_TYPES = { diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 21cdf5bd..2000d52a 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -14,6 +14,7 @@ import { CONTENT_TYPES, HUGGING_FACE, STABILITY_AI, + SAGEMAKER, } from '../globals'; import Providers from '../providers'; import { ProviderAPIConfig, endpointStrings } from '../providers/types'; @@ -729,7 +730,7 @@ export function constructConfigFromRequestHeaders( azureEndpointName: requestHeaders[`x-${POWERED_BY}-azure-endpoint-name`], }; - const bedrockConfig = { + const awsConfig = { awsAccessKeyId: requestHeaders[`x-${POWERED_BY}-aws-access-key-id`], awsSecretAccessKey: requestHeaders[`x-${POWERED_BY}-aws-secret-access-key`], awsSessionToken: requestHeaders[`x-${POWERED_BY}-aws-session-token`], @@ -739,6 +740,27 @@ export function constructConfigFromRequestHeaders( awsExternalId: requestHeaders[`x-${POWERED_BY}-aws-external-id`], }; + const sagemakerConfig = { + sagemakerCustomAttributes: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-custom-attributes`], + sagemakerTargetModel: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-target-model`], + sagemakerTargetVariant: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-target-variant`], + sagemakerTargetContainerHostname: + requestHeaders[ + `x-${POWERED_BY}-amzn-sagemaker-target-container-hostname` + ], + sagemakerInferenceId: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-inference-id`], + sagemakerEnableExplanations: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-enable-explanations`], + sagemakerInferenceComponent: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-inference-component`], + sagemakerSessionId: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-session-id`], + }; + const workersAiConfig = { workersAiAccountId: requestHeaders[`x-${POWERED_BY}-workers-ai-account-id`], }; @@ -794,10 +816,20 @@ export function constructConfigFromRequestHeaders( }; } - if (parsedConfigJson.provider === BEDROCK) { + if ( + parsedConfigJson.provider === BEDROCK || + parsedConfigJson.provider === SAGEMAKER + ) { + parsedConfigJson = { + ...parsedConfigJson, + ...awsConfig, + }; + } + + if (parsedConfigJson.provider === SAGEMAKER) { parsedConfigJson = { ...parsedConfigJson, - ...bedrockConfig, + ...sagemakerConfig, }; } @@ -862,8 +894,11 @@ export function constructConfigFromRequestHeaders( apiKey: requestHeaders['authorization']?.replace('Bearer ', ''), ...(requestHeaders[`x-${POWERED_BY}-provider`] === AZURE_OPEN_AI && azureConfig), - ...(requestHeaders[`x-${POWERED_BY}-provider`] === BEDROCK && - bedrockConfig), + ...([BEDROCK, SAGEMAKER].includes( + requestHeaders[`x-${POWERED_BY}-provider`] + ) && awsConfig), + ...(requestHeaders[`x-${POWERED_BY}-provider`] === SAGEMAKER && + sagemakerConfig), ...(requestHeaders[`x-${POWERED_BY}-provider`] === WORKERS_AI && workersAiConfig), ...(requestHeaders[`x-${POWERED_BY}-provider`] === GOOGLE_VERTEX_AI && diff --git a/src/providers/index.ts b/src/providers/index.ts index b6e7f7e0..9c70f20b 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -49,6 +49,7 @@ import { LambdaProviderConfig } from './lambda'; import { DashScopeConfig } from './dashscope'; import XAIConfig from './x-ai'; import QdrantConfig from './qdrant'; +import SagemakerConfig from './sagemaker'; const Providers: { [key: string]: ProviderConfigs } = { openai: OpenAIConfig, @@ -98,6 +99,7 @@ const Providers: { [key: string]: ProviderConfigs } = { dashscope: DashScopeConfig, 'x-ai': XAIConfig, qdrant: QdrantConfig, + sagemaker: SagemakerConfig, }; export default Providers; diff --git a/src/providers/sagemaker/api.ts b/src/providers/sagemaker/api.ts new file mode 100644 index 00000000..e8387b25 --- /dev/null +++ b/src/providers/sagemaker/api.ts @@ -0,0 +1,113 @@ +import { GatewayError } from '../../errors/GatewayError'; +import { + generateAWSHeaders, + getAssumedRoleCredentials, +} from '../bedrock/utils'; +import { ProviderAPIConfig } from '../types'; +import { env } from 'hono/adapter'; +const SagemakerAPIConfig: ProviderAPIConfig = { + getBaseURL: ({ providerOptions }) => { + return `https://runtime.sagemaker.${providerOptions.awsRegion}.amazonaws.com`; + }, + headers: async ({ + providerOptions, + transformedRequestBody, + transformedRequestUrl, + c, + }) => { + const headers: Record = { + 'content-type': 'application/json', + }; + + if (providerOptions.awsAuthType === 'assumedRole') { + try { + // Assume the role in the source account + const sourceRoleCredentials = await getAssumedRoleCredentials( + c, + env(c).AWS_ASSUME_ROLE_SOURCE_ARN, // Role ARN in the source account + env(c).AWS_ASSUME_ROLE_SOURCE_EXTERNAL_ID || '', // External ID for source role (if needed) + providerOptions.awsRegion || '' + ); + + if (!sourceRoleCredentials) { + throw new Error('Server Error while assuming internal role'); + } + + // Assume role in destination account using temporary creds obtained in first step + const { accessKeyId, secretAccessKey, sessionToken } = + (await getAssumedRoleCredentials( + c, + providerOptions.awsRoleArn || '', + providerOptions.awsExternalId || '', + providerOptions.awsRegion || '', + { + accessKeyId: sourceRoleCredentials.accessKeyId, + secretAccessKey: sourceRoleCredentials.secretAccessKey, + sessionToken: sourceRoleCredentials.sessionToken, + } + )) || {}; + providerOptions.awsAccessKeyId = accessKeyId; + providerOptions.awsSecretAccessKey = secretAccessKey; + providerOptions.awsSessionToken = sessionToken; + } catch (e) { + throw new GatewayError('Error while assuming sagemaker role'); + } + } + + const awsHeaders = await generateAWSHeaders( + transformedRequestBody, + headers, + transformedRequestUrl, + 'POST', + 'sagemaker', + providerOptions.awsRegion || '', + providerOptions.awsAccessKeyId || '', + providerOptions.awsSecretAccessKey || '', + providerOptions.awsSessionToken || '' + ); + + if (providerOptions.sagemakerCustomAttributes) { + awsHeaders['x-amzn-sagemaker-custom-attributes'] = + providerOptions.sagemakerCustomAttributes; + } + + if (providerOptions.sagemakerTargetModel) { + awsHeaders['x-amzn-sagemaker-target-model'] = + providerOptions.sagemakerTargetModel; + } + + if (providerOptions.sagemakerTargetVariant) { + awsHeaders['x-amzn-sagemaker-target-variant'] = + providerOptions.sagemakerTargetVariant; + } + + if (providerOptions.sagemakerTargetContainerHostname) { + awsHeaders['x-amzn-sagemaker-target-container-hostname'] = + providerOptions.sagemakerTargetContainerHostname; + } + + if (providerOptions.sagemakerInferenceId) { + awsHeaders['x-amzn-sagemaker-inference-id'] = + providerOptions.sagemakerInferenceId; + } + + if (providerOptions.sagemakerEnableExplanations) { + awsHeaders['x-amzn-sagemaker-enable-explanations'] = + providerOptions.sagemakerEnableExplanations; + } + + if (providerOptions.sagemakerInferenceComponent) { + awsHeaders['x-amzn-sagemaker-inference-component'] = + providerOptions.sagemakerInferenceComponent; + } + + if (providerOptions.sagemakerSessionId) { + awsHeaders['x-amzn-sagemaker-session-id'] = + providerOptions.sagemakerSessionId; + } + return awsHeaders; + }, + getEndpoint: ({ gatewayRequestURL }) => gatewayRequestURL.split('/v1')[1], +}; + +export default SagemakerAPIConfig; diff --git a/src/providers/sagemaker/index.ts b/src/providers/sagemaker/index.ts new file mode 100644 index 00000000..5d8dea49 --- /dev/null +++ b/src/providers/sagemaker/index.ts @@ -0,0 +1,8 @@ +import { ProviderConfigs } from '../types'; +import SagemakerAPIConfig from './api'; + +const SagemakerConfig: ProviderConfigs = { + api: SagemakerAPIConfig, +}; + +export default SagemakerConfig; diff --git a/src/types/requestBody.ts b/src/types/requestBody.ts index 360bb5c7..cedf93de 100644 --- a/src/types/requestBody.ts +++ b/src/types/requestBody.ts @@ -78,7 +78,7 @@ export interface Options { requestTimeout?: number; /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ transformToFormData?: boolean; - /** AWS Bedrock specific */ + /** AWS specific (used for Bedrock and Sagemaker) */ awsSecretAccessKey?: string; awsAccessKeyId?: string; awsSessionToken?: string; @@ -87,6 +87,16 @@ export interface Options { awsRoleArn?: string; awsExternalId?: string; + /** Sagemaker specific */ + sagemakerCustomAttributes?: string; + sagemakerTargetModel?: string; + sagemakerTargetVariant?: string; + sagemakerTargetContainerHostname?: string; + sagemakerInferenceId?: string; + sagemakerEnableExplanations?: string; + sagemakerInferenceComponent?: string; + sagemakerSessionId?: string; + /** Stability AI specific */ stabilityClientId?: string; stabilityClientUserId?: string; From 6ed752c6793ecfe25169538489bd6b10650f31c6 Mon Sep 17 00:00:00 2001 From: Narendranath Gogineni Date: Thu, 28 Nov 2024 22:43:06 +0530 Subject: [PATCH 20/35] rename config keys --- src/handlers/handlerUtils.ts | 16 ++++++++-------- src/providers/sagemaker/api.ts | 32 ++++++++++++++++---------------- src/types/requestBody.ts | 16 ++++++++-------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 2000d52a..52f313aa 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -741,23 +741,23 @@ export function constructConfigFromRequestHeaders( }; const sagemakerConfig = { - sagemakerCustomAttributes: + amznSagemakerCustomAttributes: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-custom-attributes`], - sagemakerTargetModel: + amznSagemakerTargetModel: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-target-model`], - sagemakerTargetVariant: + amznSagemakerTargetVariant: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-target-variant`], - sagemakerTargetContainerHostname: + amznSagemakerTargetContainerHostname: requestHeaders[ `x-${POWERED_BY}-amzn-sagemaker-target-container-hostname` ], - sagemakerInferenceId: + amznSagemakerInferenceId: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-inference-id`], - sagemakerEnableExplanations: + amznSagemakerEnableExplanations: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-enable-explanations`], - sagemakerInferenceComponent: + amznSagemakerInferenceComponent: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-inference-component`], - sagemakerSessionId: + amznSagemakerSessionId: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-session-id`], }; diff --git a/src/providers/sagemaker/api.ts b/src/providers/sagemaker/api.ts index e8387b25..1888678c 100644 --- a/src/providers/sagemaker/api.ts +++ b/src/providers/sagemaker/api.ts @@ -66,44 +66,44 @@ const SagemakerAPIConfig: ProviderAPIConfig = { providerOptions.awsSessionToken || '' ); - if (providerOptions.sagemakerCustomAttributes) { + if (providerOptions.amznSagemakerCustomAttributes) { awsHeaders['x-amzn-sagemaker-custom-attributes'] = - providerOptions.sagemakerCustomAttributes; + providerOptions.amznSagemakerCustomAttributes; } - if (providerOptions.sagemakerTargetModel) { + if (providerOptions.amznSagemakerTargetModel) { awsHeaders['x-amzn-sagemaker-target-model'] = - providerOptions.sagemakerTargetModel; + providerOptions.amznSagemakerTargetModel; } - if (providerOptions.sagemakerTargetVariant) { + if (providerOptions.amznSagemakerTargetVariant) { awsHeaders['x-amzn-sagemaker-target-variant'] = - providerOptions.sagemakerTargetVariant; + providerOptions.amznSagemakerTargetVariant; } - if (providerOptions.sagemakerTargetContainerHostname) { + if (providerOptions.amznSagemakerTargetContainerHostname) { awsHeaders['x-amzn-sagemaker-target-container-hostname'] = - providerOptions.sagemakerTargetContainerHostname; + providerOptions.amznSagemakerTargetContainerHostname; } - if (providerOptions.sagemakerInferenceId) { + if (providerOptions.amznSagemakerInferenceId) { awsHeaders['x-amzn-sagemaker-inference-id'] = - providerOptions.sagemakerInferenceId; + providerOptions.amznSagemakerInferenceId; } - if (providerOptions.sagemakerEnableExplanations) { + if (providerOptions.amznSagemakerEnableExplanations) { awsHeaders['x-amzn-sagemaker-enable-explanations'] = - providerOptions.sagemakerEnableExplanations; + providerOptions.amznSagemakerEnableExplanations; } - if (providerOptions.sagemakerInferenceComponent) { + if (providerOptions.amznSagemakerInferenceComponent) { awsHeaders['x-amzn-sagemaker-inference-component'] = - providerOptions.sagemakerInferenceComponent; + providerOptions.amznSagemakerInferenceComponent; } - if (providerOptions.sagemakerSessionId) { + if (providerOptions.amznSagemakerSessionId) { awsHeaders['x-amzn-sagemaker-session-id'] = - providerOptions.sagemakerSessionId; + providerOptions.amznSagemakerSessionId; } return awsHeaders; }, diff --git a/src/types/requestBody.ts b/src/types/requestBody.ts index cedf93de..4787cc7d 100644 --- a/src/types/requestBody.ts +++ b/src/types/requestBody.ts @@ -88,14 +88,14 @@ export interface Options { awsExternalId?: string; /** Sagemaker specific */ - sagemakerCustomAttributes?: string; - sagemakerTargetModel?: string; - sagemakerTargetVariant?: string; - sagemakerTargetContainerHostname?: string; - sagemakerInferenceId?: string; - sagemakerEnableExplanations?: string; - sagemakerInferenceComponent?: string; - sagemakerSessionId?: string; + amznSagemakerCustomAttributes?: string; + amznSagemakerTargetModel?: string; + amznSagemakerTargetVariant?: string; + amznSagemakerTargetContainerHostname?: string; + amznSagemakerInferenceId?: string; + amznSagemakerEnableExplanations?: string; + amznSagemakerInferenceComponent?: string; + amznSagemakerSessionId?: string; /** Stability AI specific */ stabilityClientId?: string; From dab68eaebb634603fbf4b919032f8fdecc08a3c8 Mon Sep 17 00:00:00 2001 From: vrushankportkey <134934501+vrushankportkey@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:39:59 +0530 Subject: [PATCH 21/35] Link to AI Engineering Hours on README --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 043c4dbd..3caccd29 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,6 @@ You can do a lot more stuff with configs in your AI gateway. [Jump to examples
- ### Enterprise Version (Private deployments) @@ -155,6 +154,16 @@ The enterprise deployment architecture for supported platforms is available here
+
+ +### AI Engineering Hours + +Join weekly community calls every Friday (8 AM PT) to kickstart your AI Gateway implementation! [Calendar Link](https://lu.ma/portkey?tag=ai%20engineer) + +Meetings of Minutes [published here](https://portkey.ai/docs/changelog/office-hour). + +
+ ## Core Features ### Reliable Routing - **Fallbacks**: Fallback to another provider or model on failed requests using the LLM gateway. You can specify the errors on which to trigger the fallback. Improves reliability of your application. @@ -269,6 +278,16 @@ The easiest way to contribute is to pick an issue with the `good first issue` ta Bug Report? [File here](https://github.com/Portkey-AI/gateway/issues) | Feature Request? [File here](https://github.com/Portkey-AI/gateway/issues) + +### Getting Started with the Community +Join our weekly AI Engineering Hours every Friday (8 AM PT) to: +- Meet other contributors and community members +- Learn advanced Gateway features and implementation patterns +- Share your experiences and get help +- Stay updated with the latest development priorities + +[Join the next session →](https://lu.ma/portkey?tag=ai%20engineer) +
## Community From c60643275df0242866f9bcf444c2320a91dc4a9b Mon Sep 17 00:00:00 2001 From: vrushankportkey <134934501+vrushankportkey@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:45:03 +0530 Subject: [PATCH 22/35] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3caccd29..2191719b 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,7 @@ Join our weekly AI Engineering Hours every Friday (8 AM PT) to: - Share your experiences and get help - Stay updated with the latest development priorities -[Join the next session →](https://lu.ma/portkey?tag=ai%20engineer) +[Join the next session →](https://lu.ma/portkey?tag=ai%20engineer) | [Meeting notes](https://portkey.ai/docs/changelog/office-hour)
From a7bfff20621807117b134ee1f714ca3d66559c54 Mon Sep 17 00:00:00 2001 From: visargD Date: Fri, 29 Nov 2024 14:38:59 +0530 Subject: [PATCH 23/35] 1.8.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3812ccb7..5cfd6c08 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@portkey-ai/gateway", - "version": "1.8.1", + "version": "1.8.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@portkey-ai/gateway", - "version": "1.8.1", + "version": "1.8.2", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", diff --git a/package.json b/package.json index f8e931d3..ccb50ccd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@portkey-ai/gateway", - "version": "1.8.1", + "version": "1.8.2", "description": "A fast AI gateway by Portkey", "repository": { "type": "git", From 3362553354173e9b38b19a8c6c438666d87758f1 Mon Sep 17 00:00:00 2001 From: vrushankportkey Date: Fri, 29 Nov 2024 17:27:45 +0530 Subject: [PATCH 24/35] Change to all dub.co links --- README.md | 170 +++++++++++++++++++++++++++--------------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 2191719b..6321a67c 100644 --- a/README.md +++ b/README.md @@ -10,19 +10,19 @@ Portkey AI Gateway Demo showing LLM routing capabilities -[Docs](https://portkey.ai/docs) | [Enterprise](https://portkey.ai/docs/product/enterprise-offering) | [Hosted Gateway](https://app.portkey.ai/) | [Changelog](https://new.portkey.ai) | [API Reference](https://portkey.ai/docs/api-reference/inference-api/introduction) +[Docs](https://portkey.wiki/gh-1) | [Enterprise](https://portkey.wiki/gh-2) | [Hosted Gateway](https://portkey.wiki/gh-3) | [Changelog](https://portkey.wiki/gh-4) | [API Reference](https://portkey.wiki/gh-5) [![License](https://img.shields.io/github/license/Ileriayo/markdown-badges)](./LICENSE) -[![Discord](https://img.shields.io/discord/1143393887742861333)](https://portkey.ai/community) -[![Twitter](https://img.shields.io/twitter/url/https/twitter/follow/portkeyai?style=social&label=Follow%20%40PortkeyAI)](https://x.com/portkeyai) -[![npm version](https://badge.fury.io/js/%40portkey-ai%2Fgateway.svg)](https://www.npmjs.com/package/@portkey-ai/gateway) -[![Better Stack Badge](https://uptime.betterstack.com/status-badges/v1/monitor/q94g.svg)](https://status.portkey.ai/?utm_source=status_badge) +[![Discord](https://img.shields.io/discord/1143393887742861333)](https://portkey.wiki/gh-6) +[![Twitter](https://img.shields.io/twitter/url/https/twitter/follow/portkeyai?style=social&label=Follow%20%40PortkeyAI)](https://portkey.wiki/gh-7) +[![npm version](https://badge.fury.io/js/%40portkey-ai%2Fgateway.svg)](https://portkey.wiki/gh-8) +[![Better Stack Badge](https://uptime.betterstack.com/status-badges/v1/monitor/q94g.svg)](https://portkey.wiki/gh-9)
-The [**AI Gateway**](https://portkey.ai/features/ai-gateway) is designed for fast, reliable & secure routing to 1600+ language, vision, audio, and image models. It is a lightweight, open-source, and enterprise-ready solution that allows you to integrate with any language model in under 2 minutes. +The [**AI Gateway**](https://portkey.wiki/gh-10) is designed for fast, reliable & secure routing to 1600+ language, vision, audio, and image models. It is a lightweight, open-source, and enterprise-ready solution that allows you to integrate with any language model in under 2 minutes. - [x] **Blazing fast** (<1ms latency) with a tiny footprint (122kb) - [x] **Battle tested**, with over 10B tokens processed everyday @@ -32,17 +32,17 @@ The [**AI Gateway**](https://portkey.ai/features/ai-gateway) is designed for fas #### What can you do with the AI Gateway? - Integrate with any LLM in under 2 minutes - [Quickstart](#quickstart-2-mins) -- Prevent downtimes through **[automatic retries](https://portkey.ai/docs/product/ai-gateway/automatic-retries)** and **[fallbacks](https://portkey.ai/docs/product/ai-gateway/fallbacks)** -- Scale AI apps with **[load balancing](https://portkey.ai/docs/product/ai-gateway/load-balancing)** and **[conditional routing](https://portkey.ai/docs/product/ai-gateway/conditional-routing)** -- Protect your AI deployments with **[guardrails](https://portkey.ai/docs/product/guardrails)** -- Go beyond text with **[multi-modal capabilities](https://portkey.ai/docs/product/ai-gateway/multimodal-capabilities)** -- Finally, explore **[agentic workflow](https://portkey.ai/docs/integrations/agents)** integrations +- Prevent downtimes through **[automatic retries](https://portkey.wiki/gh-11)** and **[fallbacks](https://portkey.wiki/gh-12)** +- Scale AI apps with **[load balancing](https://portkey.wiki/gh-13)** and **[conditional routing](https://portkey.wiki/gh-14)** +- Protect your AI deployments with **[guardrails](https://portkey.wiki/gh-15)** +- Go beyond text with **[multi-modal capabilities](https://portkey.wiki/gh-16)** +- Finally, explore **[agentic workflow](https://portkey.wiki/gh-17)** integrations

> [!TIP] > Starring this repo helps more developers discover the AI Gateway 🙏🏻 -> +> > ![star-2](https://github.com/user-attachments/assets/53597dce-6333-4ecc-a154-eb05532954e4)
@@ -58,7 +58,7 @@ npx @portkey-ai/gateway Deployment guides: -  Portkey Cloud (Recommended) +  Portkey Cloud (Recommended)   Docker   Node.js   Cloudflare @@ -91,16 +91,16 @@ client.chat.completions.create( -Supported Libraries: -  [ JS](https://www.npmjs.com/package/portkey-ai) -  [ Python](https://github.com/Portkey-AI/portkey-python-sdk) -  [ REST](https://portkey.ai/docs/api-reference/inference-api/introduction) -  [ OpenAI SDKs](https://portkey.ai/docs/guides/getting-started/getting-started-with-ai-gateway#openai-chat-completion) -  [ Langchain](https://portkey.ai/docs/integrations/libraries/langchain-python) -  [LlamaIndex](https://portkey.ai/docs/integrations/libraries/llama-index-python) -  [Autogen](https://portkey.ai/docs/integrations/agents/autogen) -  [CrewAI](https://portkey.ai/docs/integrations/agents/crewai) -  [More..](https://portkey.ai/docs/integrations/libraries) +Supported Libraries: +  [ JS](https://portkey.wiki/gh-19) +  [ Python](https://portkey.wiki/gh-20) +  [ REST](https://portkey.sh/gh-84) +  [ OpenAI SDKs](https://portkey.wiki/gh-21) +  [ Langchain](https://portkey.wiki/gh-22) +  [LlamaIndex](https://portkey.wiki/gh-23) +  [Autogen](https://portkey.wiki/gh-24) +  [CrewAI](https://portkey.wiki/gh-25) +  [More..](https://portkey.wiki/gh-26) ### 3. Routing & Guardrails @@ -129,7 +129,7 @@ client.chat.completions.create( Request flow through Portkey's AI gateway with retries and guardrails -You can do a lot more stuff with configs in your AI gateway. [Jump to examples →](https://portkey.ai/docs/product/ai-gateway/configs) +You can do a lot more stuff with configs in your AI gateway. [Jump to examples →](https://portkey.wiki/gh-27)
@@ -137,17 +137,17 @@ You can do a lot more stuff with configs in your AI gateway. [Jump to examples -[ AWS](https://portkey.ai/docs/product/enterprise-offering/private-cloud-deployments/aws) -  [ Azure](https://portkey.ai/docs/product/enterprise-offering/private-cloud-deployments/azure) -  [ GCP](https://portkey.ai/docs/product/enterprise-offering/private-cloud-deployments/gcp) -  [ OpenShift](https://github.com/Portkey-AI/helm-chart) -  [ Kubernetes](https://github.com/Portkey-AI/helm-chart) +[ AWS](https://portkey.wiki/gh-28) +  [ Azure](https://portkey.wiki/gh-29) +  [ GCP](https://portkey.wiki/gh-30) +  [ OpenShift](https://portkey.wiki/gh-31) +  [ Kubernetes](https://portkey.wiki/gh-85) -The LLM Gateway's [enterprise version](https://portkey.ai/docs/product/enterprise-offering) offers advanced capabilities for **org management**, **governance**, **security** and [more](https://portkey.ai/docs/product/enterprise-offering) out of the box. [View Feature Comparison →](https://portkey.ai/docs/product/product-feature-comparison) +The LLM Gateway's [enterprise version](https://portkey.wiki/gh-86) offers advanced capabilities for **org management**, **governance**, **security** and [more](https://portkey.wiki/gh-87) out of the box. [View Feature Comparison →](https://portkey.wiki/gh-32) -The enterprise deployment architecture for supported platforms is available here - [**Enterprise Private Cloud Deployments**](https://portkey.ai/docs/product/enterprise-offering/private-cloud-deployments) +The enterprise deployment architecture for supported platforms is available here - [**Enterprise Private Cloud Deployments**](https://portkey.wiki/gh-33) Book an enterprise AI gateway demo
@@ -158,35 +158,35 @@ The enterprise deployment architecture for supported platforms is available here ### AI Engineering Hours -Join weekly community calls every Friday (8 AM PT) to kickstart your AI Gateway implementation! [Calendar Link](https://lu.ma/portkey?tag=ai%20engineer) +Join weekly community calls every Friday (8 AM PT) to kickstart your AI Gateway implementation! [Calendar Link](https://portkey.wiki/gh-35) -Meetings of Minutes [published here](https://portkey.ai/docs/changelog/office-hour). +Meetings of Minutes [published here](https://portkey.wiki/gh-36).
## Core Features ### Reliable Routing -- **Fallbacks**: Fallback to another provider or model on failed requests using the LLM gateway. You can specify the errors on which to trigger the fallback. Improves reliability of your application. -- **Automatic Retries**: Automatically retry failed requests up to 5 times. An exponential backoff strategy spaces out retry attempts to prevent network overload. -- **Load Balancing**: Distribute LLM requests across multiple API keys or AI providers with weights to ensure high availability and optimal performance. -- **Request Timeouts**: Manage unruly LLMs & latencies by setting up granular request timeouts, allowing automatic termination of requests that exceed a specified duration. -- **Multi-modal LLM Gateway**: Call vision, audio (text-to-speech & speech-to-text), and image generation models from multiple providers — all using the familiar OpenAI signature -- **Realtime APIs**: Call realtime APIs launched by OpenAI through the integrate websockets server. +- **Fallbacks**: Fallback to another provider or model on failed requests using the LLM gateway. You can specify the errors on which to trigger the fallback. Improves reliability of your application. +- **Automatic Retries**: Automatically retry failed requests up to 5 times. An exponential backoff strategy spaces out retry attempts to prevent network overload. +- **Load Balancing**: Distribute LLM requests across multiple API keys or AI providers with weights to ensure high availability and optimal performance. +- **Request Timeouts**: Manage unruly LLMs & latencies by setting up granular request timeouts, allowing automatic termination of requests that exceed a specified duration. +- **Multi-modal LLM Gateway**: Call vision, audio (text-to-speech & speech-to-text), and image generation models from multiple providers — all using the familiar OpenAI signature +- **Realtime APIs**: Call realtime APIs launched by OpenAI through the integrate websockets server. ### Security & Accuracy -- **Guardrails**: Verify your LLM inputs and outputs to adhere to your specified checks. Choose from the 40+ pre-built guardrails to ensure compliance with security and accuracy standards. You can bring your own guardrails or choose from our many partners. -- [**Secure Key Management***](https://portkey.ai/docs/product/ai-gateway/virtual-keys): Use your own keys or generate virtual keys on the fly. -- [**Role-based access control***](https://portkey.ai/docs/product/enterprise-offering/access-control-management): Granular access control for your users, workspaces and API keys. -- **Compliance & Data Privacy**: The AI gateway is SOC2, HIPAA, GDPR, and CCPA compliant. +- **Guardrails**: Verify your LLM inputs and outputs to adhere to your specified checks. Choose from the 40+ pre-built guardrails to ensure compliance with security and accuracy standards. You can bring your own guardrails or choose from our many partners. +- [**Secure Key Management**](https://portkey.wiki/gh-45): Use your own keys or generate virtual keys on the fly. +- [**Role-based access control**](https://portkey.wiki/gh-46): Granular access control for your users, workspaces and API keys. +- **Compliance & Data Privacy**: The AI gateway is SOC2, HIPAA, GDPR, and CCPA compliant. ### Cost Management -- [**Smart caching**](https://portkey.ai/docs/product/ai-gateway/cache-simple-and-semantic): Cache responses from LLMs to reduce costs and improve latency. Supports simple and semantic* caching. -- [**Usage analytics***](https://portkey.ai/docs/product/observability/analytics): Monitor and analyze your AI and LLM usage, including request volume, latency, costs and error rates. -- [**Provider optimization***](https://portkey.ai/docs/product/ai-gateway/conditional-routing): Automatically switch to the most cost-effective provider based on usage patterns and pricing models. +- [**Smart caching**](https://portkey.wiki/gh-48): Cache responses from LLMs to reduce costs and improve latency. Supports simple and semantic* caching. +- [**Usage analytics**](https://portkey.wiki/gh-49): Monitor and analyze your AI and LLM usage, including request volume, latency, costs and error rates. +- [**Provider optimization***](https://portkey.wiki/gh-89): Automatically switch to the most cost-effective provider based on usage patterns and pricing models. ### Collaboration & Workflows -- **Agents Support**: Seamlessly integrate with popular agent frameworks to build complex AI applications. The gateway seamlessly integrates with [Autogen](https://docs.portkey.ai/docs/welcome/agents/autogen), [CrewAI](https://docs.portkey.ai/docs/welcome/agents/crewai), [LangChain](https://docs.portkey.ai/docs/welcome/agents/langchain-agents), [LlamaIndex](https://docs.portkey.ai/docs/welcome/agents/llama-agents), [Phidata](https://docs.portkey.ai/docs/welcome/agents/phidata), [Control Flow](https://docs.portkey.ai/docs/welcome/agents/control-flow), and even [Custom Agents](https://docs.portkey.ai/docs/welcome/agents/bring-your-own-agents). -- [**Prompt Template Management***](https://portkey.ai/docs/product/prompt-library): Create, manage and version your prompt templates collaboratively through a universal prompt playground. +- **Agents Support**: Seamlessly integrate with popular agent frameworks to build complex AI applications. The gateway seamlessly integrates with [Autogen](https://portkey.wiki/gh-50), [CrewAI](https://portkey.wiki/gh-51), [LangChain](https://portkey.wiki/gh-52), [LlamaIndex](https://portkey.wiki/gh-53), [Phidata](https://portkey.wiki/gh-54), [Control Flow](https://portkey.wiki/gh-55), and even [Custom Agents](https://portkey.wiki/gh-56). +- [**Prompt Template Management***](https://portkey.wiki/gh-57): Create, manage and version your prompt templates collaboratively through a universal prompt playground.

@@ -207,54 +207,54 @@ Meetings of Minutes [published here](https://portkey.ai/docs/changelog/office-ho * [Use the LLM Gateway with Vercel's AI SDK](/cookbook/integrations/vercel-ai.md) * [Monitor Llama Agents with Portkey's LLM Gateway](/cookbook/monitoring-agents/Llama_Agents_with_Telemetry.ipynb) -[View all cookbooks →](https://github.com/Portkey-AI/gateway/tree/main/cookbook) +[View all cookbooks →](https://portkey.wiki/gh-58)

## Supported Providers -Explore Gateway integrations with [45+ providers](https://portkey.ai/docs/integrations/llms) and [8+ agent frameworks](https://portkey.ai/docs/integrations/agents). +Explore Gateway integrations with [45+ providers](https://portkey.wiki/gh-59) and [8+ agent frameworks](https://portkey.wiki/gh-90). | | Provider | Support | Stream | | -------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | ------- | ------ | -| | [OpenAI](https://portkey.ai/docs/welcome/integration-guides/openai) | ✅ | ✅ | -| | [Azure OpenAI](https://portkey.ai/docs/welcome/integration-guides/azure-openai) | ✅ | ✅ | -| | [Anyscale](https://portkey.ai/docs/welcome/integration-guides/anyscale-llama2-mistral-zephyr) | ✅ | ✅ | -| | [Google Gemini & Palm](https://portkey.ai/docs/welcome/integration-guides/gemini) | ✅ | ✅ | -| | [Anthropic](https://portkey.ai/docs/welcome/integration-guides/anthropic) | ✅ | ✅ | -| | [Cohere](https://portkey.ai/docs/welcome/integration-guides/cohere) | ✅ | ✅ | -| | [Together AI](https://portkey.ai/docs/welcome/integration-guides/together-ai) | ✅ | ✅ | -| | [Perplexity](https://portkey.ai/docs/welcome/integration-guides/perplexity-ai) | ✅ | ✅ | -| | [Mistral](https://portkey.ai/docs/welcome/integration-guides/mistral-ai) | ✅ | ✅ | -| | [Nomic](https://portkey.ai/docs/welcome/integration-guides/nomic) | ✅ | ✅ | -| | [AI21](https://portkey.ai/docs/welcome/integration-guides) | ✅ | ✅ | -| | [Stability AI](https://portkey.ai/docs/welcome/integration-guides/stability-ai) | ✅ | ✅ | -| | [DeepInfra](https://portkey.ai/docs/welcome/integration-guides) | ✅ | ✅ | -| | [Ollama](https://portkey.ai/docs/welcome/integration-guides/ollama) | ✅ | ✅ | -| | [Novita AI](https://portkey.ai/docs/integrations/llms/novita-ai) | ✅ | ✅ | `/chat/completions`, `/completions` | - - -> [View the complete list of 200+ supported models here](https://portkey.ai/docs/welcome/what-is-portkey#ai-providers-supported) +| | [OpenAI](https://portkey.wiki/gh-60) | ✅ | ✅ | +| | [Azure OpenAI](https://portkey.wiki/gh-61) | ✅ | ✅ | +| | [Anyscale](https://portkey.wiki/gh-62) | ✅ | ✅ | +| | [Google Gemini](https://portkey.wiki/gh-63) | ✅ | ✅ | +| | [Anthropic](https://portkey.wiki/gh-64) | ✅ | ✅ | +| | [Cohere](https://portkey.wiki/gh-65) | ✅ | ✅ | +| | [Together AI](https://portkey.wiki/gh-66) | ✅ | ✅ | +| | [Perplexity](https://portkey.wiki/gh-67) | ✅ | ✅ | +| | [Mistral](https://portkey.wiki/gh-68) | ✅ | ✅ | +| | [Nomic](https://portkey.wiki/gh-69) | ✅ | ✅ | +| | [AI21](https://portkey.wiki/gh-91) | ✅ | ✅ | +| | [Stability AI](https://portkey.wiki/gh-71) | ✅ | ✅ | +| | [DeepInfra](https://portkey.sh/gh-92) | ✅ | ✅ | +| | [Ollama](https://portkey.wiki/gh-72) | ✅ | ✅ | +| | [Novita AI](https://portkey.wiki/gh-73) | ✅ | ✅ | `/chat/completions`, `/completions` | + + +> [View the complete list of 200+ supported models here](https://portkey.wiki/gh-74)

## Agents -Gateway seamlessly integrates with popular agent frameworks. [Read the documentation here](https://docs.portkey.ai/docs/welcome/agents). +Gateway seamlessly integrates with popular agent frameworks. [Read the documentation here](https://portkey.wiki/gh-75). | Framework | Call 200+ LLMs | Advanced Routing | Caching | Logging & Tracing* | Observability* | Prompt Management* | |------------------------------|--------|-------------|---------|------|---------------|-------------------| -| [Autogen](https://docs.portkey.ai/docs/welcome/agents/autogen) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [CrewAI](https://docs.portkey.ai/docs/welcome/agents/crewai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [LangChain](https://docs.portkey.ai/docs/welcome/agents/langchain-agents) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [Phidata](https://docs.portkey.ai/docs/welcome/agents/phidata) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [Llama Index](https://docs.portkey.ai/docs/welcome/agents/llama-agents) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [Control Flow](https://docs.portkey.ai/docs/welcome/agents/control-flow) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [Build Your Own Agents](https://docs.portkey.ai/docs/welcome/agents/bring-your-own-agents) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [Autogen](https://portkey.wiki/gh-93) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [CrewAI](https://portkey.wiki/gh-94) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [LangChain](https://portkey.wiki/gh-95) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [Phidata](https://portkey.wiki/gh-96) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [Llama Index](https://portkey.wiki/gh-97) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [Control Flow](https://portkey.wiki/gh-98) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [Build Your Own Agents](https://portkey.wiki/gh-99) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-*Available on the [hosted app](https://portkey.ai). For detailed documentation [click here](https://docs.portkey.ai/docs/welcome/agents). +*Available on the [hosted app](https://portkey.wiki/gh-76). For detailed documentation [click here](https://portkey.wiki/gh-100). ## Gateway Enterprise Version @@ -276,7 +276,7 @@ Make your AI app more reliable and forward compatible, whi The easiest way to contribute is to pick an issue with the `good first issue` tag 💪. Read the contribution guidelines [here](/.github/CONTRIBUTING.md). -Bug Report? [File here](https://github.com/Portkey-AI/gateway/issues) | Feature Request? [File here](https://github.com/Portkey-AI/gateway/issues) +Bug Report? [File here](https://portkey.wiki/gh-78) | Feature Request? [File here](https://portkey.wiki/gh-78) ### Getting Started with the Community @@ -286,7 +286,7 @@ Join our weekly AI Engineering Hours every Friday (8 AM PT) to: - Share your experiences and get help - Stay updated with the latest development priorities -[Join the next session →](https://lu.ma/portkey?tag=ai%20engineer) | [Meeting notes](https://portkey.ai/docs/changelog/office-hour) +[Join the next session →](https://portkey.wiki/gh-101) | [Meeting notes](https://portkey.wiki/gh-102)
@@ -294,13 +294,13 @@ Join our weekly AI Engineering Hours every Friday (8 AM PT) to: Join our growing community around the world, for help, ideas, and discussions on AI. -- View our official [Blog](https://portkey.ai/blog) -- Chat with us on [Discord](https://portkey.ai/community) -- Follow us on [Twitter](https://twitter.com/PortkeyAI) -- Connect with us on [LinkedIn](https://www.linkedin.com/company/portkey-ai/) +- View our official [Blog](https://portkey.wiki/gh-78) +- Chat with us on [Discord](https://portkey.wiki/community) +- Follow us on [Twitter](https://portkey.wiki/gh-79) +- Connect with us on [LinkedIn](https://portkey.wiki/gh-80) - Read the documentation in [Japanese](./.github/README.jp.md) - - +- Visit us on [YouTube](https://portkey.wiki/gh-103) +- Join our [Dev community](https://portkey.wiki/gh-82) ![Rubeus Social Share (4)](https://github.com/Portkey-AI/gateway/assets/971978/89d6f0af-a95d-4402-b451-14764c40d03f) From 9e7e23fbb628d673e87200650efd501392e792bf Mon Sep 17 00:00:00 2001 From: vrushankportkey <134934501+vrushankportkey@users.noreply.github.com> Date: Fri, 29 Nov 2024 17:32:34 +0530 Subject: [PATCH 25/35] Create link checker yaml --- .github/workflows/main.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..e513f4e7 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,40 @@ +name: Check Markdown links + +on: + push: + paths: + - '**/*.md' # Only run when markdown files change + schedule: + - cron: "0 0 * * 0" # Run weekly on Sundays + workflow_dispatch: # Allows manual triggering + +jobs: + linkChecker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Link Checker + uses: lycheeverse/lychee-action@v1.8.0 + with: + args: --verbose --no-progress './**/*.md' + fail: true # Fail the action if broken links are found + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Create an issue if the job fails (optional) + - name: Create Issue If Failed + if: failure() + uses: actions/github-script@v6 + with: + script: | + const title = '🔗 Broken links found in documentation'; + const body = 'The link checker found broken links in the documentation. Please check the [workflow run](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) for details.'; + + github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['documentation', 'broken-links'] + }); From 2922e24226a7cfd675c2525c4455b5d7dd82b2cf Mon Sep 17 00:00:00 2001 From: vrushankportkey Date: Fri, 29 Nov 2024 17:38:40 +0530 Subject: [PATCH 26/35] one broken link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6321a67c..529de1a8 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ Meetings of Minutes [published here](https://portkey.wiki/gh-36). ### ☄️ Trending - Use models from [Nvidia NIM](/cookbook/providers/nvidia.ipynb) with AI Gateway - Monitor [CrewAI Agents](/cookbook/monitoring-agents/CrewAI_with_Telemetry.ipynb) with Portkey! -- Comparing [Top 10 LMSYS Models](./use-cases/LMSYS%20Series/comparing-top10-LMSYS-models-with-Portkey.ipynb) with AI Gateway. +- Comparing [Top 10 LMSYS Models](/cookbook/use-cases/LMSYS%20Series/comparing-top10-LMSYS-models-with-Portkey.ipynb) with AI Gateway. ### 🚨 Latest * [Create Synthetic Datasets using Nemotron](/cookbook/use-cases/Nemotron_GPT_Finetuning_Portkey.ipynb) From d7df3d773fe04c81fef5833d9e5fa2e83cbe74ce Mon Sep 17 00:00:00 2001 From: Mahesh Date: Fri, 29 Nov 2024 17:42:21 +0530 Subject: [PATCH 27/35] fix: handle proxy headers stripping inside request handler --- src/handlers/handlerUtils.ts | 32 +++++++++++++++++++++++++++++--- src/handlers/proxyHandler.ts | 2 +- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 52f313aa..8aba61cc 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -43,8 +43,34 @@ export function constructRequest( method: string, forwardHeaders: string[], requestHeaders: Record, - fn: endpointStrings + fn: endpointStrings, + c: Context ) { + // Handle proxy headers + if (fn === 'proxy') { + const final = {} as Record; + const poweredByHeadersPattern = `x-${POWERED_BY}-`; + const headersToAvoidForCloudflare = ['expect']; + const headersToIgnore = [ + ...(env(c).CUSTOM_HEADERS_TO_IGNORE ?? []), + headersToAvoidForCloudflare, + ]; + headersToIgnore.push('content-length'); + Object.keys(requestHeaders).forEach((key: string) => { + if ( + !headersToIgnore.includes(key) && + !key.startsWith(poweredByHeadersPattern) + ) { + final[key] = requestHeaders[key]; + } + }); + // Remove brotli from accept-encoding because cloudflare has problems with it + if (final['accept-encoding']?.includes('br')) + final['accept-encoding'] = final['accept-encoding']?.replace('br', ''); + + return final; + } + let baseHeaders: any = { 'content-type': 'application/json', }; @@ -69,7 +95,6 @@ export function constructRequest( ...baseHeaders, ...headers, ...forwardHeadersMap, - ...(fn === 'proxy' ? requestHeaders : {}), }; let fetchOptions: RequestInit = { @@ -269,7 +294,8 @@ export async function tryPost( method, forwardHeaders, requestHeaders, - fn + fn, + c ); const headerContentType = headers[HEADER_KEYS.CONTENT_TYPE]; diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index e34f22c4..68d0e55a 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -65,7 +65,7 @@ export async function proxyHandler(c: Context): Promise { c, camelCaseConfig, request, - headersToSend(requestHeaders, env(c).CUSTOM_HEADERS_TO_IGNORE ?? []), + requestHeaders, 'proxy', c.req.method, 'config' From 8d4d872e6a997ac022691ec55c355fe879680c9d Mon Sep 17 00:00:00 2001 From: Mahesh Date: Fri, 29 Nov 2024 19:11:45 +0530 Subject: [PATCH 28/35] fix: keep method for proxy calls --- src/handlers/handlerUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 8aba61cc..1225eb88 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -68,7 +68,7 @@ export function constructRequest( if (final['accept-encoding']?.includes('br')) final['accept-encoding'] = final['accept-encoding']?.replace('br', ''); - return final; + return { headers: final, method }; } let baseHeaders: any = { From df843804bb64aa40d6e067a6e962183ad3599bf9 Mon Sep 17 00:00:00 2001 From: visargD Date: Fri, 29 Nov 2024 23:52:01 +0530 Subject: [PATCH 29/35] chore: refactoring proxy headers implementation --- src/handlers/handlerUtils.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 1225eb88..1690aab5 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -46,9 +46,9 @@ export function constructRequest( fn: endpointStrings, c: Context ) { + let proxyHeaders: Record = {}; // Handle proxy headers if (fn === 'proxy') { - const final = {} as Record; const poweredByHeadersPattern = `x-${POWERED_BY}-`; const headersToAvoidForCloudflare = ['expect']; const headersToIgnore = [ @@ -61,14 +61,14 @@ export function constructRequest( !headersToIgnore.includes(key) && !key.startsWith(poweredByHeadersPattern) ) { - final[key] = requestHeaders[key]; + proxyHeaders[key] = requestHeaders[key]; } }); // Remove brotli from accept-encoding because cloudflare has problems with it - if (final['accept-encoding']?.includes('br')) - final['accept-encoding'] = final['accept-encoding']?.replace('br', ''); - - return { headers: final, method }; + if (proxyHeaders['accept-encoding']?.includes('br')) + proxyHeaders['accept-encoding'] = proxyHeaders[ + 'accept-encoding' + ]?.replace('br', ''); } let baseHeaders: any = { @@ -95,6 +95,7 @@ export function constructRequest( ...baseHeaders, ...headers, ...forwardHeadersMap, + ...(fn === 'proxy' && proxyHeaders), }; let fetchOptions: RequestInit = { From 40688ac72ea378145fd62dd35e2404c550ec9cb6 Mon Sep 17 00:00:00 2001 From: visargD Date: Fri, 29 Nov 2024 23:52:45 +0530 Subject: [PATCH 30/35] chore: remove unused imports and function --- src/handlers/proxyHandler.ts | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/src/handlers/proxyHandler.ts b/src/handlers/proxyHandler.ts index 68d0e55a..ec55aa8a 100644 --- a/src/handlers/proxyHandler.ts +++ b/src/handlers/proxyHandler.ts @@ -1,11 +1,10 @@ import { Context } from 'hono'; -import { CONTENT_TYPES, POWERED_BY } from '../globals'; +import { CONTENT_TYPES } from '../globals'; import { constructConfigFromRequestHeaders, tryTargetsRecursively, } from './handlerUtils'; import { RouterError } from '../errors/RouterError'; -import { env } from 'hono/adapter'; async function getRequestData(request: Request, contentType: string) { let finalRequest: any; @@ -24,34 +23,6 @@ async function getRequestData(request: Request, contentType: string) { return finalRequest; } -function headersToSend( - headersObj: Record, - customHeadersToIgnore: Array -): Record { - let final: Record = {}; - const poweredByHeadersPattern = `x-${POWERED_BY}-`; - const headersToAvoidForCloudflare = ['expect']; - const headersToAvoid = [ - ...customHeadersToIgnore, - ...headersToAvoidForCloudflare, - ]; - headersToAvoid.push('content-length'); - Object.keys(headersObj).forEach((key: string) => { - if ( - !headersToAvoid.includes(key) && - !key.startsWith(poweredByHeadersPattern) - ) { - final[key] = headersObj[key]; - } - }); - - // Remove brotli from accept-encoding because cloudflare has problems with it - if (final['accept-encoding']?.includes('br')) - final['accept-encoding'] = final['accept-encoding']?.replace('br', ''); - - return final; -} - export async function proxyHandler(c: Context): Promise { try { let requestHeaders = Object.fromEntries(c.req.raw.headers); From c9e7d4d89f0afd4cc17cc83a4daca2e07fd4ea26 Mon Sep 17 00:00:00 2001 From: visargD Date: Sat, 30 Nov 2024 00:53:08 +0530 Subject: [PATCH 31/35] fix: headers to ignore list for proxy requests --- src/handlers/handlerUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 1690aab5..6bfc2f0f 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -53,7 +53,7 @@ export function constructRequest( const headersToAvoidForCloudflare = ['expect']; const headersToIgnore = [ ...(env(c).CUSTOM_HEADERS_TO_IGNORE ?? []), - headersToAvoidForCloudflare, + ...headersToAvoidForCloudflare, ]; headersToIgnore.push('content-length'); Object.keys(requestHeaders).forEach((key: string) => { From e4c7b9b231a40c032cd9e69001a9aba4dcd43d77 Mon Sep 17 00:00:00 2001 From: Narendranath Gogineni Date: Tue, 3 Dec 2024 18:17:17 +0530 Subject: [PATCH 32/35] add optional model name header for sagemaker --- src/handlers/handlerUtils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 6bfc2f0f..b3dbd4a5 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -786,6 +786,8 @@ export function constructConfigFromRequestHeaders( requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-inference-component`], amznSagemakerSessionId: requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-session-id`], + amznSagemakerModelName: + requestHeaders[`x-${POWERED_BY}-amzn-sagemaker-model-name`], }; const workersAiConfig = { From 8c615cd642d6a8d02658d684415b0f335c4a82ea Mon Sep 17 00:00:00 2001 From: vrushankportkey Date: Mon, 9 Dec 2024 15:33:47 +0530 Subject: [PATCH 33/35] minor fixes --- .../workflows/{main.yml => link-checker.yml} | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) rename .github/workflows/{main.yml => link-checker.yml} (52%) diff --git a/.github/workflows/main.yml b/.github/workflows/link-checker.yml similarity index 52% rename from .github/workflows/main.yml rename to .github/workflows/link-checker.yml index e513f4e7..35697ac9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/link-checker.yml @@ -3,10 +3,13 @@ name: Check Markdown links on: push: paths: - - '**/*.md' # Only run when markdown files change + - '**/*.md' # Only run when markdown files change + pull_request: + branches: + - main schedule: - - cron: "0 0 * * 0" # Run weekly on Sundays - workflow_dispatch: # Allows manual triggering + - cron: '0 0 * * 0' # Run weekly on Sundays + workflow_dispatch: # Allows manual triggering jobs: linkChecker: @@ -17,12 +20,10 @@ jobs: - name: Link Checker uses: lycheeverse/lychee-action@v1.8.0 with: - args: --verbose --no-progress './**/*.md' - fail: true # Fail the action if broken links are found + args: --verbose --no-progress --fail './**/*.md' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Create an issue if the job fails (optional) - name: Create Issue If Failed if: failure() uses: actions/github-script@v6 @@ -30,11 +31,20 @@ jobs: script: | const title = '🔗 Broken links found in documentation'; const body = 'The link checker found broken links in the documentation. Please check the [workflow run](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) for details.'; - - github.rest.issues.create({ + + const existingIssues = await github.rest.issues.listForRepo({ owner: context.repo.owner, repo: context.repo.repo, - title: title, - body: body, - labels: ['documentation', 'broken-links'] + labels: 'documentation,broken-links', }); + + const issueExists = existingIssues.data.some(issue => issue.title === title); + if (!issueExists) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['documentation', 'broken-links'] + }); + } From 185f0c970838c915f8867a590c10a25a279f1169 Mon Sep 17 00:00:00 2001 From: Mahesh Date: Mon, 9 Dec 2024 13:03:56 +0530 Subject: [PATCH 34/35] feat: add tools support - ollama --- src/providers/ollama/chatComplete.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/providers/ollama/chatComplete.ts b/src/providers/ollama/chatComplete.ts index 3d7cec98..0451f237 100644 --- a/src/providers/ollama/chatComplete.ts +++ b/src/providers/ollama/chatComplete.ts @@ -64,6 +64,9 @@ export const OllamaChatCompleteConfig: ProviderConfig = { default: 100, min: 0, }, + tools: { + param: 'tools', + }, }; export interface OllamaChatCompleteResponse extends ChatCompletionResponse { From be4d531758c73dd8539fff82670c74c32b184ade Mon Sep 17 00:00:00 2001 From: Mahesh Date: Tue, 10 Dec 2024 15:55:13 +0530 Subject: [PATCH 35/35] chore: add tools for groq --- src/providers/groq/chatComplete.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/providers/groq/chatComplete.ts b/src/providers/groq/chatComplete.ts index 2219a013..aa824f44 100644 --- a/src/providers/groq/chatComplete.ts +++ b/src/providers/groq/chatComplete.ts @@ -54,6 +54,10 @@ export const GroqChatCompleteConfig: ProviderConfig = { max: 1, min: 1, }, + tools: { + param: 'tools', + required: false, + }, }; export interface GroqChatCompleteResponse extends ChatCompletionResponse {}