fix: logprops for vertex and deepinfra streaming response

[narengogi]

#897

description:

1. deepinfra streaming transformer was not working, because the split pattern is incorrect.
2. vertex logprobs is a new beta feature and is only supported for a few models like gemini-1.5-flash, vertex does not send top_logprobs for an index in the same array as it sends the chosen log probs, and it does not send the bytes array like openai

*testing done#

1. deepinfra streaming:
   request:

curl --location 'http://localhost:8787/v1/chat/completions' \
--header 'x-portkey-provider: deepinfra' \
--header 'Authorization: {{DEEPINFRA_API_KEY}}' \
--header 'Content-Type: application/json' \
--data '{
    "model": "meta-llama/Meta-Llama-3-8B-Instruct",
    "max_tokens": 200,
    "stream": true,
    "messages": [
        {
            "role": "system",
            "content": "You are batman"
        },
        {
            "role": "user",
            "content": [
                {
                    "type": "text",
                    "text": "Who is the greatest detective"
                }
            ]
        }
    ]
}'

response:

data: {"id":"chatcmpl-RxfbCK2qdGrVur8vhIJuFZ3T","object":"chat.completion.chunk","created":1737994233,"model":"meta-llama/Meta-Llama-3-8B-Instruct","provider":"deepinfra","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":null},"finish_reason":null}]}

data: {"id":"chatcmpl-RxfbCK2qdGrVur8vhIJuFZ3T","object":"chat.completion.chunk","created":1737994233,"model":"meta-llama/Meta-Llama-3-8B-Instruct","provider":"deepinfra","choices":[{"index":0,"delta":{"role":"assistant","content":"(in","tool_calls":null},"finish_reason":null}]}

data: {"id":"chatcmpl-RxfbCK2qdGrVur8vhIJuFZ3T","object":"chat.completion.chunk","created":1737994233,"model":"meta-llama/Meta-Llama-3-8B-Instruct","provider":"deepinfra","choices":[{"index":0,"delta":{},"finish_reason":"length"}]}

data: [DONE]

3. vertex logprobs
   request:

curl --location 'http://localhost:8787/v1/chat/completions' \
--header 'x-portkey-provider: vertex-ai' \
--header 'x-portkey-vertex-region: us-central1' \
--header 'x-portkey-vertex-service-account-json: {{SERVICE_JSON}}' \
--header 'Content-Type: application/json' \
--header 'x-portkey-strict-open-ai-compliance: false' \
--header 'Authorization: abc' \
--data '{
    "model": "gemini-1.5-flash",
    "max_tokens": 500,
    "stream": true,
    "logprobs": true,
    
    "messages": [
        {
            "role": "developer",
            "content": "Say Hi"
        },
        {
            "role": "user",
            "content": "how are you homie"
        }
    ]
}'

response:

{
    "id": "portkey-cefb61fa-14a7-4e2c-8203-518c146e085d",
    "object": "chat_completion",
    "created": 1738051125,
    "model": "Unknown",
    "provider": "vertex-ai",
    "choices": [
        {
            "message": {
                "role": "assistant",
                "content": "Hey there, homie! I'm doing well, thanks for asking. How about you? 😊 \n"
            },
            "index": 0,
            "finish_reason": "STOP",
            "logprobs": {
                "content": [
                    {
                        "token": "Hey",
                        "logprob": -0.9982398,
                        "bytes": [
                            72,
                            101,
                            121
                        ],
                        "top_logprobs": [
                            {
                                "token": "I",
                                "logprob": -0.5607398,
                                "bytes": [
                                    73
                                ]
                            },
                            {
                                "token": "Hey",
                                "logprob": -0.9982398,
                                "bytes": [
                                    72,
                                    101,
                                    121
                                ]
                            },
                            {
                                "token": "As",
                                "logprob": -2.8107398,
                                "bytes": [
                                    65,
                                    115
                                ]
                            }
                        ]
                    },
                    {
                        "token": " there",
                        "logprob": -0.018753633,
                        "bytes": [
                            32,
                            116,
                            104,
                            101,
                            114,
                            101
                        ],
                        "top_logprobs": [
                            {
                                "token": " there",
                                "logprob": -0.018753633,
                                "bytes": [
                                    32,
                                    116,
                                    104,
                                    101,
                                    114,
                                    101
                                ]
                            },
                            {
                                "token": " homie",
                                "logprob": -4.172589,
                                "bytes": [
                                    32,
                                    104,
                                    111,
                                    109,
                                    105,
                                    101
                                ]
                            },
                            {
                                "token": ",",
                                "logprob": -6.4184747,
                                "bytes": [
                                    44
                                ]
                            }
                        ]
                    },
                    {
                        "token": ",",
                        "logprob": -1.0160006,
                        "bytes": [
                            44
                        ],
                        "top_logprobs": [
                            {
                                "token": ",",
                                "logprob": -0.45336708,
                                "bytes": [
                                    44
                                ]
                            },
                            {
                                "token": "!",
                                "logprob": -1.0160006,
                                "bytes": [
                                    33
                                ]
                            },
                            {
                                "token": " homie",
                                "logprob": -6.0041447,
                                "bytes": [
                                    32,
                                    104,
                                    111,
                                    109,
                                    105,
                                    101
                                ]
                            }
                        ]
                    },
                    {
                        "token": " homie",
                        "logprob": -0.01658828,
                        "bytes": [
                            32,
                            104,
                            111,
                            109,
                            105,
                            101
                        ],
                        "top_logprobs": [
                            {
                                "token": " homie",
                                "logprob": -0.01658828,
                                "bytes": [
                                    32,
                                    104,
                                    111,
                                    109,
                                    105,
                                    101
                                ]
                            },
                            {
                                "token": " I",
                                "logprob": -4.843561,
                                "bytes": [
                                    32,
                                    73
                                ]
                            },
                            {
                                "token": " doing",
                                "logprob": -4.9144,
                                "bytes": [
                                    32,
                                    100,
                                    111,
                                    105,
                                    110,
                                    103
                                ]
                            }
                        ]
                    },
                    {
                        "token": "!",
                        "logprob": -0.0002379069,
                        "bytes": [
                            33
                        ],
                        "top_logprobs": [
                            {
                                "token": "!",
                                "logprob": -0.0002379069,
                                "bytes": [
                                    33
                                ]
                            },
                            {
                                "token": ".",
                                "logprob": -8.793017,
                                "bytes": [
                                    46
                                ]
                            },
                            {
                                "token": ",",
                                "logprob": -9.938626,
                                "bytes": [
                                    44
                                ]
                            }
                        ]
                    },
                    {
                        "token": " I",
                        "logprob": -0.18067561,
                        "bytes": [
                            32,
                            73
                        ],
                        "top_logprobs": [
                            {
                                "token": " I",
                                "logprob": -0.18067561,
                                "bytes": [
                                    32,
                                    73
                                ]
                            },
                            {
                                "token": "  ",
                                "logprob": -2.0306568,
                                "bytes": [
                                    32,
                                    32
                                ]
                            },
                            {
                                "token": " Doing",
                                "logprob": -4.2882576,
                                "bytes": [
                                    32,
                                    68,
                                    111,
                                    105,
                                    110,
                                    103
                                ]
                            }
                        ]
                    },
                    {
                        "token": "'",
                        "logprob": -0.0004127187,
                        "bytes": [
                            39
                        ],
                        "top_logprobs": [
                            {
                                "token": "'",
                                "logprob": -0.0004127187,
                                "bytes": [
                                    39
                                ]
                            },
                            {
                                "token": " am",
                                "logprob": -7.9259825,
                                "bytes": [
                                    32,
                                    97,
                                    109
                                ]
                            },
                            {
                                "token": "’",
                                "logprob": -9.905042,
                                "bytes": [
                                    8217
                                ]
                            }
                        ]
                    },
                    {
                        "token": "m",
                        "logprob": 0,
                        "bytes": [
                            109
                        ],
                        "top_logprobs": [
                            {
                                "token": "m",
                                "logprob": 0,
                                "bytes": [
                                    109
                                ]
                            },
                            {
                                "token": "ve",
                                "logprob": -18.266674,
                                "bytes": [
                                    118,
                                    101
                                ]
                            },
                            {
                                "token": "M",
                                "logprob": -21.443516,
                                "bytes": [
                                    77
                                ]
                            }
                        ]
                    },
                    {
                        "token": " doing",
                        "logprob": -0.009197044,
                        "bytes": [
                            32,
                            100,
                            111,
                            105,
                            110,
                            103
                        ],
                        "top_logprobs": [
                            {
                                "token": " doing",
                                "logprob": -0.009197044,
                                "bytes": [
                                    32,
                                    100,
                                    111,
                                    105,
                                    110,
                                    103
                                ]
                            },
                            {
                                "token": " an",
                                "logprob": -5.08057,
                                "bytes": [
                                    32,
                                    97,
                                    110
                                ]
                            },
                            {
                                "token": " just",
                                "logprob": -5.874527,
                                "bytes": [
                                    32,
                                    106,
                                    117,
                                    115,
                                    116
                                ]
                            }
                        ]
                    },
                    {
                        "token": " well",
                        "logprob": -0.12513755,
                        "bytes": [
                            32,
                            119,
                            101,
                            108,
                            108
                        ],
                        "top_logprobs": [
                            {
                                "token": " well",
                                "logprob": -0.12513755,
                                "bytes": [
                                    32,
                                    119,
                                    101,
                                    108,
                                    108
                                ]
                            },
                            {
                                "token": " great",
                                "logprob": -2.1553938,
                                "bytes": [
                                    32,
                                    103,
                                    114,
                                    101,
                                    97,
                                    116
                                ]
                            },
                            {
                                "token": " good",
                                "logprob": -7.3035183,
                                "bytes": [
                                    32,
                                    103,
                                    111,
                                    111,
                                    100
                                ]
                            }
                        ]
                    },
                    {
                        "token": ",",
                        "logprob": -0.0014161067,
                        "bytes": [
                            44
                        ],
                        "top_logprobs": [
                            {
                                "token": ",",
                                "logprob": -0.0014161067,
                                "bytes": [
                                    44
                                ]
                            },
                            {
                                "token": " -",
                                "logprob": -7.5779905,
                                "bytes": [
                                    32,
                                    45
                                ]
                            },
                            {
                                "token": " thanks",
                                "logprob": -7.990432,
                                "bytes": [
                                    32,
                                    116,
                                    104,
                                    97,
                                    110,
                                    107,
                                    115
                                ]
                            }
                        ]
                    },
                    {
                        "token": " thanks",
                        "logprob": -0.006875369,
                        "bytes": [
                            32,
                            116,
                            104,
                            97,
                            110,
                            107,
                            115
                        ],
                        "top_logprobs": [
                            {
                                "token": " thanks",
                                "logprob": -0.006875369,
                                "bytes": [
                                    32,
                                    116,
                                    104,
                                    97,
                                    110,
                                    107,
                                    115
                                ]
                            },
                            {
                                "token": " just",
                                "logprob": -5.377832,
                                "bytes": [
                                    32,
                                    106,
                                    117,
                                    115,
                                    116
                                ]
                            },
                            {
                                "token": " thank",
                                "logprob": -6.253822,
                                "bytes": [
                                    32,
                                    116,
                                    104,
                                    97,
                                    110,
                                    107
                                ]
                            }
                        ]
                    },
                    {
                        "token": " for",
                        "logprob": -0.000049587838,
                        "bytes": [
                            32,
                            102,
                            111,
                            114
                        ],
                        "top_logprobs": [
                            {
                                "token": " for",
                                "logprob": -0.000049587838,
                                "bytes": [
                                    32,
                                    102,
                                    111,
                                    114
                                ]
                            },
                            {
                                "token": ".",
                                "logprob": -10.641565,
                                "bytes": [
                                    46
                                ]
                            },
                            {
                                "token": "!",
                                "logprob": -10.946255,
                                "bytes": [
                                    33
                                ]
                            }
                        ]
                    },
                    {
                        "token": " asking",
                        "logprob": -0.000030640404,
                        "bytes": [
                            32,
                            97,
                            115,
                            107,
                            105,
                            110,
                            103
                        ],
                        "top_logprobs": [
                            {
                                "token": " asking",
                                "logprob": -0.000030640404,
                                "bytes": [
                                    32,
                                    97,
                                    115,
                                    107,
                                    105,
                                    110,
                                    103
                                ]
                            },
                            {
                                "token": " checking",
                                "logprob": -10.448876,
                                "bytes": [
                                    32,
                                    99,
                                    104,
                                    101,
                                    99,
                                    107,
                                    105,
                                    110,
                                    103
                                ]
                            },
                            {
                                "token": "  ",
                                "logprob": -14.063285,
                                "bytes": [
                                    32,
                                    32
                                ]
                            }
                        ]
                    },
                    {
                        "token": ".",
                        "logprob": -0.039338674,
                        "bytes": [
                            46
                        ],
                        "top_logprobs": [
                            {
                                "token": ".",
                                "logprob": -0.039338674,
                                "bytes": [
                                    46
                                ]
                            },
                            {
                                "token": "!",
                                "logprob": -3.2782512,
                                "bytes": [
                                    33
                                ]
                            },
                            {
                                "token": " 😊",
                                "logprob": -7.4142833,
                                "bytes": [
                                    32,
                                    55357
                                ]
                            }
                        ]
                    },
                    {
                        "token": " How",
                        "logprob": -0.7216999,
                        "bytes": [
                            32,
                            72,
                            111,
                            119
                        ],
                        "top_logprobs": [
                            {
                                "token": " How",
                                "logprob": -0.7216999,
                                "bytes": [
                                    32,
                                    72,
                                    111,
                                    119
                                ]
                            },
                            {
                                "token": "  ",
                                "logprob": -1.0847313,
                                "bytes": [
                                    32,
                                    32
                                ]
                            },
                            {
                                "token": " 😊",
                                "logprob": -2.6281102,
                                "bytes": [
                                    32,
                                    55357
                                ]
                            }
                        ]
                    },
                    {
                        "token": " about",
                        "logprob": -0.12144157,
                        "bytes": [
                            32,
                            97,
                            98,
                            111,
                            117,
                            116
                        ],
                        "top_logprobs": [
                            {
                                "token": " about",
                                "logprob": -0.12144157,
                                "bytes": [
                                    32,
                                    97,
                                    98,
                                    111,
                                    117,
                                    116
                                ]
                            },
                            {
                                "token": " are",
                                "logprob": -2.1693122,
                                "bytes": [
                                    32,
                                    97,
                                    114,
                                    101
                                ]
                            },
                            {
                                "token": "'",
                                "logprob": -9.279811,
                                "bytes": [
                                    39
                                ]
                            }
                        ]
                    },
                    {
                        "token": " you",
                        "logprob": -0.06651771,
                        "bytes": [
                            32,
                            121,
                            111,
                            117
                        ],
                        "top_logprobs": [
                            {
                                "token": " you",
                                "logprob": -0.06651771,
                                "bytes": [
                                    32,
                                    121,
                                    111,
                                    117
                                ]
                            },
                            {
                                "token": " yourself",
                                "logprob": -2.7434645,
                                "bytes": [
                                    32,
                                    121,
                                    111,
                                    117,
                                    114,
                                    115,
                                    101,
                                    108,
                                    102
                                ]
                            },
                            {
                                "token": "  ",
                                "logprob": -12.335888,
                                "bytes": [
                                    32,
                                    32
                                ]
                            }
                        ]
                    },
                    {
                        "token": "?",
                        "logprob": -0.000026582527,
                        "bytes": [
                            63
                        ],
                        "top_logprobs": [
                            {
                                "token": "?",
                                "logprob": -0.000026582527,
                                "bytes": [
                                    63
                                ]
                            },
                            {
                                "token": ",",
                                "logprob": -10.650625,
                                "bytes": [
                                    44
                                ]
                            },
                            {
                                "token": "?\r",
                                "logprob": -13.6296835,
                                "bytes": [
                                    63,
                                    13
                                ]
                            }
                        ]
                    },
                    {
                        "token": " 😊",
                        "logprob": -0.68744195,
                        "bytes": [
                            32,
                            55357
                        ],
                        "top_logprobs": [
                            {
                                "token": " 😊",
                                "logprob": -0.68744195,
                                "bytes": [
                                    32,
                                    55357
                                ]
                            },
                            {
                                "token": " 😄",
                                "logprob": -1.446374,
                                "bytes": [
                                    32,
                                    55357
                                ]
                            },
                            {
                                "token": "  ",
                                "logprob": -2.4553194,
                                "bytes": [
                                    32,
                                    32
                                ]
                            }
                        ]
                    },
                    {
                        "token": " ",
                        "logprob": -0.058495812,
                        "bytes": [
                            32
                        ],
                        "top_logprobs": [
                            {
                                "token": " ",
                                "logprob": -0.058495812,
                                "bytes": [
                                    32
                                ]
                            },
                            {
                                "token": "  ",
                                "logprob": -2.9220326,
                                "bytes": [
                                    32,
                                    32
                                ]
                            },
                            {
                                "token": " What",
                                "logprob": -6.1501746,
                                "bytes": [
                                    32,
                                    87,
                                    104,
                                    97,
                                    116
                                ]
                            }
                        ]
                    },
                    {
                        "token": "\n",
                        "logprob": -0.00000643695,
                        "bytes": [
                            10
                        ],
                        "top_logprobs": [
                            {
                                "token": "\n",
                                "logprob": -0.00000643695,
                                "bytes": [
                                    10
                                ]
                            },
                            {
                                "token": "\n\n",
                                "logprob": -12.289864,
                                "bytes": [
                                    10,
                                    10
                                ]
                            },
                            {
                                "token": "🤖",
                                "logprob": -13.272081,
                                "bytes": [
                                    55358
                                ]
                            }
                        ]
                    }
                ]
            },
            "safetyRatings": [
                {
                    "category": "HARM_CATEGORY_HATE_SPEECH",
                    "probability": "NEGLIGIBLE",
                    "probabilityScore": 0.05419922,
                    "severity": "HARM_SEVERITY_NEGLIGIBLE",
                    "severityScore": 0.01977539
                },
                {
                    "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
                    "probability": "NEGLIGIBLE",
                    "probabilityScore": 0.46875,
                    "severity": "HARM_SEVERITY_NEGLIGIBLE",
                    "severityScore": 0.14453125
                },
                {
                    "category": "HARM_CATEGORY_HARASSMENT",
                    "probability": "NEGLIGIBLE",
                    "probabilityScore": 0.061035156,
                    "severity": "HARM_SEVERITY_NEGLIGIBLE",
                    "severityScore": 0.025146484
                },
                {
                    "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
                    "probability": "NEGLIGIBLE",
                    "probabilityScore": 0.30664063,
                    "severity": "HARM_SEVERITY_NEGLIGIBLE",
                    "severityScore": 0.078125
                }
            ]
        }
    ],
    "usage": {
        "prompt_tokens": 6,
        "completion_tokens": 22,
        "total_tokens": 28
    }
}