gpt-35-turbo-instruct-fullpolicy.xml

<policies>
    <inbound>
        <base />
        <set-variable name="deploymentId" value="/deployments/gpt-35-turbo-instruct" />
        <set-variable name="urlId" value="@(new Random(context.RequestId.GetHashCode()).Next(1, 3))" />
        <choose>
            <when condition="@(context.Variables.GetValueOrDefault<int>("urlId") == 1)">
                <set-variable name="backendUrl" value="{{aoai-australiaeast-endpoint}}" />
                <set-header name="api-key" exists-action="override">
                    <value>{{aoai-australiaeast-key}}</value>
                </set-header>
            </when>
            <when condition="@(context.Variables.GetValueOrDefault<int>("urlId") == 2)">
                <set-variable name="backendUrl" value="{{aoai-swedencentral-endpoint}}" />
                <set-header name="api-key" exists-action="override">
                    <value>{{aoai-swedencentral-key}}</value>
                </set-header>
            </when>
        </choose>
        <cache-lookup vary-by-developer="false" vary-by-developer-groups="false" downstream-caching-type="none">
            <vary-by-header>Accept</vary-by-header>
            <vary-by-header>Accept-Charset</vary-by-header>
            <vary-by-header>Authorization</vary-by-header>
        </cache-lookup>
        <set-backend-service base-url="@((string)context.Variables["backendUrl"] + (string)context.Variables["deploymentId"])" />
    </inbound>
    <backend>
        <retry condition="@(context.Response.StatusCode == 429)" count="18" interval="4" delta="2" first-fast-retry="true">
            <set-variable name="urlId" value="@(context.Variables.GetValueOrDefault<int>("urlId") % 2 + 1)" />
            <choose>
                <when condition="@(context.Response != null && (context.Response.StatusCode == 429 ) && context.Variables.GetValueOrDefault<int>("urlId") == 1)">
                    <set-variable name="backendUrl" value="{{aoai-australiaeast-endpoint}}" />
                    <set-header name="api-key" exists-action="override">
                        <value>{{aoai-australiaeast-key}}</value>
                    </set-header>
                </when>
                <when condition="@(context.Response != null && (context.Response.StatusCode == 429 ) && context.Variables.GetValueOrDefault<int>("urlId") == 2)">
                    <set-variable name="backendUrl" value="{{aoai-swedencentral-endpoint}}" />
                    <set-header name="api-key" exists-action="override">
                        <value>{{aoai-swedencentral-key}}</value>
                    </set-header>
                </when>
            </choose>
            <set-backend-service base-url="@((string)context.Variables["backendUrl"] + (string)context.Variables["deploymentId"])" />
            <forward-request buffer-request-body="true" />
        </retry>
    </backend>
    <outbound>
        <set-header name="Backend-Service-URL" exists-action="override">
            <value>@((string)context.Variables["backendUrl"])</value>
        </set-header>
        <cache-store duration="20" />
        <base />
    </outbound>
    <on-error>
        <base />
    </on-error>
</policies>