The new APIM policies for Azure OpenAI do not play nicely with some m…

…odels like GPT-4-Vision. (#58) * The new APIM policies for Azure OpenAI do not play nicely with some models like GPT-4-Vision. Added a condition to detect these calls and prevent using these policies.
Azure · Jun 6, 2024 · 6208a13 · 6208a13
1 parent 08956e4
commit 6208a13
Showing 1 changed file with 28 additions and 10 deletions.
diff --git a/infra/modules/apim/main.tf b/infra/modules/apim/main.tf
@@ -130,16 +130,34 @@ resource "azurerm_api_management_api_policy" "policy" {
                 </required-claims>
             </validate-jwt>
 
-            <azure-openai-emit-token-metric
-                namespace="AzureOpenAI">   
-                <dimension name="API ID" />
-                <dimension name="Operation ID" />
-                <dimension name="Client IP" value="@(context.Request.IpAddress)" />
-            </azure-openai-emit-token-metric>
-
-            <azure-openai-token-limit
-              counter-key="@(context.Request.IpAddress)"
-              tokens-per-minute="10000" estimate-prompt-tokens="false" remaining-tokens-variable-name="remainingTokens" />
+            <choose>
+                <when condition="@(context.Request.Body.As<JObject>(preserveContent: true)["messages"]?.All(message => message["content"].All(content => !(content is JObject))) == true)">
+                    
+                    <!-- If all type properties are 'text' or there are no type properties, apply the new Azure OpenAI policies -->
+                    
+                    <trace source="Azure OpenAI Policies" severity="information">
+                        <message>Using Azure OpenAI policies.</message>
+                        <metadata name="Using_Azure_OpenAI_Policies" value="true" />
+                    </trace>
+                    
+                    <azure-openai-emit-token-metric
+                        namespace="AzureOpenAI">
+                        <dimension name="API ID" />
+                        <dimension name="Operation ID" />
+                        <dimension name="Client IP" value="@(context.Request.IpAddress)" />
+                    </azure-openai-emit-token-metric>
+
+                    <azure-openai-token-limit
+                      counter-key="@(context.Request.IpAddress)"
+                      tokens-per-minute="10000" estimate-prompt-tokens="false" remaining-tokens-variable-name="remainingTokens" />
+                </when>
+                <otherwise>
+                    <trace source="Azure OpenAI Policies" severity="information">
+                        <message>Not using Azure OpenAI policies.</message>
+                        <metadata name="Using_Azure_OpenAI_Policies" value="false" />
+                    </trace>
+                </otherwise>
+            </choose>
 
             <set-backend-service backend-id="${azapi_resource.apim_backend_pool.name}" />
         </inbound>