diff --git a/checklists/aoai_checklist.en.json b/checklists/aoai_checklist.en.json index 741f2ba32..2bc9daa20 100644 --- a/checklists/aoai_checklist.en.json +++ b/checklists/aoai_checklist.en.json @@ -5,6 +5,7 @@ "subcategory": "Metaprompting", "text": "Follow Metaprompting guardrails for resonsible AI", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "a85b86ad-884f-48e3-9273-4b875ba18f10", "id": "AOAI.1", "severity": "High", @@ -15,6 +16,7 @@ "subcategory": "Content Safety", "text": "Review and implement Azure AI content safety", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "cd288bed-6b17-4cb8-8454-51e1aed3453a", "id": "AOAI.2", "severity": "High", @@ -25,6 +27,7 @@ "subcategory": "UX best practice", "text": "Review the considerations in HAI toolkit guidance and apply those interaction practices for the slution", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "ec723923-7a15-42d6-ac5e-402925387e5c", "id": "AOAI.3", "severity": "Medium", @@ -35,6 +38,7 @@ "subcategory": "Jail break Safety", "text": "Implement Prompt shields and groundedness detection using Content Safety ", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "a9c27d9c-42bb-46bd-8c69-99a246f3389a", "id": "AOAI.4", "severity": "High", @@ -45,6 +49,7 @@ "subcategory": "Token Optimization", "text": "Use prompt compression tools like LLMLingua or gprtrim", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "7f42c78e-78cb-46a2-8ad1-90916e6a8d8f", "id": "AOAI.5", "severity": "Medium", @@ -55,6 +60,7 @@ "subcategory": "Token Optimization", "text": "Use tiktoken to understand token sizes for token optimizations in conversational mode", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "adfe27be-e297-401a-a352-baaab79b088d", "id": "AOAI.6", "severity": "High", @@ -65,6 +71,7 @@ "subcategory": "Costing Model", "text": "Evaluate usage of billing models - PAYG vs PTU", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "aa80932c-8ec9-4d1b-a770-26e5e6beba9e", "id": "AOAI.7", "severity": "High", @@ -75,6 +82,7 @@ "subcategory": "Quota Management", "text": "Consider Quota management practices", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "d5f3547c-c346-4d81-9028-a71ffe1b9b5d", "id": "AOAI.8", "severity": "High", @@ -85,6 +93,7 @@ "subcategory": "Load Balancing", "text": "Use Load balancer solutions like APIM based gateway for balancing load and capacity across services and regions", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "9de0d5d7-31d4-41e3-911c-817bfafbc410", "id": "AOAI.9", "severity": "Medium", @@ -95,6 +104,7 @@ "subcategory": "Load Balancing", "text": "Consider Gateway patterns with APIM or solutions like AI central for better rate limiting, load balancing, authentication and logging", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "d4391898-cd28-48be-b6b1-7cb8245451e1", "id": "AOAI.10", "severity": "High", @@ -105,6 +115,7 @@ "subcategory": "Monitoring", "text": "Enable monitoring for your AOAI instances", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "aed3453a-ec72-4392-97a1-52d6cc5e4029", "id": "AOAI.11", "severity": "High", @@ -115,6 +126,7 @@ "subcategory": "Alerts", "text": "Create alerts to notify teams of events such as an entry in the activity log created by an action performed on the resource, such as regenerating its subscription keys or a metric threshold such as the number of errors exceeding 10 in an hour", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "697cb391-ed16-4b2d-886f-0a0241addde6", "id": "AOAI.12", "severity": "High", @@ -125,6 +137,7 @@ "subcategory": "Monitoring", "text": "Monitor token usage to prevent service disruptions due to capacity", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "8a477cde-b486-41bc-9bc1-0ae66e25d4d5", "id": "AOAI.13", "severity": "High", @@ -135,6 +148,7 @@ "subcategory": "Observability", "text": "observe metrics like processed inference tokens, generated completion tokens monitor for rate limit", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "a3aec2c4-e243-46b0-936c-b45e17960eee", "id": "AOAI.14", "severity": "Medium", @@ -145,6 +159,7 @@ "subcategory": "Observability", "text": "If the diagnostics are not sufficient for you, consider using a gateway such as Azure API Managements in front of Azure OpenAI to log both incoming prompts and outgoing responses, where permitted", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "fbdf4cc2-eec4-4d76-8c31-d25ffbb46a39", "id": "AOAI.15", "severity": "Low", @@ -155,6 +170,7 @@ "subcategory": "Infrastructure Deployment", "text": "Use Infrastructure as code to deploy the Azure OpenAI Service, model deployments, and all related resources", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "3af30ed3-2947-498b-8178-a2c5a46ceb54", "id": "AOAI.16", "severity": "High", @@ -165,6 +181,7 @@ "subcategory": "Authentication", "text": "Use Microsoft Entra Authentication with Managed Identity instead of API Key", "waf": "Security", + "service": "Azure OpenAI", "guid": "4350d092-d234-4292-a752-8537a551c5bf", "id": "AOAI.17", "severity": "High", @@ -174,7 +191,8 @@ "category": "Responsible AI", "subcategory": "Evaluation", "text": "Evaluate the performance/accuracy of the system with a known golden dataset which has the inputs and the correct answers. Leverage capabilities in PromptFlow for Evaluation.", - "waf": "Operational Execellenxce", + "waf": "Operational Execellence", + "service": "Azure OpenAI", "guid": "4e4f1854-287d-45cd-a126-cc031af5b1fc", "id": "AOAI.18", "severity": "High", @@ -185,6 +203,7 @@ "subcategory": "Hosting model", "text": "Evaluate usage of Provisioned throughput model ", "waf": "Performance", + "service": "Azure OpenAI", "guid": "68889535-e327-4897-b31b-67d67be5962a", "id": "AOAI.19", "severity": "High", @@ -195,6 +214,7 @@ "subcategory": "Throughput definition", "text": "Define and evaluate the throughput of the system based on tokens & response per minute and align with requirements", "waf": "Performance", + "service": "Azure OpenAI", "guid": "1193846d-697c-4b39-8ed1-6b2d186f0a02", "id": "AOAI.20", "severity": "High", @@ -205,6 +225,7 @@ "subcategory": "Latency improvement", "text": "Improve latency of the system by limiting token sizes, streaming options", "waf": "Performance", + "service": "Azure OpenAI", "guid": "41addde6-8a47-47cd-bb48-61bc3bc10ae6", "id": "AOAI.21", "severity": "Medium", @@ -215,6 +236,7 @@ "subcategory": "Elasticity segregation", "text": "Estimate elasticity demands to determine synchronous and batch request segregation based on priority. For high priority, use synchronous approach and for low priority, asynchronous batch processing with queue is preferred", "waf": "Performance", + "service": "Azure OpenAI", "guid": "6e25d4d5-a3ae-4c2c-9e24-36b0336cb45e", "id": "AOAI.22", "severity": "Medium", @@ -225,6 +247,7 @@ "subcategory": "Benchmarking", "text": "Benchmark token consumption requirements based on estimated demands from consumers. Consider using the Azure OpenAI benchmarking tool to help you validate the throughput if you are using Provisioned Throughput Unit deployments", "waf": "Performance", + "service": "Azure OpenAI", "guid": "5bda4332-4f24-4811-9331-82ba51752694", "id": "AOAI.23", "severity": "High", @@ -235,6 +258,7 @@ "subcategory": "Elasticity ", "text": "If you are using Provisioned Throughput Units (PTUs), consider deploying a token-per-minute (TPM) deployment for overflow requests. Use a gateway to route requests to the TPM deployment when the PTU limits are reached.", "waf": "Performance", + "service": "Azure OpenAI", "guid": "4008ae7d-7e47-4432-96d8-bdcf55bce619", "id": "AOAI.24", "severity": "Medium", @@ -245,6 +269,7 @@ "subcategory": "Model choice", "text": "Choose the right model for the right task. Pick models with right tradeoff between speed, quality of response and output complexity", "waf": "Performance", + "service": "Azure OpenAI", "guid": "e8a13f98-8794-424d-9267-86d60b96c97b", "id": "AOAI.25", "severity": "High", @@ -255,6 +280,7 @@ "subcategory": "Fine tuning", "text": "Have a baseline for performance without fine-tuning for knowing whether or not fine-tuning has improved model performance", "waf": "Performance", + "service": "Azure OpenAI", "guid": "e9951904-8384-45c9-a6cb-2912156a1147", "id": "AOAI.26", "severity": "Medium", @@ -265,6 +291,7 @@ "subcategory": "Multi-region architecture", "text": "Deploy multiple OAI instances across regions", "waf": "Reliability", + "service": "Azure OpenAI", "guid": "5e39f541-accc-4d97-a376-bcdb3750ab2a", "id": "AOAI.27", "severity": "Low", @@ -275,6 +302,7 @@ "subcategory": "Load balancing", "text": "Implement retry & healthchecks with Gateway pattern like APIM", "waf": "Reliability", + "service": "Azure OpenAI", "guid": "b039da6d-55d7-4c89-8adb-107d5325af62", "id": "AOAI.28", "severity": "High", @@ -285,6 +313,7 @@ "subcategory": "Quotas", "text": "Ensure having adequate quotas of TPM & RPM for the workload", "waf": "Reliability", + "service": "Azure OpenAI", "guid": "5ca44e46-85e2-4223-ace8-bb12308ca5f1", "id": "AOAI.29", "severity": "Medium", @@ -295,6 +324,7 @@ "subcategory": "Load balancing", "text": "Deploy separate fine tuned models across regions if finetuning is employed", "waf": "Reliability", + "service": "Azure OpenAI", "guid": "7f154e3a-a369-4282-ae7e-316183687a04", "id": "AOAI.30", "severity": "Medium", @@ -305,6 +335,7 @@ "subcategory": "Data Backup and Disaster Recovery", "text": "Regularly backup and replicate critical data to ensure data availability and recoverability in case of data loss or system failures. Leverage Azure's backup and disaster recovery services to protect your data.", "waf": "Reliability", + "service": "Azure OpenAI", "guid": "77a1f893-5bda-4433-84f2-4811633182ba", "id": "AOAI.31", "severity": "Medium", @@ -315,6 +346,7 @@ "subcategory": "SLA considerations", "text": "Azure AI search service tiers should be choosen to have a SLA ", "waf": "Reliability", + "service": "Azure OpenAI", "guid": "95b96ad8-844c-4e3b-8b38-b876ba2cf204", "id": "AOAI.32", "severity": "High", @@ -325,6 +357,7 @@ "subcategory": "Data Sensitivity", "text": "Classify data and sensitivity, labeling with Microsoft Purview before generating the embeddings and make sure to treat the embeddings generated with same sensitivity and classification", "waf": "Security", + "service": "Azure OpenAI", "guid": "99013a5d-3ce4-474d-acbd-8682a6abca2a", "id": "AOAI.33", "severity": "Low", @@ -335,6 +368,7 @@ "subcategory": "Encryption at Rest", "text": "Encrypt data used for RAG with SSE/Disk encryption with optional BYOK", "waf": "Security", + "service": "Azure OpenAI", "guid": "4fda1dbf-3dd9-45d4-ac7c-891dca1f6d56", "id": "AOAI.34", "severity": "High", @@ -345,6 +379,7 @@ "subcategory": "Transit Encryption", "text": "Ensure TLS is enforced for data in transit across data sources, AI search used for Retrieval-Augmented Generation (RAG) and LLM communication", "waf": "Security", + "service": "Azure OpenAI", "guid": "59ae558b-937d-4498-9e11-12dbd7ba012f", "id": "AOAI.35", "severity": "High", @@ -355,6 +390,7 @@ "subcategory": "Access Control", "text": "Use RBAC to manage access to Azure OpenAI services. Assign appropriate permissions to users and restrict access based on their roles and responsibilities", "waf": "Security", + "service": "Azure OpenAI", "guid": "7b94ef6e-047d-42ea-8992-b1cd6e2054b2", "id": "AOAI.36", "severity": "High", @@ -365,6 +401,7 @@ "subcategory": "Data Masking and Redaction", "text": "Implement data encryption, masking or redaction techniques to hide sensitive data or replace it with obfuscated values in non-production environments or when sharing data for testing or troubleshooting purposes", "waf": "Security", + "service": "Azure OpenAI", "guid": "9769e4a6-91e8-4838-ac93-6667e13c0056", "id": "AOAI.37", "severity": "Medium", @@ -375,6 +412,7 @@ "subcategory": "Threat Detection and Monitoring", "text": "Utilize Azure Defender to detect and respond to security threats and set up monitoring and alerting mechanisms to identify suspicious activities or breaches. Leverage Azure Sentinel for advanced threat detection and response", "waf": "Security", + "service": "Azure OpenAI", "guid": "74b1e945-b459-4837-be7a-d6c6d3b375a5", "id": "AOAI.38", "severity": "High", @@ -385,6 +423,7 @@ "subcategory": "Data Retention and Disposal", "text": "Establish data retention and disposal policies to adhere to compliance regulations. Implement secure deletion methods for data that is no longer required and maintain an audit trail of data retention and disposal activities", "waf": "Security", + "service": "Azure OpenAI", "guid": "c7acbe48-abe5-44cd-99f2-e87768468c55", "id": "AOAI.39", "severity": "Medium", @@ -395,6 +434,7 @@ "subcategory": "Data Privacy and Compliance", "text": "Ensure compliance with relevant data protection regulations, such as GDPR or HIPAA, by implementing privacy controls and obtaining necessary consents or permissions for data processing activities.", "waf": "Security", + "service": "Azure OpenAI", "guid": "a775c6ee-95b9-46ad-a844-ce3b2b38b876", "id": "AOAI.40", "severity": "High", @@ -405,6 +445,7 @@ "subcategory": "Employee Awareness and Training", "text": "Educate your employees about data security best practices, the importance of handling data securely, and potential risks associated with data breaches. Encourage them to follow data security protocols diligently.", "waf": "Security", + "service": "Azure OpenAI", "guid": "ba2cf204-9901-43a5-b3ce-474dccbd8682", "id": "AOAI.41", "severity": "Medium" @@ -414,6 +455,7 @@ "subcategory": "Environment segregation", "text": "Keep production data separate from development and testing data. Only use real sensitive data in production and utilize anonymized or synthetic data in development and test environments.", "waf": "Security", + "service": "Azure OpenAI", "guid": "eae01e6e-842e-452f-9721-d928c1b1cd52", "id": "AOAI.42", "severity": "High" @@ -423,6 +465,7 @@ "subcategory": "Index Segregation", "text": "If you have varying levels of data sensitivity, consider creating separate indexes for each level. For instance, you could have one index for general data and another for sensitive data, each governed by different access protocols", "waf": "Security", + "service": "Azure OpenAI", "guid": "1e54a29a-9de3-499c-bd7b-28dc93555620", "id": "AOAI.43", "severity": "Medium" @@ -432,6 +475,7 @@ "subcategory": "Sensitive Data in Separate Instances", "text": "Take segregation a step further by placing sensitive datasets in different instances of the service. Each instance can be controlled with its own specific set of RBAC policies", "waf": "Security", + "service": "Azure OpenAI", "guid": "2bfe4564-b0d8-434a-948b-263e6dd60512", "id": "AOAI.44", "severity": "Medium" @@ -441,6 +485,7 @@ "subcategory": "Embedding and Vector handling", "text": "Recognize that embeddings and vectors generated from sensitive information are themselves sensitive. This data should be afforded the same protective measures as the source material", "waf": "Security", + "service": "Azure OpenAI", "guid": "a36498f6-dbad-438e-ad53-cc7ce1d7aaab", "id": "AOAI.45", "severity": "High" @@ -450,6 +495,7 @@ "subcategory": "Access control", "text": "Apply RBAC to th data stores having embeddings and vectors and scope access based on role's access requirements", "waf": "Security", + "service": "Azure OpenAI", "guid": "3571449a-b805-43d8-af89-dc7b33be2a1a", "id": "AOAI.46", "severity": "High", @@ -460,6 +506,7 @@ "subcategory": "Network security", "text": "Configure private endpoint for AI services to restrict service access within your network", "waf": "Security", + "service": "Azure OpenAI", "guid": "27f7b9e9-1be1-4f38-aef3-9812bd463cbb", "id": "AOAI.47", "severity": "High", @@ -470,6 +517,7 @@ "subcategory": "Network security", "text": "Enforce strict inbound and outbound traffic control with Azure Firewall and UDRs and limit the external integration points", "waf": "Security", + "service": "Azure OpenAI", "guid": "ac8ac199-ebb9-41a3-9d90-cae2cc881370", "id": "AOAI.48", "severity": "High" @@ -479,6 +527,7 @@ "subcategory": "Control Network Access", "text": "Implement network segmentation and access controls to restrict access to the LLM application only to authorized users and systems and prevent lateral movement", "waf": "Security", + "service": "Azure OpenAI", "guid": "6f7c0cba-fe51-4464-add4-57e927138b82", "id": "AOAI.49", "severity": "High" @@ -488,6 +537,7 @@ "subcategory": "Secure APIs and Endpoints", "text": "Ensure that APIs and endpoints used by the LLM application are properly secured with authentication and authorization mechanisms, such as Managed identities, API keys or OAuth, to prevent unauthorized access.", "waf": "Security", + "service": "Azure OpenAI", "guid": "1102cac6-eae0-41e6-b842-e52f4721d928", "id": "AOAI.50", "severity": "High", @@ -498,6 +548,7 @@ "subcategory": "Implement Strong Authentication", "text": "Enforce strong end user authentication mechanisms, such as multi-factor authentication, to prevent unauthorized access to the LLM application and associated network resources", "waf": "Security", + "service": "Azure OpenAI", "guid": "c1b1cd52-1e54-4a29-a9de-399cfd7b28dc", "id": "AOAI.51", "severity": "Medium", @@ -508,6 +559,7 @@ "subcategory": "Use Network Monitoring", "text": "Implement network monitoring tools to detect and analyze network traffic for any suspicious or malicious activities. Enable logging to capture network events and facilitate forensic analysis in case of security incidents", "waf": "Security", + "service": "Azure OpenAI", "guid": "93555620-2bfe-4456-9b0d-834a348b263e", "id": "AOAI.52", "severity": "Medium" @@ -517,6 +569,7 @@ "subcategory": "Security Audits and Penetration Testing", "text": "Conduct security audits and penetration testing to identify and address any network security weaknesses or vulnerabilities in the LLM application's network infrastructure", "waf": "Security", + "service": "Azure OpenAI", "guid": "6dd60512-a364-498f-9dba-d38ead53cc7c", "id": "AOAI.53", "severity": "Medium" @@ -528,6 +581,7 @@ "waf": "Operational Excellence", "guid": "e1d7aaab-3571-4449-ab80-53d89f89dc7b", "id": "AOAI.54", + "service": "Azure OpenAI", "severity": "Low", "link": "https://learn.microsoft.com/azure/azure-resource-manager/management/tag-resources?tabs=json" }, @@ -536,6 +590,7 @@ "subcategory": "Infrastructure Deployment", "text": "Azure AI Service accounts follows organizational naming conventions", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "77036e5e-6b4b-4ed3-b503-547c1347dc56", "id": "AOAI.55", "severity": "Low", @@ -546,6 +601,7 @@ "subcategory": "Diagnostics Logging", "text": "Diagnostic logs in Azure AI services resources should be enabled", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "028a71ff-e1ce-415d-b3f0-d5e772d41e36", "id": "AOAI.56", "severity": "High", @@ -556,6 +612,7 @@ "subcategory": "Entra ID based access", "text": "Key access (local authentication) is recommended to be disabled for security. After disabling key based access, Microsoft Entra ID becomes the only access method, which allows maintaining minimum privilege principle and granular control. ", "waf": "Security", + "service": "Azure OpenAI", "guid": "11cc57b4-a4b1-4410-b439-58a8c2289b3d", "id": "AOAI.57", "severity": "High", @@ -566,6 +623,7 @@ "subcategory": "Secure Key Management", "text": "Store and manage keys securely using Azure Key Vault. Avoid hard-coding or embedding sensitive keys within your LLM application's code and retrieve them securely from Azure Key Vault using managed identities", "waf": "Security", + "service": "Azure OpenAI", "guid": "6b57cfc6-5546-41e1-a3e3-453a3c863964", "id": "AOAI.58", "severity": "High", @@ -576,6 +634,7 @@ "subcategory": "Key Rotation and Expiration", "text": "Regularly rotate and expire keys stored in Azure Key Vault to minimize the risk of unauthorized access.", "waf": "Security", + "service": "Azure OpenAI", "guid": "8b652d6c-15f5-4129-9539-8e6ded227dd1", "id": "AOAI.59", "severity": "High", @@ -586,6 +645,7 @@ "subcategory": "Secure coding practice", "text": "Follow secure coding practices to prevent common vulnerabilities such as injection attacks, cross-site scripting (XSS), or security misconfigurations", "waf": "Security", + "service": "Azure OpenAI", "guid": "42b06c21-d799-49a6-96f4-389a7f42c78e", "id": "AOAI.60", "severity": "High", @@ -596,6 +656,7 @@ "subcategory": "Patching and updates", "text": "Setup a process to regularly update and patch the LLM libraries and other system components", "waf": "Security", + "service": "Azure OpenAI", "guid": "78c06a73-a22a-4495-9e6a-8dc4a20e27c3", "id": "AOAI.61", "severity": "High", @@ -606,6 +667,7 @@ "subcategory": "Governance", "text": "Adhere to Azure OpenAI or other LLMs terms of use, policies and guidance and allowed use cases", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "e29711b1-352b-4eee-879b-588defc4972c", "id": "AOAI.62", "severity": "High", @@ -616,6 +678,7 @@ "subcategory": "Cost familiarization", "text": "Understand difference in cost of base models and fine tuned models and token step sizes", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "d3cd21bf-7703-46e5-b6b4-bed3d503547c", "id": "AOAI.63", "severity": "Medium", @@ -626,6 +689,7 @@ "subcategory": "Batch processing", "text": "Batch requests, where possible, to minimize the per-call overhead which can reduce overall costs. Ensure you optimize batch size", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "1347dc56-028a-471f-be1c-e15dd3f0d5e7", "id": "AOAI.64", "severity": "High", @@ -636,6 +700,7 @@ "subcategory": "Cost monitoring", "text": "Set up a cost tracking system that monitors model usage and use that information to help inform model choices and prompt sizes", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "72d41e36-11cc-457b-9a4b-1410d43958a8", "id": "AOAI.65", "severity": "Medium", @@ -646,6 +711,7 @@ "subcategory": "Token limit", "text": "Set a maximum limit on the number of tokens per model response. Optimize the size to ensure it is large enough for a valid response", "waf": "Cost Optimization", + "service": "Azure OpenAI", "guid": "166cd072-af9b-4141-a898-a535e737897e", "id": "AOAI.66", "severity": "Medium", @@ -656,6 +722,7 @@ "subcategory": "AI Search Reliability", "text": "Review the guidance provided on setting up AI search for Reliability", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "71ca7da8-cfa9-462a-8594-946da97dc3a2", "id": "AOAI.67", "severity": "Medium", @@ -666,6 +733,7 @@ "subcategory": "AI Search Vector Limits", "text": "Plan and manage AI Search Vector storage", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "3266b225-86f4-4a16-92bd-ddea8a487cde", "id": "AOAI.68", "severity": "Medium", @@ -676,6 +744,7 @@ "subcategory": "DevOps", "text": "Apply LLMOps practices to automate the lifecycle management of your GenAI applications", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "b4861bc3-bc14-4aeb-9e66-e8d9a3aec218", "id": "AOAI.69", "severity": "Medium", @@ -686,6 +755,7 @@ "subcategory": "DevOps", "text": "Evaluate the quality of prompts and applications when switching between model versions", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "e6436b07-36db-455f-9796-03334bdf9cc2", "id": "AOAI.70", "severity": "Medium", @@ -696,6 +766,7 @@ "subcategory": "Development", "text": "Evaluate, monitor and refine your GenAI apps for features like groundedness, relevance, accuracy, coherence, fluency, �", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "3418db61-2712-4650-9bb4-7a393a080327", "id": "AOAI.71", "severity": "Medium", @@ -706,6 +777,7 @@ "subcategory": "Development", "text": "Evaluate your Azure AI Search results based on different search parameters", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "294798b1-578b-4219-a46c-eb5443513592", "id": "AOAI.72", "severity": "Medium" @@ -715,6 +787,7 @@ "subcategory": "Development", "text": "Look at fine tuning models as way of increasing accuracy only when you have tried other basic approaches like prompt engineering and RAG with your data", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "2744293b-b628-4537-a551-19b08e8f5854", "id": "AOAI.73", "severity": "Medium", @@ -725,6 +798,7 @@ "subcategory": "Development", "text": "Use prompt engineering techniques to improve the accuracy of LLM responses", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "287d9cec-166c-4d07-8af9-b141a898a535", "id": "AOAI.74", "severity": "Medium", @@ -735,6 +809,7 @@ "subcategory": "Security Audits and Penetration Testing", "text": "Red team your GenAI applications", "waf": "Security", + "service": "Azure OpenAI", "guid": "e737897e-71ca-47da-acfa-962a1594946d", "id": "AOAI.75", "severity": "Medium", @@ -745,6 +820,7 @@ "subcategory": "End user feedback", "text": "Provide end users with scoring options for LLM responses and track these scores. ", "waf": "Operational Excellence", + "service": "Azure OpenAI", "guid": "edb117e6-76aa-4f66-aca4-8e5a95f2223e", "id": "AOAI.76", "severity": "Medium",