/azure-ai-projects-py

Custom Evaluators Reference

Create custom evaluators when built-in evaluators don't meet your needs using the azure-ai-projects SDK.

Evaluator Types

Code-Based Evaluators

Use Python code for deterministic evaluation logic.

Basic Code Evaluator

from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import (
    EvaluatorVersion,
    EvaluatorCategory,
    EvaluatorType,
    CodeBasedEvaluatorDefinition,
    EvaluatorMetric,
    EvaluatorMetricType,
    EvaluatorMetricDirection,
)
from azure.identity import DefaultAzureCredential
import os

endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]

with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
):
    evaluator = project_client.evaluators.create_version(
        name="word_count_evaluator",
        evaluator_version=EvaluatorVersion(
            evaluator_type=EvaluatorType.CUSTOM,
            categories=[EvaluatorCategory.QUALITY],
            display_name="Word Count",
            description="Counts words in response and checks for conciseness",
            definition=CodeBasedEvaluatorDefinition(
                code_text='''
def grade(sample, item) -> dict:
    response = item.get("response", "")
    word_count = len(response.split())
    return {
        "word_count": word_count,
        "is_concise": word_count < 100
    }
''',
                data_schema={
                    "type": "object",
                    "properties": {
                        "response": {"type": "string"}
                    },
                    "required": ["response"]
                },
                metrics={
                    "word_count": EvaluatorMetric(
                        type=EvaluatorMetricType.ORDINAL,
                        desirable_direction=EvaluatorMetricDirection.DECREASE,
                        min_value=0,
                        max_value=10000,
                    ),
                    "is_concise": EvaluatorMetric(
                        type=EvaluatorMetricType.BINARY,
                    ),
                },
            ),
        ),
    )
    print(f"Created evaluator: {evaluator.name} (version {evaluator.version})")

Code Evaluator: Keyword Checker

evaluator = project_client.evaluators.create_version(
    name="disclaimer_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Disclaimer Checker",
        description="Verifies required disclaimers are present in response",
        definition=CodeBasedEvaluatorDefinition(
            code_text='''
def grade(sample, item) -> dict:
    response = item.get("response", "").lower()
    required_keywords = ["disclaimer", "not financial advice", "consult a professional"]
    
    found = [kw for kw in required_keywords if kw in response]
    missing = [kw for kw in required_keywords if kw not in response]
    
    score = len(found) / len(required_keywords) if required_keywords else 1.0
    
    return {
        "compliance_score": score,
        "missing_disclaimers": ", ".join(missing) if missing else "none",
        "passes": score >= 0.8
    }
''',
            data_schema={
                "type": "object",
                "properties": {"response": {"type": "string"}},
                "required": ["response"]
            },
            metrics={
                "compliance_score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    desirable_direction=EvaluatorMetricDirection.INCREASE,
                    min_value=0.0,
                    max_value=1.0,
                ),
                "passes": EvaluatorMetric(type=EvaluatorMetricType.BINARY),
            },
        ),
    ),
)

Code Evaluator: JSON Format Validator

evaluator = project_client.evaluators.create_version(
    name="json_format_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="JSON Format Validator",
        description="Checks if response is valid JSON with required fields",
        definition=CodeBasedEvaluatorDefinition(
            code_text='''
import json

def grade(sample, item) -> dict:
    response = item.get("response", "")
    required_fields = item.get("required_fields", [])
    
    try:
        parsed = json.loads(response)
        is_valid_json = True
        
        if required_fields:
            missing = [f for f in required_fields if f not in parsed]
            has_required_fields = len(missing) == 0
        else:
            has_required_fields = True
            missing = []
            
    except json.JSONDecodeError:
        is_valid_json = False
        has_required_fields = False
        missing = required_fields
    
    return {
        "is_valid_json": is_valid_json,
        "has_required_fields": has_required_fields,
        "missing_fields": ", ".join(missing) if missing else "none"
    }
''',
            data_schema={
                "type": "object",
                "properties": {
                    "response": {"type": "string"},
                    "required_fields": {"type": "array", "items": {"type": "string"}}
                },
                "required": ["response"]
            },
            metrics={
                "is_valid_json": EvaluatorMetric(type=EvaluatorMetricType.BINARY),
                "has_required_fields": EvaluatorMetric(type=EvaluatorMetricType.BINARY),
            },
        ),
    ),
)

Prompt-Based Evaluators

Use LLM judgment for subjective evaluation.

Basic Prompt Evaluator

from azure.ai.projects.models import PromptBasedEvaluatorDefinition

evaluator = project_client.evaluators.create_version(
    name="helpfulness_evaluator",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Helpfulness Evaluator",
        description="Evaluates how helpful the response is to the user",
        definition=PromptBasedEvaluatorDefinition(
            prompt_text='''
You are an expert evaluator. Rate the helpfulness of the AI assistant's response.

Query: {query}
Response: {response}

Scoring (1-5):
1 = Not helpful at all, doesn't address the query
2 = Slightly helpful, partially addresses the query
3 = Moderately helpful, addresses most of the query
4 = Very helpful, fully addresses the query
5 = Extremely helpful, exceeds expectations

Return ONLY valid JSON: {"score": <1-5>, "reason": "<brief explanation>"}
''',
            init_parameters={
                "type": "object",
                "properties": {
                    "deployment_name": {"type": "string", "description": "Model deployment name"}
                },
                "required": ["deployment_name"]
            },
            data_schema={
                "type": "object",
                "properties": {
                    "query": {"type": "string"},
                    "response": {"type": "string"}
                },
                "required": ["query", "response"]
            },
            metrics={
                "score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    desirable_direction=EvaluatorMetricDirection.INCREASE,
                    min_value=1,
                    max_value=5,
                ),
            },
        ),
    ),
)

Prompt Evaluator: Brand Tone Checker

evaluator = project_client.evaluators.create_version(
    name="brand_tone_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Brand Tone Checker",
        description="Evaluates if response matches company brand voice guidelines",
        definition=PromptBasedEvaluatorDefinition(
            prompt_text='''
You are evaluating if an AI assistant's response matches brand voice guidelines.

Brand Guidelines:
- Professional but friendly
- Avoid jargon, use simple language
- Always offer next steps or additional help
- Never use negative language about competitors
- End with a helpful call-to-action

Response to evaluate:
{response}

Score the response from 1-5:
5 = Perfectly matches brand voice
4 = Mostly matches, minor issues
3 = Partially matches
2 = Significant tone issues
1 = Does not match brand voice

Return ONLY valid JSON: {"score": <1-5>, "reason": "<brief explanation>", "suggestions": "<improvement suggestions>"}
''',
            init_parameters={
                "type": "object",
                "properties": {"deployment_name": {"type": "string"}},
                "required": ["deployment_name"]
            },
            data_schema={
                "type": "object",
                "properties": {"response": {"type": "string"}},
                "required": ["response"]
            },
            metrics={
                "score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    min_value=1,
                    max_value=5,
                ),
            },
        ),
    ),
)

Prompt Evaluator: Factual Accuracy

evaluator = project_client.evaluators.create_version(
    name="factual_accuracy_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Factual Accuracy",
        description="Checks if response claims are supported by context",
        definition=PromptBasedEvaluatorDefinition(
            prompt_text='''
Evaluate whether the response contains only facts supported by the provided context.

Context (source of truth):
{context}

Response to evaluate:
{response}

Analysis steps:
1. Identify each factual claim in the response
2. Check if each claim is supported by the context
3. Note any unsupported or fabricated claims

Scoring (1-5):
1 = Mostly fabricated or incorrect
2 = Many unsupported claims
3 = Mixed: some facts but notable errors
4 = Mostly factual, minor issues
5 = Fully factual, no unsupported claims

Return ONLY valid JSON: {"score": <1-5>, "reason": "<explanation>", "unsupported_claims": ["<list of unsupported claims>"]}
''',
            init_parameters={
                "type": "object",
                "properties": {"deployment_name": {"type": "string"}},
                "required": ["deployment_name"]
            },
            data_schema={
                "type": "object",
                "properties": {
                    "context": {"type": "string"},
                    "response": {"type": "string"}
                },
                "required": ["context", "response"]
            },
            metrics={
                "score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    min_value=1,
                    max_value=5,
                ),
            },
        ),
    ),
)

Using Custom Evaluators

In Testing Criteria

testing_criteria = [
    # Built-in evaluator
    {
        "type": "azure_ai_evaluator",
        "name": "coherence",
        "evaluator_name": "builtin.coherence",
        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
        "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
    },
    # Custom code-based evaluator
    {
        "type": "azure_ai_evaluator",
        "name": "word_count",
        "evaluator_name": "word_count_evaluator",
        "data_mapping": {"response": "{{item.response}}"}
    },
    # Custom prompt-based evaluator
    {
        "type": "azure_ai_evaluator",
        "name": "helpfulness",
        "evaluator_name": "helpfulness_evaluator",
        "initialization_parameters": {"deployment_name": "gpt-4o-mini"},
        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}
    },
]

eval_object = openai_client.evals.create(
    name="Mixed Evaluators Test",
    data_source_config=data_source_config,
    testing_criteria=testing_criteria,
)

Managing Custom Evaluators

List Custom Evaluators

evaluators = project_client.evaluators.list_latest_versions(type="custom")
for e in evaluators:
    print(f"{e.name} (v{e.version}): {e.display_name}")

Get Evaluator Details

evaluator = project_client.evaluators.get_version(
    name="helpfulness_evaluator",
    version="latest"
)
print(f"Data Schema: {evaluator.definition.data_schema}")
print(f"Metrics: {evaluator.definition.metrics}")

Update Evaluator

updated = project_client.evaluators.update_version(
    name="word_count_evaluator",
    version="1",
    evaluator_version={
        "description": "Updated description",
        "display_name": "Word Count v2",
    }
)

Delete Evaluator

project_client.evaluators.delete_version(
    name="word_count_evaluator",
    version="1"
)

Best Practices

1. Use code-based for deterministic logic - Pattern matching, format validation, keyword checking
2. Use prompt-based for subjective judgment - Quality assessment, tone evaluation, semantic analysis
3. Always define data_schema - Ensures correct data mapping
4. Define meaningful metrics - Use appropriate types (ORDINAL, BINARY)
5. Test before production - Run evaluator on sample data first
6. Version your evaluators - Create new versions instead of modifying existing ones

Related Documentation

• Custom Evaluators
• Code-based evaluator sample
• Prompt-based evaluator sample