Testing Strategies for AI Applications

Introduction

Testing AI applications differs fundamentally from testing deterministic software. Model outputs are probabilistic, edge cases are infinite, and a passing unit test does not guarantee correct behavior in production. A comprehensive AI testing strategy combines traditional software testing with AI-specific evaluation methodologies to catch regressions, hallucinations, and performance degradations before they reach users.

Evaluation Datasets

Curate high-quality evaluation datasets that reflect real-world usage:

from dataclasses import dataclass

from typing import List, Callable

import json

@dataclass

class TestCase:

id: str

input: str

expected_output: str

domain: str

difficulty: str # "easy", "medium", "hard"

tags: List[str]

# For non-deterministic evaluation

criteria: List[Callable[[str], bool]]

class EvalDataset:

def __init__(self, name: str):

self.name = name

self.test_cases: List[TestCase] = []

def add_golden_set(self, path: str):

"""Load curated golden test cases."""

with open(path) as f:

data = json.load(f)

for item in data:

self.test_cases.append(TestCase(

id=item["id"],

input=item["input"],

expected_output=item["expected_output"],

domain=item.get("domain", "general"),

difficulty=item.get("difficulty", "medium"),

tags=item.get("tags", []),

criteria=[

lambda output, expected=item["expected_output"]:

expected.lower() in output.lower(),

],

))

def add_adversarial(self, path: str):

"""Load adversarial test cases (edge cases, jailbreaks)."""

with open(path) as f:

data = json.load(f)

for item in data:

self.test_cases.append(TestCase(

id=f"adv_{item['id']}",

input=item["input"],

expected_output="",

domain="adversarial",

difficulty="hard",

tags=["adversarial"],

criteria=[

lambda output:

"I cannot" in output or "I'm unable" in output,

],

))

Regression Testing

Automated regression testing catches model behavior changes:

class ModelRegressionTest:

def __init__(self, eval_dataset: EvalDataset):

self.dataset = eval_dataset

self.results_history: List[dict] = []

async def run_regression_suite(

self,

model_name: str,

previous_results: dict = None,

) -> dict:

results = {

"model": model_name,

"timestamp": datetime.utcnow().isoformat(),

"total": len(self.dataset.test_cases),

"passed": 0,

"failed": 0,

"failures": [],

"score": 0.0,

}

for test_case in self.dataset.test_cases:

try:

output = await self._invoke_model(model_name, test_case.input)

# Evaluate against all criteria

passed = all(

criterion(output) for criterion in test_case.criteria

)

if passed:

results["passed"] += 1

else:

results["failed"] += 1

results["failures"].append({

"id": test_case.id,

"input": test_case.input,

"expected": test_case.expected_output,

"actual": output,

"domain": test_case.domain,

})

except Exception as e:

results["failed"] += 1

results["failures"].append({

"id": test_case.id,

"error": str(e),

})

results["score"] = results["passed"] / results["total"]

# Compare with previous run

if previous_results:

score_delta = results["score"] - previous_results["score"]

results["regression"] = score_delta < -0.02

results["score_delta"] = score_delta

return results

def fail_pipeline_if_regression(self, results: dict):

"""Fail CI if model regressed beyond threshold."""

if results.get("regression", False):

raise Exception(

f"Model regression detected! "

f"Score dropped from "

f"{results.get('previous_score', 1.0):.2%} to "

f"{results['score']:.2%}"

)

A/B Evaluation

Compare model versions side by side with structured evaluation:

class ABEvaluation:

def __init__(self, judge_model: str = "claude-opus-4-20260512"):

self.judge = judge_model

async def evaluate_pair(

self,

prompt: str,

output_a: str,

output_b: str,

criteria: List[str],

) -> dict:

"""Use an independent judge model to compare outputs."""

evaluation_prompt = f"""

Compare these two AI responses to the same prompt.

Prompt: "{prompt}"

Response A: "{output_a}"

Response B: "{output_b}"

Evaluate on these criteria: {', '.join(criteria)}

For each criterion, state which response is better (A, B, or tie)

and provide a brief justification.

"""

response = client.messages.create(

model=self.judge,

max_tokens=1024,

messages=[{"role": "user", "content": evaluation_prompt}],

)

return self._parse_evaluation(response.content[0].text)

async def batch_evaluate(

self,

prompts: List[str],

model_a: str,

model_b: str,

criteria: List[str],

) -> dict:

results = {"model_a_wins": 0, "model_b_wins": 0, "ties": 0}

for prompt in prompts:

output_a = await self._invoke(model_a, prompt)

output_b = await self._invoke(model_b, prompt)

evaluation = await self.evaluate_pair(

prompt, output_a, output_b, criteria

)

# Aggregate across criteria

winner = evaluation["winner"]

results[f"{winner}_wins"] += 1

results["total"] = len(prompts)

results["a_win_rate"] = results["model_a_wins"] / results["total"]

results["b_win_rate"] = results["model_b_wins"] / results["total"]

return results

Hallucination Detection

Automated hallucination checks verify factual accuracy:

class HallucinationDetector:

def __init__(self, knowledge_base: Callable):

self.kb = knowledge_base

async def check_factual_claims(self, output: str) -> List[dict]:

"""Extract factual claims and verify them against a knowledge base."""

# 1. Extract atomic claims

claims = await self._extract_claims(output)

verified_claims = []

for claim in claims:

# 2. Search knowledge base

evidence = await self.kb.search(claim["text"])

# 3. Verify claim against evidence

verification = await self._verify_claim(

claim["text"],

evidence,

claim["context"],

)

verified_claims.append({

"claim": claim["text"],

"confidence": verification["confidence"],

"supported": verification["supported"],

"evidence": evidence[:3],

"context": claim["context"],

})

return verified_claims

async def _verify_claim(

self,

claim: str,

evidence: List[str],

context: str,

) -> dict:

prompt = f"""

Claim: "{claim}"

Context: "{context}"

Evidence: {' '.join(evidence[:3])}

Is this claim supported by the evidence?

Respond with JSON:

{{"supported": bool, "confidence": 0.0-1.0, "reasoning": "..."}}

"""

response = await self._llm_call(prompt)

return json.loads(response)

def compute_hallucination_rate(

self, verified_claims: List[dict]

) -> float:

unsupported = sum(

1 for c in verified_claims if not c["supported"]

)

total = len(verified_claims)

return unsupported / total if total > 0 else 0.0

Prompt Testing

Version-control prompts with structured testing:

class PromptRegistry:

def __init__(self):

self.prompts = {}

def register(

self,

name: str,

template: str,

version: str,

tests: List[Callable] = None,

):

self.prompts[name] = {

"template": template,

"version": version,

"tests": tests or [],

"performance": [],

}

async def test_prompt(

self, name: str, test_cases: List[dict]

) -> dict:

prompt = self.prompts[name]

results = []

for case in test_cases:

# Render template with test inputs

rendered = prompt["template"].format(**case["inputs"])

output = await self._invoke(rendered)

# Run tests

test_results = [

test(output) for test in prompt["tests"]

]

results.append({

"case": case["name"],

"output": output,

"tests_passed": all(test_results),

"test_details": test_results,

})

return {

"prompt": name,

"version": prompt["version"],

"pass_rate": sum(r["tests_passed"] for r in results) / len(results),

"results": results,

}

Performance Testing

Benchmark latency, throughput, and cost across model versions:

class AIPerformanceTest:

async def benchmark(

self,

model: str,

concurrency: int = 10,

requests: int = 100,

) -> dict:

import time

import asyncio

semaphore = asyncio.Semaphore(concurrency)

latencies = []

async def single_request():

async with semaphore:

start = time.monotonic()

await self._invoke_model(model, "Test prompt")

latencies.append(time.monotonic() - start)

tasks = [single_request() for _ in range(requests)]

await asyncio.gather(*tasks)

latencies.sort()

return {

"model": model,

"p50": latencies[len(latencies) // 2],

"p95": latencies[int(len(latencies) * 0.95)],

"p99": latencies[int(len(latencies) * 0.99)],

"avg": sum(latencies) / len(latencies),

"throughput": requests / sum(latencies),

"total_cost": self._calculate_cost(model, requests),

}

A mature AI testing pipeline runs golden set regression on every PR, A/B evaluations before model upgrades, hallucination detection on every production response, and performance benchmarks weekly. No single metric captures model quality; combine automated tests with human evaluation for production releases.