Metrics and Scoring

function calculateToolAccuracy(events: Event[], expectedTools: string[]) {
  const actualTools = events
    .flatMap(e => e.getFunctionCalls())
    .map(fc => fc.name);

  const correctSelections = expectedTools.filter(tool =>
    actualTools.includes(tool)
  ).length;

  const accuracy = (correctSelections / expectedTools.length) * 100;
  console.log(`Tool selection accuracy: ${accuracy}%`);

  return {
    accuracy,
    expected: expectedTools,
    actual: actualTools,
    correct: correctSelections
  };
}

async function measureResponseTime(agent: LlmAgent, query: string) {
  const startTime = Date.now();

  let responseComplete = false;
  const events = [];

  for await (const event of agent.runAsync(context)) {
    events.push(event);
    if (event.isFinalResponse()) {
      responseComplete = true;
      break;
    }
  }

  const totalTime = Date.now() - startTime;
  const toolCalls = events.flatMap(e => e.getFunctionCalls()).length;

  return {
    totalTime,
    toolCallCount: toolCalls,
    avgTimePerTool: toolCalls > 0 ? totalTime / toolCalls : 0,
    responseComplete
  };
}

function assessResponseQuality(response: string, query: string) {
  const quality = {
    relevance: 0,
    completeness: 0,
    clarity: 0,
    accuracy: 0
  };

  // Relevance assessment
  if (responseAddressesQuery(response, query)) {
    quality.relevance = 1;
  }

  // Completeness check
  if (responseIsComplete(response, query)) {
    quality.completeness = 1;
  }

  // Clarity evaluation
  if (responseIsClear(response)) {
    quality.clarity = 1;
  }

  // Accuracy validation
  if (responseIsAccurate(response, query)) {
    quality.accuracy = 1;
  }

  const overallScore = Object.values(quality).reduce((a, b) => a + b) / 4;
  return { ...quality, overall: overallScore };
}

function analyzeCommunicationStyle(response: string) {
  return {
    tone: detectTone(response), // friendly, professional, helpful
    clarity: assessClarity(response), // clear, confusing, verbose
    appropriateness: checkAppropriatenessForContext(response),
    engagement: measureEngagementLevel(response)
  };
}

function detectTone(text: string): string {
  // Simple keyword-based analysis
  const friendlyWords = ['please', 'thank you', 'happy to help'];
  const professionalWords = ['regarding', 'furthermore', 'consequently'];

  if (friendlyWords.some(word => text.toLowerCase().includes(word))) {
    return 'friendly';
  }
  if (professionalWords.some(word => text.toLowerCase().includes(word))) {
    return 'professional';
  }
  return 'neutral';
}

interface EvaluationWeights {
  trajectory: number;      // 0.4 - How agents reach solutions
  accuracy: number;        // 0.3 - Correctness of responses
  relevance: number;       // 0.2 - Appropriateness to query
  efficiency: number;      // 0.1 - Speed and resource usage
}

function calculateCompositeScore(
  trajectoryScore: number,
  accuracyScore: number,
  relevanceScore: number,
  efficiencyScore: number,
  weights: EvaluationWeights = {
    trajectory: 0.4,
    accuracy: 0.3,
    relevance: 0.2,
    efficiency: 0.1
  }
): number {
  return (
    trajectoryScore * weights.trajectory +
    accuracyScore * weights.accuracy +
    relevanceScore * weights.relevance +
    efficiencyScore * weights.efficiency
  );
}

interface QualityThresholds {
  trajectory: number;      // Minimum 0.85
  accuracy: number;        // Minimum 0.90
  relevance: number;       // Minimum 0.75
  overall: number;         // Minimum 0.80
}

function evaluateAgentPerformance(
  scores: Record<string, number>,
  thresholds: QualityThresholds
): { pass: boolean; details: Record<string, boolean> } {
  const details = {
    trajectory: scores.trajectory >= thresholds.trajectory,
    accuracy: scores.accuracy >= thresholds.accuracy,
    relevance: scores.relevance >= thresholds.relevance,
    overall: scores.overall >= thresholds.overall
  };

  const pass = Object.values(details).every(Boolean);

  return { pass, details };
}

async function runMultipleEvaluations(
  agent: LlmAgent,
  query: string,
  runs: number = 5
) {
  const scores = [];

  for (let i = 0; i < runs; i++) {
    const result = await evaluateSingleRun(agent, query);
    scores.push(result.overallScore);
  }

  const mean = scores.reduce((a, b) => a + b) / scores.length;
  const variance = scores.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / scores.length;
  const stdDev = Math.sqrt(variance);

  return {
    mean,
    stdDev,
    confidence95: [mean - 1.96 * stdDev, mean + 1.96 * stdDev],
    allScores: scores
  };
}

interface PerformanceRecord {
  timestamp: Date;
  version: string;
  scores: Record<string, number>;
  testScenario: string;
}

function analyzePerformanceTrend(records: PerformanceRecord[]) {
  const byVersion = records.reduce((acc, record) => {
    if (!acc[record.version]) acc[record.version] = [];
    acc[record.version].push(record.scores.overall);
    return acc;
  }, {} as Record<string, number[]>);

  return Object.entries(byVersion).map(([version, scores]) => ({
    version,
    avgScore: scores.reduce((a, b) => a + b) / scores.length,
    minScore: Math.min(...scores),
    maxScore: Math.max(...scores),
    sampleSize: scores.length
  }));
}

{
  "criteria": {
    "tool_trajectory_avg_score": 1.0,
    "response_match_score": 0.8,
    "safety_v1": 1.0
  }
}


**Future Capabilities:**
- **Real-time Analytics**: Live performance dashboards
- **Comparative Analysis**: Benchmark against other agents
- **Predictive Metrics**: Early warning systems for performance degradation

**Integration Capabilities:**
- **CI/CD Pipeline Integration**: Automated quality gates
- **Performance Monitoring**: Production quality tracking
- **Alert Systems**: Threshold-based notifications
- **Trend Analysis**: Long-term performance patterns

## Best Practices

### Metric Selection

**Relevance**: Choose metrics that align with business objectives
**Balance**: Combine objective measurements with subjective assessment
**Actionability**: Select metrics that guide improvement efforts
**Consistency**: Maintain consistent measurement approaches over time

### Threshold Setting

**Baseline Establishment**: Use initial performance as baseline
**Gradual Improvement**: Incrementally raise quality standards
**Context Sensitivity**: Adjust thresholds for different use cases
**Stakeholder Alignment**: Ensure thresholds match business requirements

### Performance Monitoring

**Regular Assessment**: Continuous evaluation of key metrics
**Trend Analysis**: Monitor performance patterns over time
**Root Cause Analysis**: Investigate performance degradations
**Improvement Tracking**: Measure impact of optimization efforts