AI Assistance Evaluation Report

// Copyright 2026 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import * as fs from 'node:fs'; import * as path from 'node:path'; import type {Conversation} from '../types.js'; import { type BinaryStats, calculateStats, calculateWeightedScore, type JudgeStats, type Result, type ResultStore, type RougeStats, } from './evaluators.ts'; /** * Renders the results for BINARY type evaluations. */ function renderBinaryResults(result: Extract): string { const stats = calculateStats(result) as BinaryStats; let html = `

Pass Rate: ${stats.success} / ${stats.total}

`; html += result.details .map(detail => { const statusClass = detail.success ? 'tag-pass' : 'tag-fail'; const statusText = detail.success ? 'PASS' : 'FAIL'; return renderConversationCard( detail.conversation, 'Conversation', `${statusText}`, ''); }) .join(''); return html; } /** * Renders the results for JUDGE type evaluations. */ function renderJudgeResults(result: Extract): string { const stats = calculateStats(result) as JudgeStats; let html = `

Average Overall: ${stats.overallStats.average.toFixed(2)}

`; html += result.details .map( detail => { const ratingsHtml = `

Ratings

${detail.rubricScores.map(rubric => ` `).join('')}

Rubric	Score	Justification
${rubric.rubric}	${rubric.score}	${rubric.reason.replace(/\n/g, ' ')}

`; const overallScore = calculateWeightedScore(detail.rubricScores, result.rubricWeights); return renderConversationCard( detail.conversation, 'Conversation', `${overallScore.toFixed(2)}`, ratingsHtml); }) .join(''); return html; } /** * Renders the results for ROUGE type evaluations. */ function renderRougeResults(result: Extract): string { const stats = calculateStats(result) as RougeStats; let html = `

ROUGE-Lsum Average: ${stats.average.toFixed(2)}

`; html += result.details .map(detail => { const goldenHtml = `

Golden Response

${detail.goldenResponse}

`; return renderConversationCard( detail.conversation, 'Conversation', `${detail.score.toFixed(2)}`, goldenHtml); }) .join(''); return html; } /** * Registry of renderers for different result types. */ const RESULT_RENDERERS: {[K in Result['type']]: (result: Extract) => string} = { BINARY: renderBinaryResults, JUDGE: renderJudgeResults, ROUGE: renderRougeResults, }; /** * Renders a standard expandable card for a conversation. */ function renderConversationCard( conversation: Conversation, headerLabel: string, scoreElement: string, detailsHtml: string): string { return `

${headerLabel} ${conversation.id} ${scoreElement}

Model: ${conversation.model.id} (${conversation.model.version}) Chrome: ${conversation.chromeVersion}

${renderConversationTranscript(conversation)} ${detailsHtml}

`; } /** * Renders the transcript of a conversation. */ function renderConversationTranscript(conversation: Conversation): string { return conversation.queries .map(query => { let html = ''; if (query.request.prompt || query.request.functionCallResponse) { html += '

Request

'; if (query.request.prompt) { html += `

Query:\n${query.request.prompt}

`; } if (query.request.functionCallResponse) { html += `

Function Response:\n${query.request.functionCallResponse}

`; } } if (query.response.text || query.response.functionCallRequests?.length) { html += '

Response

'; if (query.response.text) { html += `

Explanation:\n${query.response.text}

`; } if (query.response.functionCallRequests?.length) { const calls = query.response.functionCallRequests.map(r => `${r.name}(${JSON.stringify(r.args, null, 2)})`) .join('\n\n'); html += `

Function Calls:\n${calls}

`; } } return html; }) .join(''); } /** * Renders the sidebar navigation. */ function renderSidebar(groups: Record, allStores: ResultStore[]): string { return Object.entries(groups) .map(([type, stores]) => `

${type}

${stores.map(store => { const idx = allStores.indexOf(store); return `

${store.label}

`; }).join('')}

`).join(''); } /** * Renders the main content area. */ function renderContent(allStores: ResultStore[]): string { return allStores .map((store, sIdx) => { let groupHtml = `

${store.type} / ${store.label}

`; for (const [test, dateToResult] of store.results) { groupHtml += `

${test}

`; const sortedDates = Array.from(dateToResult.keys()).sort(); groupHtml += renderSummaryTable(dateToResult, sortedDates); for (const date of sortedDates) { const result = dateToResult.get(date); if (!result) { continue; } groupHtml += `

Results for ${date}

`; const renderer = RESULT_RENDERERS[result.type] as (r: Result) => string; groupHtml += renderer(result); } } groupHtml += `

`; return groupHtml; }) .join(''); } export function generateReport(stores: ResultStore|ResultStore[]): void { const allStores = Array.isArray(stores) ? stores : [stores]; if (allStores.length === 0) { return; } const types = [...new Set(allStores.map(s => s.type))].sort(); const typesSuffix = types.length > 0 ? `-${types.join('-')}` : ''; const reportPath = path.join(process.cwd(), `eval-report${typesSuffix}.html`); const groups: Record = {}; for (const store of allStores) { groups[store.type] = groups[store.type] ?? []; groups[store.type].push(store); } const sidebarHtml = renderSidebar(groups, allStores); const contentHtml = renderContent(allStores); const finalHtml = ` AI Assistance Evaluation Report

${contentHtml}

`; fs.writeFileSync(reportPath, finalHtml, 'utf8'); console.log(`HTML Report generated: file://${reportPath}`); } /** * Renders a summary table comparing results across different dates for a specific test. */ function renderSummaryTable(dateToResult: Map, sortedDates: string[]): string { const results = Array.from(dateToResult.values()); if (results.length === 0) { return ''; } const firstResult = results[0]; let html = ''; for (const date of sortedDates) { html += ``; } html += ''; if (firstResult.type === 'JUDGE') { const rubrics = new Set(); for (const res of dateToResult.values()) { const stats = calculateStats(res); if (stats && 'statsByRubric' in stats) { Object.keys(stats.statsByRubric).forEach(r => rubrics.add(r)); } } const sortedRubrics = Array.from(rubrics).sort(); html += ``; for (const date of sortedDates) { const res = dateToResult.get(date); const stats = res ? calculateStats(res) : null; const score = stats && 'overallStats' in stats ? stats.overallStats.average.toFixed(2) : '-'; html += ``; } html += ''; for (const rubric of sortedRubrics) { html += ``; for (const date of sortedDates) { const res = dateToResult.get(date); const stats = res ? calculateStats(res) : null; const val = (stats && 'statsByRubric' in stats) ? stats.statsByRubric[rubric]?.average.toFixed(2) : '-'; html += ``; } html += ''; } } else if (firstResult.type === 'BINARY') { html += ''; for (const date of sortedDates) { const res = dateToResult.get(date); const stats = res ? calculateStats(res) : null; html += ``; } html += ''; } else if (firstResult.type === 'ROUGE') { html += ''; for (const date of sortedDates) { const res = dateToResult.get(date); const stats = res ? calculateStats(res) : null; html += ``; } html += ''; } html += '

Rubric	${date}
Weighted Overall	${score}
${rubric}	${val ?? '-'}
Pass Rate	${(stats && 'success' in stats) ? `${stats.success} / ${stats.total}` : '-'}
ROUGE-Lsum	${(stats && 'average' in stats) ? stats.average.toFixed(2) : '-'}

'; return html; } const CSS = ` :root { --primary: #1a73e8; --primary-hover: #185abc; --success: #1e8e3e; --error: #d93025; --bg: #f8f9fa; --sidebar-bg: #ffffff; --border: #dadce0; --text: #3c4043; --text-light: #70757a; --card-bg: #ffffff; --shadow: 0 1px 2px 0 rgba(60,64,67,.3), 0 1px 3px 1px rgba(60,64,67,.15); } body { font-family: 'Google Sans', Roboto, Helvetica, Arial, sans-serif; margin: 0; display: flex; height: 100vh; color: var(--text); background: var(--bg); } aside { width: 280px; background: var(--sidebar-bg); border-right: 1px solid var(--border); overflow-y: auto; padding: 24px; flex-shrink: 0; display: flex; flex-direction: column; } aside h1 { font-size: 20px; margin: 0 0 24px 0; color: var(--primary); } .nav-group { margin-bottom: 24px; } .nav-group-title { font-size: 12px; font-weight: 700; text-transform: uppercase; color: var(--text-light); margin-bottom: 8px; letter-spacing: .8px; } .nav-item { padding: 8px 12px; cursor: pointer; border-radius: 4px; margin-bottom: 4px; font-size: 14px; transition: background 0.2s; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .nav-item:hover { background: #f1f3f4; } main { flex-grow: 1; overflow-y: auto; padding: 48px; scroll-behavior: smooth; } section { max-width: 1000px; margin: 0 auto 64px auto; } .group-header { border-bottom: 1px solid var(--border); padding-bottom: 16px; margin-bottom: 32px; } .group-title { font-size: 28px; margin: 0; color: var(--text); } h3 { font-size: 20px; margin: 40px 0 16px 0; color: var(--text); } h4 { font-size: 16px; margin: 24px 0 12px 0; color: var(--text-light); } .card { background: var(--card-bg); border: 1px solid var(--border); border-radius: 8px; margin-bottom: 16px; overflow: hidden; transition: box-shadow 0.2s; } .card:hover { box-shadow: var(--shadow); } .card-header { padding: 16px 24px; background: #ffffff; display: flex; justify-content: space-between; align-items: center; cursor: pointer; user-select: none; } .card-header:hover { background: #f8f9fa; } .card-content { padding: 24px; display: none; border-top: 1px solid var(--border); background: #ffffff; } .card.open .card-content { display: block; } .tag { padding: 4px 12px; border-radius: 16px; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.5px; } .tag-pass { background: #e6f4ea; color: var(--success); } .tag-fail { background: #fce8e6; color: var(--error); } .score { font-size: 16px; font-weight: 700; } .score-badge { padding: 4px 12px; border-radius: 16px; background: #f1f3f4; font-size: 14px; display: inline-block; margin-bottom: 16px; } .metadata { color: var(--text-light); font-size: 12px; margin-bottom: 24px; display: flex; gap: 16px; } pre { background: #f8f9fa; padding: 16px; border-radius: 8px; overflow-x: auto; border: 1px solid var(--border); white-space: pre-wrap; font-family: 'Roboto Mono', monospace; font-size: 12px; line-height: 1.5; margin: 8px 0; } table { width: 100%; border-collapse: separate; border-spacing: 0; margin: 16px 0; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; } th, td { text-align: left; padding: 12px 16px; border-bottom: 1px solid var(--border); vertical-align: top; } th { background: #f8f9fa; font-weight: 600; font-size: 13px; color: var(--text-light); } tr:last-child td { border-bottom: none; } .reason { font-size: 13px; line-height: 1.6; color: var(--text); } .transcript-header { font-weight: 700; margin: 24px 0 12px 0; color: var(--primary); text-transform: uppercase; font-size: 11px; letter-spacing: 1px; display: flex; align-items: center; gap: 8px; } .transcript-header::after { content: ''; flex-grow: 1; height: 1px; background: var(--border); } .summary-table { margin: 24px 0 48px 0; } .summary-table td:first-child { width: 300px; font-weight: 500; } `;