{ "meta": { "built": "2026-11-11", "n_requests": 81, "mix": { "qa": 37, "classify": 24, "summarize": 12, "report": 8 }, "monthly_requests": 1200000, "monthly_requests_basis": "assumption: the 81-request mix repeats at 1.2M requests a month (about 10 percent of volume is per-account nightly digests)", "models": { "big": "claude-sonnet-5-5-high", "small": "gemini-3.1-flash-lite", "judge": "gemini-4-argon-high" }, "pricing_date": "2026-11-11", "cache_hit_rate_assumption": 0.95, "min_cache_tokens_assumption": 1024, "token_source": "API usage field (prompt_tokens, completion_tokens) from the local OpenAI-compatible proxy; prefix sizes measured per model", "company": "Corvid Cloud (fictional); 'Corvid Cloud support AI' is a fictional workload" }, "headline": { "family": "anthropic", "before_month": 23116.77037037037, "after_month": 432.8861148148149, "saved_pct": 0.9812739362861146, "quality_before": 0.9629629629629629, "quality_after": 0.9382716049382716, "pass_before": 78, "pass_after": 76 }, "families": { "anthropic": { "label": "Anthropic", "big": { "model": "Claude Sonnet 5.5", "input": 2.0, "cached_input": 0.1, "cache_write": 2.5, "output": 10.0 }, "small": { "model": "Claude Haiku 5.5 (prompts up to 100k)", "input": 0.1, "cached_input": 0.01, "cache_write": 0.125, "output": 0.5 }, "before_month": 23116.77037037037, "after_month": 432.8861148148149, "saved_month": 22683.884255555557, "saved_pct": 0.9812739362861146, "waterfall": [ { "key": "trim", "label": "Trim prompt and RAG context", "desc": "Shorter system prompt with every rule still in it, and 3 reranked chunks instead of the top 10.", "before": 23116.77037037037, "after": 9706.785185185186, "saving": 13409.985185185184, "per_request_before": 0.019263975308641976, "per_request_after": 0.008088987654320989 }, { "key": "caps", "label": "Cap output and reasoning", "desc": "Terse output format, max_tokens caps, and a low reasoning budget on the big model.", "before": 9706.785185185186, "after": 4707.22962962963, "saving": 4999.555555555557, "per_request_before": 0.008088987654320989, "per_request_after": 0.003922691358024691 }, { "key": "routing", "label": "Route easy calls to a cheaper model", "desc": "Classifier, summaries and nightly digests go to the small model; routine support questions too. Escalation re-runs doubtful answers on the big model.", "before": 4707.22962962963, "after": 1018.0814814814815, "saving": 3689.1481481481483, "per_request_before": 0.003922691358024691, "per_request_after": 0.0008484012345679013 }, { "key": "caching", "label": "Prompt caching", "desc": "The stable prefix (policy pack, label definitions) is billed at the cached-input rate once it is warm.", "before": 1018.0814814814815, "after": 451.12618518518525, "saving": 566.9552962962963, "per_request_before": 0.0008484012345679013, "per_request_after": 0.00037593848765432104 }, { "key": "dedupe", "label": "Dedupe repeated calls", "desc": "Exact-match and normalized-text cache: a repeated question is answered without a model call.", "before": 451.12618518518525, "after": 441.38092962962975, "saving": 9.745255555555502, "per_request_before": 0.00037593848765432104, "per_request_after": 0.00036781744135802477 }, { "key": "batch", "label": "Batch the nightly job", "desc": "The nightly digest is not urgent, so it runs through the Batch API at half price.", "before": 441.38092962962975, "after": 432.8861148148149, "saving": 8.494814814814845, "per_request_before": 0.00036781744135802477, "per_request_after": 0.0003607384290123457 } ], "per_task": { "qa": { "before": 8420.444444444445, "after": 401.99130000000014, "requests_share": 0.4567901234567901 }, "classify": { "before": 4581.866666666667, "after": 10.481481481481483, "requests_share": 0.2962962962962963 }, "summarize": { "before": 3107.2000000000003, "after": 11.91851851851852, "requests_share": 0.14814814814814814 }, "report": { "before": 7007.25925925926, "after": 8.494814814814816, "requests_share": 0.09876543209876543 } }, "stage_totals_month": [ 23116.77037037037, 9706.785185185186, 4707.22962962963, 1018.0814814814815, 451.12618518518525, 441.38092962962975, 432.8861148148149 ], "stage_per_request": [ 0.019263975308641976, 0.008088987654320989, 0.003922691358024691, 0.0008484012345679013, 0.00037593848765432104, 0.00036781744135802477, 0.0003607384290123457 ] }, "openai": { "label": "OpenAI", "big": { "model": "gpt-6.1-sol", "input": 2.0, "cached_input": 0.1, "output": 10.0 }, "small": { "model": "gpt-6-luna", "input": 0.1, "cached_input": 0.01, "output": 0.5 }, "before_month": 23116.77037037037, "after_month": 425.04913333333343, "saved_month": 22691.721237037036, "saved_pct": 0.9816129534306343, "waterfall": [ { "key": "trim", "label": "Trim prompt and RAG context", "desc": "Shorter system prompt with every rule still in it, and 3 reranked chunks instead of the top 10.", "before": 23116.77037037037, "after": 9706.785185185186, "saving": 13409.985185185184, "per_request_before": 0.019263975308641976, "per_request_after": 0.008088987654320989 }, { "key": "caps", "label": "Cap output and reasoning", "desc": "Terse output format, max_tokens caps, and a low reasoning budget on the big model.", "before": 9706.785185185186, "after": 4707.22962962963, "saving": 4999.555555555557, "per_request_before": 0.008088987654320989, "per_request_after": 0.003922691358024691 }, { "key": "routing", "label": "Route easy calls to a cheaper model", "desc": "Classifier, summaries and nightly digests go to the small model; routine support questions too. Escalation re-runs doubtful answers on the big model.", "before": 4707.22962962963, "after": 1018.0814814814815, "saving": 3689.1481481481483, "per_request_before": 0.003922691358024691, "per_request_after": 0.0008484012345679013 }, { "key": "caching", "label": "Prompt caching", "desc": "The stable prefix (policy pack, label definitions) is billed at the cached-input rate once it is warm.", "before": 1018.0814814814815, "after": 443.1260000000001, "saving": 574.9554814814815, "per_request_before": 0.0008484012345679013, "per_request_after": 0.0003692716666666667 }, { "key": "dedupe", "label": "Dedupe repeated calls", "desc": "Exact-match and normalized-text cache: a repeated question is answered without a model call.", "before": 443.1260000000001, "after": 433.5439481481483, "saving": 9.582051851851816, "per_request_before": 0.0003692716666666667, "per_request_after": 0.0003612866234567902 }, { "key": "batch", "label": "Batch the nightly job", "desc": "The nightly digest is not urgent, so it runs through the Batch API at half price.", "before": 433.5439481481483, "after": 425.04913333333343, "saving": 8.494814814814845, "per_request_before": 0.0003612866234567902, "per_request_after": 0.00035420761111111114 } ], "per_task": { "qa": { "before": 8420.444444444445, "after": 394.1543185185186, "requests_share": 0.4567901234567901 }, "classify": { "before": 4581.866666666667, "after": 10.481481481481483, "requests_share": 0.2962962962962963 }, "summarize": { "before": 3107.2000000000003, "after": 11.91851851851852, "requests_share": 0.14814814814814814 }, "report": { "before": 7007.25925925926, "after": 8.494814814814816, "requests_share": 0.09876543209876543 } }, "stage_totals_month": [ 23116.77037037037, 9706.785185185186, 4707.22962962963, 1018.0814814814815, 443.1260000000001, 433.5439481481483, 425.04913333333343 ], "stage_per_request": [ 0.019263975308641976, 0.008088987654320989, 0.003922691358024691, 0.0008484012345679013, 0.0003692716666666667, 0.0003612866234567902, 0.00035420761111111114 ] }, "google": { "label": "Google", "big": { "model": "gemini-4-carbon", "input": 2.0, "cached_input": 0.1, "output": 10.0 }, "small": { "model": "gemini-3.1-flash-lite", "input": 0.25, "cached_input": 0.025, "output": 1.5 }, "before_month": 23116.77037037037, "after_month": 523.9430555555556, "saved_month": 22592.827314814815, "saved_pct": 0.9773349370538753, "waterfall": [ { "key": "trim", "label": "Trim prompt and RAG context", "desc": "Shorter system prompt with every rule still in it, and 3 reranked chunks instead of the top 10.", "before": 23116.77037037037, "after": 9706.785185185186, "saving": 13409.985185185184, "per_request_before": 0.019263975308641976, "per_request_after": 0.008088987654320989 }, { "key": "caps", "label": "Cap output and reasoning", "desc": "Terse output format, max_tokens caps, and a low reasoning budget on the big model.", "before": 9706.785185185186, "after": 4707.22962962963, "saving": 4999.555555555557, "per_request_before": 0.008088987654320989, "per_request_after": 0.003922691358024691 }, { "key": "routing", "label": "Route easy calls to a cheaper model", "desc": "Classifier, summaries and nightly digests go to the small model; routine support questions too. Escalation re-runs doubtful answers on the big model.", "before": 4707.22962962963, "after": 1219.6, "saving": 3487.62962962963, "per_request_before": 0.003922691358024691, "per_request_after": 0.0010163333333333333 }, { "key": "caching", "label": "Prompt caching", "desc": "The stable prefix (policy pack, label definitions) is billed at the cached-input rate once it is warm.", "before": 1219.6, "after": 572.8815185185186, "saving": 646.7184814814813, "per_request_before": 0.0010163333333333333, "per_request_after": 0.00047740126543209885 }, { "key": "dedupe", "label": "Dedupe repeated calls", "desc": "Exact-match and normalized-text cache: a repeated question is answered without a model call.", "before": 572.8815185185186, "after": 546.7115740740742, "saving": 26.169944444444468, "per_request_before": 0.00047740126543209885, "per_request_after": 0.00045559297839506174 }, { "key": "batch", "label": "Batch the nightly job", "desc": "The nightly digest is not urgent, so it runs through the Batch API at half price.", "before": 546.7115740740742, "after": 523.9430555555556, "saving": 22.76851851851859, "per_request_before": 0.00045559297839506174, "per_request_after": 0.000436619212962963 } ], "per_task": { "qa": { "before": 8420.444444444445, "after": 441.20046296296306, "requests_share": 0.4567901234567901 }, "classify": { "before": 4581.866666666667, "after": 26.37037037037037, "requests_share": 0.2962962962962963 }, "summarize": { "before": 3107.2000000000003, "after": 33.6037037037037, "requests_share": 0.14814814814814814 }, "report": { "before": 7007.25925925926, "after": 22.76851851851852, "requests_share": 0.09876543209876543 } }, "stage_totals_month": [ 23116.77037037037, 9706.785185185186, 4707.22962962963, 1219.6, 572.8815185185186, 546.7115740740742, 523.9430555555556 ], "stage_per_request": [ 0.019263975308641976, 0.008088987654320989, 0.003922691358024691, 0.0010163333333333333, 0.00047740126543209885, 0.00045559297839506174, 0.000436619212962963 ] } }, "levers": [ { "key": "trim", "label": "Trim prompt and RAG context", "desc": "Shorter system prompt with every rule still in it, and 3 reranked chunks instead of the top 10.", "example": { "id": "R074", "task": "qa", "title": "How do I reset 2FA for a teammate who lost their phone?", "before": { "prompt_tokens": 4371, "completion_tokens": 481, "model": "claude-sonnet-5-5-high", "role": "big", "cost": 0.013552 }, "after": { "prompt_tokens": 2266, "completion_tokens": 341, "model": "claude-sonnet-5-5-high", "role": "big", "cost": 0.007942 }, "before_in": "System prompt 3,073 tokens + 10 retrieved chunks", "after_in": "System prompt 1,792 tokens (same rules, no filler) + 3 reranked chunks", "before_text": "Hello, and thank you so much for reaching out to Corvid Cloud support. I'm sorry your teammate has lost their phone. I know that can be stressful, but this is quick to fix if you are a workspace admin.\n\n**How to reset 2FA for a teammate**\n\nIf you are a workspace admin, you can reset their two-factor authentication (the extra verification step from their authenticator app) like this:\n\n1. Open **Members** in your workspace.\n2. Select the teammate who lost their phone.\n3. Choose **Reset 2FA**. [K08]\n\nOnce the reset is done, your teammate can set up two-factor authentication again on their new device under **Profile > Security**. [K08]\n\n**Good to know**\n\n- Backup codes are shown only once, at", "after_text": "{\"answer\": \"A workspace admin can reset it: go to Members > the member > Reset 2FA. The teammate can then set up authenticator-app 2FA again under Profile > Security. Backup codes are shown only once at setup and cannot be shown again.\", \"cite\": [\"K08\"], \"confident\": true, \"escalate_to\": null}", "before_prompt": "Here are the knowledge base excerpts retrieved for this question:\n\n[K08] Two-factor authentication\nAny member can enable authenticator-app two-factor authentication under Profile > Security. Business workspaces can require 2FA for all members. If a member loses their device, a workspace admin can reset their 2FA from Members > the member > Reset 2FA. Backup …", "after_prompt": "EXCERPTS\n[K08] Two-factor authentication\nAny member can enable authenticator-app two-factor authentication under Profile > Security. Business workspaces can require 2FA for all members. If a member loses their device, a workspace admin can reset their 2FA from Members > the member > Reset 2FA. Backup codes are shown once at setup and cannot be shown again.\n\n…" }, "standalone_by_family": {}, "saving_by_family": { "anthropic": 13409.985185185184, "openai": 13409.985185185184, "google": 13409.985185185184 } }, { "key": "caps", "label": "Cap output and reasoning", "desc": "Terse output format, max_tokens caps, and a low reasoning budget on the big model.", "example": { "id": "R060", "task": "classify", "title": "Lost my phone, locked out of the owner account because of 2FA.", "before": { "prompt_tokens": 486, "completion_tokens": 424, "model": "claude-sonnet-5-5-high", "role": "big", "cost": 0.005212 }, "after": { "prompt_tokens": 454, "completion_tokens": 5, "model": "claude-sonnet-5-5-high", "role": "big", "cost": 0.000958 }, "before_in": "Asked to explain its reasoning, then give a label", "after_in": "One label, max 12 tokens, low reasoning budget", "before_text": "account_access", "after_text": "account_access", "before_prompt": "Ticket: Lost my phone, locked out of the owner account because of 2FA.\nLabel:", "after_prompt": "Ticket: Lost my phone, locked out of the owner account because of 2FA.\nLabel:" }, "standalone_by_family": {}, "saving_by_family": { "anthropic": 4999.555555555557, "openai": 4999.555555555557, "google": 4999.555555555557 } }, { "key": "routing", "label": "Route easy calls to a cheaper model", "desc": "Classifier, summaries and nightly digests go to the small model; routine support questions too. Escalation re-runs doubtful answers on the big model.", "example": { "id": "R077", "task": "summarize", "title": "Ticket T-7105 | Customer: Aisha (Business)", "before": { "prompt_tokens": 290, "completion_tokens": 126, "model": "claude-sonnet-5-5-high", "role": "big", "cost": 0.00184 }, "after": { "prompt_tokens": 184, "completion_tokens": 52, "model": "gemini-3.1-flash-lite", "role": "small", "cost": 4.440000000000001e-05 }, "before_in": "Big model: claude-sonnet-5-5-high", "after_in": "Small model: gemini-3.1-flash-lite", "before_text": "Issue: Aisha (Business plan) needed Q3 login history with IPs, plus role-change events, for an auditor.\nStatus: Resolved. Agent pointed her to Settings > Security > Audit log, which logs sign-ins (IP, timestamp) and role changes, retained 12 months, exportable as CSV with date filter. Customer confirmed.\nNext: None.", "after_text": "Issue: Customer needed Q3 login history with IPs for an auditor; agent confirmed audit logs cover sign-ins and role changes for 12 months, exportable as CSV.\nStatus: Resolved; instructions provided and confirmed working.\nNext: None.", "before_prompt": "Ticket T-7105 | Customer: Aisha (Business)\nAisha: Our auditor wants login history for Q3 including IPs.\nAgent: Audit logs under Settings > Security > Audit log cover sign-ins with IP and timestamp, kept 12 months, exportable as CSV.\nAisha: Found it. Can we get role-change events too?\nAgent: Yes, role changes are logged as well. Export both with the date filt…", "after_prompt": "Ticket T-7105 | Customer: Aisha (Business)\nAisha: Our auditor wants login history for Q3 including IPs.\nAgent: Audit logs under Settings > Security > Audit log cover sign-ins with IP and timestamp, kept 12 months, exportable as CSV.\nAisha: Found it. Can we get role-change events too?\nAgent: Yes, role changes are logged as well. Export both with the date filt…" }, "standalone_by_family": {}, "saving_by_family": { "anthropic": 3689.1481481481483, "openai": 3689.1481481481483, "google": 3487.62962962963 } }, { "key": "caching", "label": "Prompt caching", "desc": "The stable prefix (policy pack, label definitions) is billed at the cached-input rate once it is warm.", "example": { "id": "R074", "task": "qa", "title": "How do I reset 2FA for a teammate who lost their phone?", "before": { "prompt_tokens": 1527, "completion_tokens": 48, "model": "gemini-3.1-flash-lite", "role": "small", "cost": 0.0001767 }, "after": { "prompt_tokens": 1527, "completion_tokens": 48, "model": "gemini-3.1-flash-lite", "role": "small", "cost": 7.273550000000001e-05, "cached_tokens": 1172 }, "before_in": "Every token billed at the normal input rate", "after_in": "1,172 of 1,527 prompt tokens are the stable prefix, billed at the cached rate", "before_text": "A workspace admin can reset a teammate's 2FA from Members > the member > Reset 2FA.", "after_text": "A workspace admin can reset a teammate's 2FA from Members > the member > Reset 2FA. (identical: caching changes the price, not the answer)", "before_prompt": "EXCERPTS\n[K08] Two-factor authentication\nAny member can enable authenticator-app two-factor authentication under Profile > Security. Business workspaces can require 2FA for all members. If a member loses their device, a workspace admin can reset their 2FA from Members > the member > Reset 2FA. Backup codes are shown once at setup and cannot be shown again.\n\n…", "after_prompt": "You are Corvid Cloud support. Answer from the provided excerpts only. Output JSON only.\n\nPOLICY PACK (authoritative)\nPlans: Starter $12, Team $29, Business $59 per seat/month, monthly billing. Annual = 15% off. Prices exclude VAT/sales tax.\nLimits: Starter …" }, "standalone_by_family": { "anthropic": 6563.927999999998, "openai": 6656.117999999999, "google": 6656.117999999999 }, "saving_by_family": { "anthropic": 566.9552962962963, "openai": 574.9554814814815, "google": 646.7184814814813 } }, { "key": "dedupe", "label": "Dedupe repeated calls", "desc": "Exact-match and normalized-text cache: a repeated question is answered without a model call.", "example": { "id": "R026", "task": "qa", "title": "What happens if my card payment fails?", "before": { "prompt_tokens": 1558, "completion_tokens": 95, "model": "gemini-3.1-flash-lite", "role": "small", "cost": 9.722925e-05 }, "after": { "prompt_tokens": 0, "completion_tokens": 0, "model": "cache", "role": "cache", "cost": 0.0 }, "before_in": "A model call for a question we already answered", "after_in": "normalized cache hit on R022: \"What happens when my card payment fails\"", "before_text": "If a card payment fails we retry on days 3, 5 and 7. After the third failure the workspace becomes read-only and after day 14 it is suspended. Updating the card under Settings > Billing > Payment method triggers an immediate retry. Data is retained for 60 days after suspension.", "after_text": "If a card payment fails we retry on days 3, 5 and 7. After the third failure the workspace becomes read-only and after day 14 it is suspended. Updating the card under Settings > Billing > Payment method triggers an immediate retry. Data is retained for 60 days after suspension.", "before_prompt": "What happens if my card payment fails?", "after_prompt": "What happens when my card payment fails" }, "standalone_by_family": { "anthropic": 1718.0740740740728, "openai": 1718.0740740740728, "google": 1718.0740740740728 }, "saving_by_family": { "anthropic": 9.745255555555502, "openai": 9.582051851851816, "google": 26.169944444444468 } }, { "key": "batch", "label": "Batch the nightly job", "desc": "The nightly digest is not urgent, so it runs through the Batch API at half price.", "example": { "id": "R009", "task": "report", "title": "Nightly digest: Fjord Analytics", "before": { "prompt_tokens": 913, "completion_tokens": 98, "model": "gemini-3.1-flash-lite", "role": "small", "cost": 0.00014030000000000002 }, "after": { "prompt_tokens": 913, "completion_tokens": 98, "model": "gemini-3.1-flash-lite", "role": "small", "cost": 7.015000000000001e-05 }, "before_in": "Synchronous call, nobody is waiting for it at 02:00", "after_in": "Same call through the Batch API (50 percent off, results within 24 h)", "before_text": "Total: 30 tickets\nBy category: billing=8, bug=8, how_to=4, feature_request=4, account_access=4, security=2\nOpen sev1: 0\nLongest open: T-8409 (80h)\nRisks: Account access and billing items remain pending past 77 hours, impacting user onboarding.\nAction: Escalate pending billing and account access tickets to prevent churn.", "after_text": "Total: 30 tickets\nBy category: billing=8, bug=8, how_to=4, feature_request=4, account_access=4, security=2\nOpen sev1: 0\nLongest open: T-8409 (80h)\nRisks: Account access and billing items remain pending past 77 hours, impacting user onboarding.\nAction: Escalate pending billing and account access tickets to prevent churn. (identical: batch changes the price, not the answer)", "before_prompt": "", "after_prompt": "" }, "standalone_by_family": { "anthropic": 3503.629629629628, "openai": 3503.629629629628, "google": 3503.629629629628 }, "saving_by_family": { "anthropic": 8.494814814814845, "openai": 8.494814814814845, "google": 22.76851851851859 } } ], "quality": { "baseline": { "pass": 78, "total": 81, "by_task": { "qa": { "pass": 36, "total": 37 }, "classify": { "pass": 23, "total": 24 }, "summarize": { "pass": 12, "total": 12 }, "report": { "pass": 7, "total": 8 } } }, "optimized_v1": { "pass": 59, "total": 81, "by_task": { "qa": { "pass": 28, "total": 37 }, "classify": { "pass": 23, "total": 24 }, "summarize": { "pass": 7, "total": 12 }, "report": { "pass": 1, "total": 8 } } }, "optimized_v2": { "pass": 62, "total": 81, "by_task": { "qa": { "pass": 31, "total": 37 }, "classify": { "pass": 23, "total": 24 }, "summarize": { "pass": 7, "total": 12 }, "report": { "pass": 1, "total": 8 } } }, "optimized_v3": { "pass": 72, "total": 81, "by_task": { "qa": { "pass": 31, "total": 37 }, "classify": { "pass": 23, "total": 24 }, "summarize": { "pass": 10, "total": 12 }, "report": { "pass": 8, "total": 8 } } }, "optimized_final": { "pass": 76, "total": 81, "by_task": { "qa": { "pass": 35, "total": 37 }, "classify": { "pass": 23, "total": 24 }, "summarize": { "pass": 10, "total": 12 }, "report": { "pass": 8, "total": 8 } } }, "escalation": { "escalated": 4, "numeric_or_sensitive_to_big": 7, "rule": "Cheap answer is re-run on the big model if it says confident=false, cites nothing, or cites a chunk that was not retrieved. Arithmetic/multi-number questions route straight to the big model.", "rule_flags_without_big_answer": 0 }, "failures": [ { "id": "R008", "task": "qa", "input": "How do I cancel my subscription? Will I get my money back?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello, and thank you for contacting Corvid Cloud support. I'm happy to help with both questions.\n\n**How to cancel your subscription**\n\nOnly the workspace owner can cancel the plan. If that's you, please follow these steps [K31]:\n\n1. Open **Settings**.\n2. Choose **Billing**.\n3. Select **Cancel plan** and confirm.\n\n**What happens after you cancel** [K31]\n\n- Your workspace stays active until the end of the period you", "v1_answer": "Owners can cancel under Settings > Billing > Cancel plan. Cancelling does not trigger a refund. Monthly plans are refundable within 14 days of the first charge and annual plans within 30 days. After the window, we do not issue refunds, but you can cancel renewal.", "v2_answer": "Owners can cancel under Settings > Billing > Cancel plan. Cancelling does not trigger a refund. Monthly plans are refundable within 14 days of the first charge and annual plans within 30 days. After the window, we do not issue refunds, but you can cancel renewal.", "fin_answer": "Owners can cancel under Settings > Billing > Cancel plan. The workspace stays active until the end of the paid period and then becomes read-only for 60 days before deletion. Cancelling does not trigger a refund. Monthly plans are refundable within 14 days of the first charge only; annual plans within 30 days. After the window we do not issue refunds, but we can cancel renewal. Requests outside the window or above $50", "gold": null, "fin_note": "Accurate, covers all required facts, and follows refund policy rules.", "v1_note": "Missing must-have fact that the workspace stays active until the end of the paid period.", "v2_note": "Missing must-have fact that the workspace stays active until the end of the paid period.", "baseline_note": "Accurate, complete, and follows all policy rules.", "v2_escalated": false, "cache": null }, { "id": "R009", "task": "report", "input": "T-8395 | bug | sev2 | open | 20h | Webhook not retried\nT-8397 | bug | sev3 | resolved | 65h | Calendar sync lag\nT-8399 | how_to | sev2 | pending | 5h | Add a guest\nT-8400 | how_to | sev3 | resolved | 48h | Add a guest\nT-8403 | bug | sev1 | resolved | 75h | Tasks vanish on refresh\nT-8404 | bug | sev3", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is the nightly account health report for **Fjord Analytics**.\n\n## Executive summary\nFjord Analytics had 30 tickets in this window: 14 resolved (47%), 11 pending and 5 open. **No severity-1 tickets are unresolved**, since all three were resolved. Several issues have recurred, though, including one that was sev1 earlier, and there is one potentially serious security ticket sitting open. The main concerns ar", "v1_answer": "Total: 30 tickets\nBy category: bug=9, how_to=4, billing=8, account_access=4, feature_request=3, security=2\nOpen sev1: 0 (status is not resolved)\nLongest open: T-8408 (77h, unresolved only)\nRisks: Unresolved billing (T-8408) and security (T-8446) tickets are lingering past 70 hours.\nAction: Escalate T-8408 and T-8446 to senior support immediately.", "v2_answer": "Total: 30 tickets\nBy category: bug=9, how_to=4, billing=8, account_access=4, feature_request=3, security=2\nOpen sev1: 0 (status is not resolved)\nLongest open: T-8408 (77h, unresolved only)\nRisks: Unresolved billing (T-8408) and security (T-8446) tickets are lingering past 70 hours.\nAction: Escalate T-8408 and T-8446 to senior support immediately.", "fin_answer": "Total: 30 tickets\nBy category: billing=8, bug=8, how_to=4, feature_request=4, account_access=4, security=2\nOpen sev1: 0\nLongest open: T-8409 (80h)\nRisks: Account access and billing items remain pending past 77 hours, impacting user onboarding.\nAction: Escalate pending billing and account access tickets to prevent churn.", "gold": null, "fin_note": "All must-have facts and counts are accurate and present; no policy violations.", "v1_note": "Wrong counts for bug (9 vs 8) and feature_request (3 vs 4), and wrong longest-open ticket (T-8408 vs T-8409).", "v2_note": "Wrong counts for bug (9 vs 8) and feature_request (3 vs 4), and wrong longest-open ticket (T-8408 vs T-8409).", "baseline_note": "All counts, facts, and KB references are accurate and complete.", "v2_escalated": false, "cache": null }, { "id": "R010", "task": "report", "input": "T-8078 | how_to | sev3 | resolved | 25h | Set up Slack\nT-8082 | feature_request | sev2 | resolved | 46h | Gantt dependencies\nT-8083 | feature_request | sev3 | open | 78h | Dark mode\nT-8087 | feature_request | sev3 | pending | 47h | Gantt dependencies\nT-8089 | billing | sev3 | pending | 62h | Refund ", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is the nightly account health report for Brightside Dental.\n\n# Nightly Account Health Report: Brightside Dental\n\n## Data caveats\n- The export is described as the \"last 24 hours\", but 24 of the 34 tickets show more than 24 hours open (up to 96h). I included all 34 as provided. If the export was meant to be a rolling 24-hour window, the query may be pulling by last-updated rather than created date.\n- \"Hours", "v1_answer": "Total: 34 tickets\nBy category: how_to=7, feature_request=9, billing=5, account_access=5, bug=6, security=2\nOpen sev1: 0\nLongest open: T-8104 (93h, unresolved only)\nRisks: Multiple unresolved feature requests and billing issues pending.\nAction: Address the 93h open guest setup ticket and pending billing items.", "v2_answer": "Total: 34 tickets\nBy category: how_to=7, feature_request=9, billing=5, account_access=5, bug=6, security=2\nOpen sev1: 0\nLongest open: T-8104 (93h, unresolved only)\nRisks: Multiple unresolved feature requests and billing issues pending.\nAction: Address the 93h open guest setup ticket and pending billing items.", "fin_answer": "Total: 34 tickets\nBy category: billing=5, bug=6, how_to=7, feature_request=9, account_access=5, security=2\nOpen sev1: 0\nLongest open: T-8145 (96h)\nRisks: Unresolved bug regarding export hangs (T-8145) has been open for 96 hours.\nAction: Escalate the pending export hang bug to engineering for immediate review.", "gold": null, "fin_note": "All counts and must-have facts are accurate and complete.", "v1_note": "Incorrectly identified longest-open unresolved ticket as T-8104 (93h) instead of T-8145 (96h).", "v2_note": "Incorrectly identified longest-open unresolved ticket as T-8104 (93h) instead of T-8145 (96h).", "baseline_note": "All counts, categories, and KB references are accurate and complete.", "v2_escalated": false, "cache": null }, { "id": "R019", "task": "summarize", "input": "Ticket T-7108 | Customer: Hiro (Business)\nHiro: We need our data in the EU region for a new German client. Can you move our workspace?\nAgent: Region can't be changed after creation. You'd need to create a new EU workspace, export and import.\nHiro: That's a lot of work. Any tool?\nAgent: Export ZIP an", "baseline_pass": 1, "v1_pass": 1, "v2_pass": 1, "fin_pass": 0, "baseline_answer": "Hello! Here is a summary of ticket T-7108.\n\n**Customer and request**\nHiro, a Business plan customer, asked whether Corvid Cloud could move their existing workspace to the EU region. They need their data hosted in the EU because they are onboarding a new client in Germany.\n\n**What was discussed**\nThe agent explained that a workspace's region cannot be changed after creation, so a move is not possible. The agent rec", "v1_answer": "Issue: Customer needs to move their workspace to the EU region for a German client, but Corvid Cloud cannot migrate existing regions.\nStatus: Resolved; explained that a new EU workspace must be created and data migrated via ZIP export and CSV/JSON import (files up to 5 GB separately). Customer will plan this for the weekend.\nNext: None.", "v2_answer": "Issue: Customer needs to move their workspace to the EU region for a German client, but Corvid Cloud cannot migrate existing regions.\nStatus: Resolved; explained that a new EU workspace must be created and data migrated via ZIP export and CSV/JSON import (files up to 5 GB separately). Customer will plan this for the weekend.\nNext: None.", "fin_answer": "Issue: Business workspace region change requested for German client; region cannot be changed post-creation; export/import required; files separate up to 5 GB via ZIP export and CSV/JSON import.\nStatus: Resolved; agent explained method and file limit.\nNext: Customer will plan migration for the weekend.", "gold": null, "fin_note": "Missed specifying EU region and creating a new EU workspace.", "v1_note": "Accurate, complete, and adheres to all policies.", "v2_note": "Accurate, complete, and adheres to all policies.", "baseline_note": "Accurate, includes all must-have facts, and stays within KB and thread facts.", "v2_escalated": false, "cache": null }, { "id": "R024", "task": "qa", "input": "How many requests per minute can I make to the API on Team?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello and thank you for reaching out to Corvid Cloud support, I am happy to help!\n\nOn the Team plan, you can make up to **600 requests per minute per token** to the REST API [K16]. Here are a few details that may be useful:\n\n- **Limit applies per token:** The 600 requests per minute applies to each API token separately [K16].\n- **If you go over the limit:** The API returns an HTTP 429 response with a Retry-After h", "v1_answer": "Limits are 600 requests per minute per token on Team.", "v2_answer": "Limits are 600 requests per minute per token on Team.", "fin_answer": "The REST API is available on Team with a limit of 600 requests per minute per token. Exceeding the limit returns HTTP 429 with a Retry-After header.", "gold": null, "fin_note": "Accurate, complete, and follows all policies.", "v1_note": "Missing the must-have fact that exceeding the limit returns HTTP 429.", "v2_note": "Missing the must-have fact that exceeding the limit returns HTTP 429.", "baseline_note": "Accurate, complete, and adheres to all policies.", "v2_escalated": false, "cache": null }, { "id": "R031", "task": "qa", "input": "My invoice has the wrong company name. Can you reissue last month's?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 0, "baseline_answer": "Hello, and thank you for reaching out to Corvid Cloud support. I'm sorry your invoice shows the wrong company name, and I'd be glad to help you get it sorted.\n\n**Reissuing last month's invoice**\nI'm not able to approve or guarantee an invoice reissue myself. What I can tell you is that our only stated limit is that invoices more than 12 months old can't be edited or reissued [K04], and last month's invoice is well", "v1_answer": "Corvid Cloud cannot edit or reissue invoices that are more than 12 months old. To change the company name on future invoices, edit Settings > Billing > Company details.", "v2_answer": "Corvid Cloud cannot edit or reissue invoices that are more than 12 months old. To change the company name on future invoices, edit Settings > Billing > Company details.", "fin_answer": "To change the company name, address or VAT ID on future invoices, edit Settings > Billing > Company details. Corvid Cloud cannot edit or reissue invoices that are more than 12 months old.", "gold": null, "fin_note": "Did not offer to check with the Billing team about reissuing last month's invoice.", "v1_note": "Does not offer the Billing team for reissuing last month's invoice.", "v2_note": "Does not offer the Billing team for reissuing last month's invoice.", "baseline_note": "Accurate, covers all must-have facts, and avoids promising a reissue.", "v2_escalated": false, "cache": null }, { "id": "R032", "task": "qa", "input": "How do I cancel my subscription? Will I get my money back?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello, and thank you for contacting Corvid Cloud support. I'm happy to help with both questions.\n\n**How to cancel your subscription**\n\nOnly the workspace owner can cancel. If that's you, please follow these steps:\n\n1. Open **Settings**.\n2. Choose **Billing**.\n3. Select **Cancel plan** and confirm.\n\nYour workspace stays active until the end of the period you've already paid for. After that it becomes read-only for ", "v1_answer": "Owners can cancel under Settings > Billing > Cancel plan. Cancelling does not trigger a refund. Monthly plans are refundable within 14 days of the first charge and annual plans within 30 days. After the window, we do not issue refunds, but you can cancel renewal.", "v2_answer": "Owners can cancel under Settings > Billing > Cancel plan. Cancelling does not trigger a refund. Monthly plans are refundable within 14 days of the first charge and annual plans within 30 days. After the window, we do not issue refunds, but you can cancel renewal.", "fin_answer": "Owners can cancel under Settings > Billing > Cancel plan. The workspace stays active until the end of the paid period and then becomes read-only for 60 days before deletion. Cancelling does not trigger a refund. Monthly plans are refundable within 14 days of the first charge only; annual plans within 30 days. After the window we do not issue refunds, but we can cancel renewal. Requests outside the window or above $50", "gold": null, "fin_note": "Accurate, covers all required facts, and follows refund policy rules.", "v1_note": "Missing must-have fact that the workspace stays active until the end of the paid period.", "v2_note": "Missing must-have fact that the workspace stays active until the end of the paid period.", "baseline_note": "Accurate, complete, and follows all policy guidelines.", "v2_escalated": false, "cache": { "kind": "exact", "source": "R008" } }, { "id": "R035", "task": "summarize", "input": "Ticket T-7104 | Customer: Jonas (Team)\nJonas: Import of 18,000 rows from Asana CSV fails immediately.\nAgent: The limit is 10,000 rows or 50 MB per file. Can you split the file?\nJonas: I split into two files but the second shows errors on date columns.\nAgent: Dates must be ISO format YYYY-MM-DD. Your", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is a summary of ticket T-7104 for your colleague.\n\n**Customer and problem**\n\nJonas, a customer on the Team plan, tried to import a CSV export from Asana containing 18,000 rows. The import failed immediately.\n\n**What was tried**\n\n1. The agent explained that imports are limited to 10,000 rows or 50 MB per file, and asked Jonas to split the file. The 18,000-row file exceeded the row limit.\n2. Jonas split the", "v1_answer": "Issue: Asana CSV import failed due to exceeding the 10,000-row limit (18,000 rows) and incorrect date format (03/11/2026 instead of ISO YYYY-MM-DD).\nStatus: pending customer; files split and date format error identified.\nNext: Customer is fixing date formats to ISO YYYY-MM-DD and will retry import.", "v2_answer": "Issue: Asana CSV import failed due to exceeding the 10,000-row limit (18,000 rows) and incorrect date format (03/11/2026 instead of ISO YYYY-MM-DD).\nStatus: pending customer; files split and date format error identified.\nNext: Customer is fixing date formats to ISO YYYY-MM-DD and will retry import.", "fin_answer": "Issue: Asana CSV import of 18,000 rows failed due to the 10,000 rows or 50 MB per file limit. Split file two had date column errors because dates were 03/11/2026 instead of ISO format YYYY-MM-DD.\nStatus: pending customer; agent stated the 10,000 rows or 50 MB limit and ISO format YYYY-MM-DD rule.\nNext: customer will fix date formats.", "gold": null, "fin_note": "Accurate, complete, and adheres to all policies.", "v1_note": "Did not explicitly mention that the second file had the date format errors.", "v2_note": "Did not explicitly mention that the second file had the date format errors.", "baseline_note": "Accurate, includes all must-have facts, and does not invent facts or violate policy.", "v2_escalated": false, "cache": null }, { "id": "R037", "task": "qa", "input": "How much is Team for 12 people billed annually?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 1, "fin_pass": 1, "baseline_answer": "Hello and thank you for your interest in Corvid Cloud! I'm happy to help with the pricing.\n\nHere is the breakdown for the Team plan with 12 seats on annual billing [K01]:\n\n1. **List price:** Team is $29 per seat per month, so 12 seats come to $348 per month on monthly billing.\n2. **Annual discount:** Annual billing takes 15 percent off, which brings the price to $24.65 per seat per month.\n3. **12 seats, per month ", "v1_answer": "Team is $29 per seat per month. For 12 seats, the annual cost with a 15% discount is $2,968.20 (($29 * 12 * 12) * 0.85).", "v2_answer": "Team is $29 per seat/month. For 12 seats: 12 × $29 = $348/month on monthly billing. Annual billing takes 15% off: $348 × 0.85 = $295.80/month, or $3,549.60 per year. Prices exclude VAT/sales tax.", "fin_answer": "Team is $29 per seat/month. For 12 seats: 12 × $29 = $348/month on monthly billing. Annual billing takes 15% off: $348 × 0.85 = $295.80/month, or $3,549.60 per year. Prices exclude VAT/sales tax.", "gold": null, "fin_note": "Accurate calculations and includes all required facts.", "v1_note": "Arithmetic error: ($29 * 12 * 12) * 0.85 is $3,549.60, not $2,968.20; also missing monthly figures.", "v2_note": "Accurate calculations and includes all required facts.", "baseline_note": "Accurate math, includes all must-have facts and KB-supported extra details.", "v2_escalated": false, "cache": null }, { "id": "R038", "task": "summarize", "input": "Ticket T-7106 | Customer: Tom (Team)\nTom: We hit HTTP 429 from the API every night around 02:00 with our sync job.\nAgent: Team limit is 600 requests/min per token. Does the job use one token?\nTom: Yes, and it fires about 2,000 requests in a minute.\nAgent: Please throttle to under 600/min and honor R", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is a summary of ticket T-7106 for your reference.\n\n**Customer and problem**\nTom, a customer on the Team plan, reported that his nightly sync job receives HTTP 429 errors (the \"too many requests\" response) from the Corvid Cloud API. The errors occur every night at around 02:00, when the job runs.\n\n**What was discussed and tried**\nThe agent first explained that the Team plan limit is 600 requests per min", "v1_answer": "Issue: HTTP 429 errors every night at 02:00 due to sync job firing ~2,000 req/min against the 600 req/min team limit.\nStatus: Resolved; customer advised to throttle under 600/min and honor Retry-After, with business offering a limit raise to 3,000/min.\nNext: None.", "v2_answer": "Issue: HTTP 429 errors every night at 02:00 due to sync job firing ~2,000 req/min against the 600 req/min team limit.\nStatus: Resolved; customer advised to throttle under 600/min and honor Retry-After, with business offering a limit raise to 3,000/min.\nNext: None.", "fin_answer": "Issue: HTTP 429 errors from API every night around 02:00 on sync job for Team plan due to 2,000 requests/min against 600 requests/min per token limit. Business plan raises limit to 3,000/min.\nStatus: resolved. Agent stated Team limit is 600 requests/min per token and Business raises limit to 3,000/min; advised throttling under 600/min and honoring Retry-After.\nNext: customer will throttle first, considering upgrading", "gold": null, "fin_note": "Accurate, complete, and adheres to all guidelines.", "v1_note": "Misses that the customer may upgrade to Business later (and that the limit is per token).", "v2_note": "Misses that the customer may upgrade to Business later (and that the limit is per token).", "baseline_note": "Accurate, complete, and policy-safe summary of the thread.", "v2_escalated": false, "cache": null }, { "id": "R044", "task": "qa", "input": "Is there a discount for nonprofits?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello and thank you so much for reaching out to Corvid Cloud support. I'm happy to help!\n\nYes, there is a discount for nonprofits. Verified nonprofits and accredited educational institutions receive 30 percent off the Team and Business plans [K32]. The Starter plan isn't included in this discount.\n\nHere is how to apply:\n\n1. Open the discount form.\n2. Submit it with an official document or a domain email that shows", "v1_answer": "Verified nonprofits and accredited educational institutions receive 30 percent off Team and Business plans. Apply via the discount form with an official document or domain email. Approval takes up to 5 business days, and the discount is not retroactive.", "v2_answer": "Verified nonprofits and accredited educational institutions receive 30 percent off Team and Business plans. Apply via the discount form with an official document or domain email. Approval takes up to 5 business days, and the discount is not retroactive.", "fin_answer": "Verified nonprofits and accredited educational institutions receive 30 percent off Team and Business plans. Apply via the discount form with an official document or domain email. Approval takes up to 5 business days, and the discount applies from the next invoice, not retroactively.", "gold": null, "fin_note": "Accurate, complete, and follows all policies.", "v1_note": "Omitted that the discount applies from the next invoice.", "v2_note": "Omitted that the discount applies from the next invoice.", "baseline_note": "Accurate, includes all must-have facts, and adheres to all policies.", "v2_escalated": false, "cache": null }, { "id": "R045", "task": "qa", "input": "How do I set up a webhook and verify it's really from you?", "baseline_pass": 0, "v1_pass": 0, "v2_pass": 0, "fin_pass": 0, "baseline_answer": "Hello and thank you for reaching out to Corvid Cloud support, I'm happy to help!\n\nI don't have the step-by-step webhook setup instructions in the material available to me. I also have no information on how Corvid Cloud signs or authenticates webhook requests, so I can't tell you how to verify that they come from us. I don't want to guess, because a wrong answer on security verification could cause you real problems.\n", "v1_answer": "I do not have information on that. A human colleague can confirm. [escalate to: human]", "v2_answer": "I do not have setup steps for webhooks in the excerpts provided. The policy pack only says webhooks are available on Team and Business plans, are signed with HMAC-SHA256 in the X-Corvid-Signature header, and are retried 5 times over 24 h. A human colleague can confirm setup steps. [escalate to: human]", "fin_answer": "I do not have setup steps for webhooks in the excerpts provided. The policy pack only says webhooks are available on Team and Business plans, are signed with HMAC-SHA256 in the X-Corvid-Signature header, and are retried 5 times over 24 h. A human colleague can confirm setup steps. [escalate to: human]", "gold": null, "fin_note": "Omitted Settings > Integrations > Webhooks and falsely claimed setup location wasn't in the policy.", "v1_note": "Failed to provide the required webhook setup path and signature verification header.", "v2_note": "Omitted Settings > Integrations > Webhooks and falsely claimed setup location wasn't in the policy.", "baseline_note": "Failed to include the required webhook setup path and signature verification header.", "v2_escalated": true, "cache": null }, { "id": "R047", "task": "summarize", "input": "Ticket T-7107 | Customer: Sofia (Starter)\nSofia: I invited 3 guests and now it says I'm over my seat limit.\nAgent: On Starter, guests count as seats and the plan has a 5 seat limit. You have 4 members plus 3 guests = 7.\nSofia: That's silly. Can you make an exception?\nAgent: I can't change limits, bu", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is a summary of ticket T-7107 for your reference.\n\n**Customer and plan**\nThe customer is Sofia, who is on the Starter plan.\n\n**The customer's problem**\nSofia invited three guests to her workspace and then saw a message saying she was over her seat limit. She did not expect the invitations to cause this.\n\n**What was discussed and tried**\nThe agent explained that on Starter, guests consume seats just like m", "v1_answer": "Issue: Starter plan seat limit exceeded (7 seats used: 4 members, 3 guests; limit is 5; guests count on Starter).\nStatus: open, customer deciding on upgrading to Team plan to make guests free and raise the cap to 100.\nNext: Wait for customer response on the Team plan upgrade.", "v2_answer": "Issue: Starter plan seat limit exceeded (7 seats used: 4 members, 3 guests; limit is 5; guests count on Starter).\nStatus: open, customer deciding on upgrading to Team plan to make guests free and raise the cap to 100.\nNext: Wait for customer response on the Team plan upgrade.", "fin_answer": "Issue: T-7107, Starter plan, Sofia invited 3 guests, hit seat limit. 4 members plus 3 guests equals 7, exceeding 5 seat limit because guests count as seats.\nStatus: Open; agent explained Starter has a 5 seat limit where guests count as seats, and stated agents cannot change limits, noting upgrading to Team makes guests free and raises cap to 100.\nNext: Sofia is considering upgrading to the Team plan.", "gold": null, "fin_note": "Accurate, complete, and adheres to all policies.", "v1_note": "Omitted the must-have fact that no exception to the limit is possible.", "v2_note": "Omitted the must-have fact that no exception to the limit is possible.", "baseline_note": "Accurate, complete, and policy-safe summary.", "v2_escalated": false, "cache": null }, { "id": "R049", "task": "summarize", "input": "Ticket T-7109 | Customer: Dana (Team)\nDana: Charged $1,740 on the 1st but we only have 60 seats at $29. Should be $1,740... wait, that's right? We had 62 seats.\nAgent: 60 x $29 = $1,740. Two seats were added on the 20th and prorated on the next invoice.\nDana: Ok so next month will include 2 more.\nAg", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 0, "baseline_answer": "Hello! Here is a summary of ticket T-7109 for your reference.\n\n**Customer and plan:** The customer is Dana, whose workspace is on the Team plan, billed at $29 per seat per month.\n\n**The customer's problem:** Dana contacted support about a charge of $1,740 on the 1st of the month. Dana first believed the workspace had only 60 seats at $29 and wondered whether the amount was right. Dana then realized mid-message that t", "v1_answer": "Issue: Customer questioned $1,740 charge; math confirmed 60 seats at $29 plus two prorated seats added on the 20th.\nStatus: resolved; verified pricing and confirmed next month includes the two additional seats.\nNext: none", "v2_answer": "Issue: Customer questioned $1,740 charge; math confirmed 60 seats at $29 plus two prorated seats added on the 20th.\nStatus: resolved; verified pricing and confirmed next month includes the two additional seats.\nNext: none", "fin_answer": "Issue: Team plan charge of $1,740 on the 1st for 60 seats at $29 was correct because 62 seats existed, with two added on the 20th prorated next invoice.\nStatus: resolved. Agent stated two seats added on the 20th are prorated on the next invoice.\nNext: customer will include 2 more next month.", "gold": null, "fin_note": "Missing the 'no action' part of the status.", "v1_note": "Implies the $1,740 included the two prorated seats, and omits that the two seats are prorated on the next invoice.", "v2_note": "Implies the $1,740 included the two prorated seats, and omits that the two seats are prorated on the next invoice.", "baseline_note": "Accurate, includes all must-have facts, and adheres to policies.", "v2_escalated": false, "cache": null }, { "id": "R052", "task": "qa", "input": "What's the difference between Team and Business?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 1, "fin_pass": 1, "baseline_answer": "Hello, and thank you for reaching out to Corvid Cloud! Team and Business share the core features, but Business adds more capacity, enterprise-level security features and faster support. Here are the main differences.\n\n**Price (per seat per month, billed monthly)** [K01]\n- Team: $29\n- Business: $59\n- Annual billing takes 15 percent off both plans. Prices exclude VAT and sales tax, which are added where required by ", "v1_answer": "Team includes up to 100 seats, unlimited projects, 1 TB, up to 50 guests, and 600 req/min API. Business has no seat cap, 5 TB, unlimited guests, 3,000 req/min API, SSO (SAML 2.0), audit logs (12 months), 99.95% uptime SLA, EU region choice at creation, and weekend sev-1 coverage.", "v2_answer": "Team is $29/seat/month, up to 100 seats, 1 TB, up to 50 guests, API 600 req/min. Business is $59/seat/month, no seat cap, 5 TB, unlimited guests, API 3,000 req/min. Business only: SSO (SAML 2.0), audit logs (12 months), 99.95% uptime SLA, EU region choice, weekend sev-1 coverage.", "fin_answer": "Team is $29/seat/month, up to 100 seats, 1 TB, up to 50 guests, API 600 req/min. Business is $59/seat/month, no seat cap, 5 TB, unlimited guests, API 3,000 req/min. Business only: SSO (SAML 2.0), audit logs (12 months), 99.95% uptime SLA, EU region choice, weekend sev-1 coverage.", "gold": null, "fin_note": "Accurate, includes all must-have facts and additional valid KB details.", "v1_note": "Missing pricing ($29 per seat for Team, $59 per seat for Business).", "v2_note": "Accurate, includes all must-have facts and additional valid KB details.", "baseline_note": "Accurate, complete, and policy-safe.", "v2_escalated": true, "cache": null }, { "id": "R056", "task": "classify", "input": "Is there a way to switch the Kanban board to dark mode?", "baseline_pass": 0, "v1_pass": 0, "v2_pass": 0, "fin_pass": 0, "baseline_answer": "how_to", "v1_answer": "how_to", "v2_answer": "how_to", "fin_answer": "how_to", "gold": "feature_request", "fin_note": null, "v1_note": null, "v2_note": null, "baseline_note": null, "v2_escalated": false, "cache": null }, { "id": "R058", "task": "report", "input": "T-8467 | feature_request | sev3 | resolved | 18h | Gantt dependencies\nT-8468 | security | sev3 | resolved | 25h | Password changed unexpectedly\nT-8472 | bug | sev3 | resolved | 88h | Tasks vanish on refresh\nT-8473 | account_access | sev3 | open | 42h | SSO loop\nT-8476 | account_access | sev3 | open ", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is the nightly account health report for **Granite Builders**. No knowledge base excerpts came with this request, so there are no excerpt IDs to cite. Everything below is calculated directly from the ticket list you provided.\n\n## Executive summary\nGranite Builders raised **27 tickets**, of which **15 are resolved** and **12 are unresolved** (8 open, 4 pending). Two of the unresolved tickets are severity 1", "v1_answer": "Total: 26 tickets\nBy category: feature_request=2, security=2, bug=7, account_access=3, how_to=7, billing=5\nOpen sev1: 1 (status is not resolved)\nLongest open: T-8498 (81h, unresolved only)\nRisks: Unresolved billing sev1 and growing authentication/access tickets threaten core operations.\nAction: Immediately resolve T-8521 duplicate charge and address SSO/2FA access blockages.", "v2_answer": "Total: 26 tickets\nBy category: feature_request=2, security=2, bug=7, account_access=3, how_to=7, billing=5\nOpen sev1: 1 (status is not resolved)\nLongest open: T-8498 (81h, unresolved only)\nRisks: Unresolved billing sev1 and growing authentication/access tickets threaten core operations.\nAction: Immediately resolve T-8521 duplicate charge and address SSO/2FA access blockages.", "fin_answer": "Total: 27 tickets\nBy category: billing=6, bug=7, how_to=7, feature_request=2, account_access=3, security=2\nOpen sev1: 2\nLongest open: T-8498 (81h)\nRisks: Two unresolved sev1 tickets (including billing and account/access issues) threaten account stability.\nAction: Immediately resolve open sev1 tickets T-8521 and T-8503, then address long-standing access and bug tickets.", "gold": null, "fin_note": "All must-have facts and counts are present and accurate.", "v1_note": "Incorrect counts for total tickets (26 vs 27), billing (5 vs 6), and unresolved sev1 (1 vs 2).", "v2_note": "Incorrect counts for total tickets (26 vs 27), billing (5 vs 6), and unresolved sev1 (1 vs 2).", "baseline_note": "All counts, categories, and ticket details are accurate, complete, and policy-safe.", "v2_escalated": false, "cache": null }, { "id": "R069", "task": "report", "input": "T-8337 | bug | sev3 | resolved | 11h | Webhook not retried\nT-8339 | how_to | sev3 | open | 5h | Bulk import\nT-8343 | bug | sev3 | open | 38h | Calendar sync lag\nT-8344 | bug | sev3 | pending | 25h | Tasks vanish on refresh\nT-8348 | bug | sev3 | open | 80h | Mobile crash on upload\nT-8350 | account_ac", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is tonight's account health report for **Eastgate Retail**.\n\n# Nightly Account Health Report: Eastgate Retail\n\n## 1. Executive summary\nEastgate Retail had **26 tickets** in this period, and only **4 (15%) are resolved**. All **4 severity-1 tickets are unresolved** (2FA lockout, card declined, two proration questions), and the oldest, T-8356, has been open for 92 hours. Billing is the biggest pressure poin", "v1_answer": "Total: 26 tickets\nBy category: bug=10, how_to=6, account_access=3, billing=5, security=1, feature_request=3\nOpen sev1: 3\nLongest open: T-8356 (92h, unresolved only)\nRisks: High volume of unresolved billing and critical access tickets stalling account operations.\nAction: Escalate Sev1 billing and 2FA lockout issues to support leads immediately.", "v2_answer": "Total: 26 tickets\nBy category: bug=10, how_to=6, account_access=3, billing=5, security=1, feature_request=3\nOpen sev1: 3\nLongest open: T-8356 (92h, unresolved only)\nRisks: High volume of unresolved billing and critical access tickets stalling account operations.\nAction: Escalate Sev1 billing and 2FA lockout issues to support leads immediately.", "fin_answer": "Total: 26 tickets\nBy category: billing=6, bug=8, how_to=5, feature_request=3, account_access=3, security=1\nOpen sev1: 4\nLongest open: T-8356 (92h)\nRisks: Four unresolved severity-1 tickets including billing and 2FA lockouts pose critical operational risks.\nAction: Immediately assign senior support to resolve the pending sev1 billing and 2FA tickets.", "gold": null, "fin_note": "All counts and facts match the reference requirements.", "v1_note": "Wrong category counts (bug=10, how_to=6, billing=5) and wrong unresolved sev1 count (3 instead of 4).", "v2_note": "Wrong category counts (bug=10, how_to=6, billing=5) and wrong unresolved sev1 count (3 instead of 4).", "baseline_note": "All counts, facts, and KB references are accurate and complete.", "v2_escalated": false, "cache": null }, { "id": "R071", "task": "report", "input": "T-8002 | bug | sev1 | resolved | 13h | Webhook not retried\nT-8003 | account_access | sev2 | open | 54h | Reset link expired\nT-8005 | billing | sev3 | resolved | 16h | Invoice VAT ID missing\nT-8006 | how_to | sev3 | resolved | 6h | Find audit log\nT-8008 | bug | sev2 | open | 40h | Calendar sync lag\nT", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is the nightly account health report for Acme Freight, built only from the ticket list you provided.\n\n## Executive summary\nAcme Freight's account health is **at risk**. Of 31 tickets, only 12 (39%) are resolved, and 19 are still open or pending. One severity-1 ticket is unresolved (T-8045, calendar sync lag, 48h). Three security-related tickets (two reports of a leaked token and one unexpected password ch", "v1_answer": "Total: 30 tickets\nBy category: bug=7, account_access=5, billing=7, how_to=9, security=2\nOpen sev1: 1 (status is not resolved)\nLongest open: T-8059 (82h, unresolved only)\nRisks: High volume of unresolved billing and access issues threatening customer satisfaction.\nAction: Immediately triage pending billing and sev1 bug tickets.", "v2_answer": "Total: 30 tickets\nBy category: bug=7, account_access=5, billing=7, how_to=9, security=2\nOpen sev1: 1 (status is not resolved)\nLongest open: T-8059 (82h, unresolved only)\nRisks: High volume of unresolved billing and access issues threatening customer satisfaction.\nAction: Immediately triage pending billing and sev1 bug tickets.", "fin_answer": "Total: 31 tickets\nBy category: billing=7, bug=7, how_to=9, feature_request=0, account_access=5, security=3\nOpen sev1: 1\nLongest open: T-8059 (82h)\nRisks: Active sev1 bug and prolonged unresolved billing and access inquiries threaten customer satisfaction.\nAction: Prioritize dispatch for open sev1 bug T-8045 and review aging pending billing tickets.", "gold": null, "fin_note": "All counts, facts, and ticket references are accurate and complete.", "v1_note": "Incorrect total (30 instead of 31), security count (2 instead of 3), and missing feature_request=0.", "v2_note": "Incorrect total (30 instead of 31), security count (2 instead of 3), and missing feature_request=0.", "baseline_note": "All counts, categories, and ticket details are accurate and complete.", "v2_escalated": false, "cache": null }, { "id": "R073", "task": "report", "input": "T-8249 | bug | sev1 | resolved | 58h | Webhook not retried\nT-8253 | bug | sev3 | open | 75h | Export hangs\nT-8255 | feature_request | sev2 | pending | 78h | Custom fields\nT-8258 | account_access | sev3 | open | 63h | Cannot see Billing tab\nT-8259 | billing | sev3 | resolved | 52h | Proration questio", "baseline_pass": 0, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is tonight's account health report for **Dunmore Legal**.\n\n# Nightly Account Health Report: Dunmore Legal\n\n## 1. Executive summary\n**Health: At risk (my assessment).** There are 33 tickets, and 17 (52%) are still unresolved. Two of the four severity-1 tickets are unresolved and have been open for 76h and 84h. A cluster of five \"Export hangs\" tickets (all unresolved) points to one underlying problem that i", "v1_answer": "Total: 32 tickets\nBy category: bug=12, feature_request=2, account_access=5, billing=5, how_to=6, security=2\nOpen sev1: 1 (status is not resolved)\nLongest open: T-8267 (76h, unresolved only)\nRisks: Sev1 export hangs unaddressed for 76 hours alongside multiple lingering bug and billing tickets.\nAction: Immediately escalate pending sev1 export ticket T-8267 and review aging open bug queue.", "v2_answer": "Total: 32 tickets\nBy category: bug=12, feature_request=2, account_access=5, billing=5, how_to=6, security=2\nOpen sev1: 1 (status is not resolved)\nLongest open: T-8267 (76h, unresolved only)\nRisks: Sev1 export hangs unaddressed for 76 hours alongside multiple lingering bug and billing tickets.\nAction: Immediately escalate pending sev1 export ticket T-8267 and review aging open bug queue.", "fin_answer": "Total: 33 tickets\nBy category: billing=5, bug=12, how_to=7, feature_request=2, account_access=5, security=2\nOpen sev1: 2\nLongest open: T-8336 (84h)\nRisks: Two unresolved sev1 tickets remain active, including a long-pending bulk import issue and a sev1 export hang.\nAction: Immediately escalate unresolved sev1 tickets to engineering.", "gold": null, "fin_note": "All counts and facts match the reference data accurately.", "v1_note": "Wrong total, how_to count, unresolved sev1 count, and longest-open ticket (missed T-8336).", "v2_note": "Wrong total, how_to count, unresolved sev1 count, and longest-open ticket (missed T-8336).", "baseline_note": "Bug resolved/unresolved count is wrong (6 resolved, 6 unresolved; answer says 7 and 5).", "v2_escalated": false, "cache": null }, { "id": "R075", "task": "qa", "input": "Our uptime was 99.7% last month on Business. What SLA credit do we get?", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 1, "fin_pass": 1, "baseline_answer": "Hello, and thank you for contacting Corvid Cloud. I'm sorry your workspace had less uptime than you should expect on the Business plan.\n\n**How the SLA credit works**\n\nBusiness workspaces have a 99.95 percent monthly uptime SLA. If uptime falls below that, the workspace is eligible for a service credit of 10 percent of that month's fee for each *full* 0.1 percent below the target, up to a maximum of 30 percent [K23", "v1_answer": "For monthly uptime falling below 99.95%, Business workspaces are eligible for a service credit of 10% of that month's fee for each full 0.1% below the target, up to 30%. Credits must be requested within 30 days.", "v2_answer": "99.95% − 99.7% = 0.25%, which is 2 full 0.1% steps, so the credit is 2 × 10% = 20% of that month's fee (max 30%). Request it within 30 days. I cannot approve it; Billing will review your request. [escalate to: billing]", "fin_answer": "99.95% − 99.7% = 0.25%, which is 2 full 0.1% steps, so the credit is 2 × 10% = 20% of that month's fee (max 30%). Request it within 30 days. I cannot approve it; Billing will review your request. [escalate to: billing]", "gold": null, "fin_note": "Accurate calculation and includes all required SLA details without making unauthorized promises.", "v1_note": "Missing the specific 20% credit calculation for 99.7% uptime.", "v2_note": "Accurate calculation and includes all required SLA details without making unauthorized promises.", "baseline_note": "Accurate calculation and includes all required SLA facts without promising a credit.", "v2_escalated": false, "cache": null }, { "id": "R078", "task": "report", "input": "T-8536 | how_to | sev3 | resolved | 38h | Set up Slack\nT-8537 | account_access | sev1 | open | 31h | 2FA locked out\nT-8539 | feature_request | sev2 | resolved | 43h | Custom fields\nT-8541 | bug | sev3 | resolved | 86h | Calendar sync lag\nT-8545 | how_to | sev3 | open | 4h | Create template\nT-8547 | ", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is the nightly account health report for **Harbor Studio**, based on the 30 tickets provided.\n\n## Executive summary\nHarbor Studio's account health is **at risk**. Of 30 tickets, 16 (53%) are still unresolved, including **two open severity-1 account-access tickets** (a 2FA lockout and a missing Billing tab) that have been open for 31h and 34h. Seven of the ten severity-2 tickets are also unresolved. Five u", "v1_answer": "Total: 30 tickets\nBy category: how_to=9, account_access=4, feature_request=5, bug=8, billing=3, security=1\nOpen sev1: 2\nLongest open: T-8547 (80h, unresolved only)\nRisks: Two unresolved sev1 access tickets and multiple aging open bugs (Export hangs, Calendar sync) threaten client retention.\nAction: Immediately assign senior support to resolve both sev1 tickets and triage the open export and sync bugs.", "v2_answer": "Total: 30 tickets\nBy category: how_to=9, account_access=4, feature_request=5, bug=8, billing=3, security=1\nOpen sev1: 2\nLongest open: T-8547 (80h, unresolved only)\nRisks: Two unresolved sev1 access tickets and multiple aging open bugs (Export hangs, Calendar sync) threaten client retention.\nAction: Immediately assign senior support to resolve both sev1 tickets and triage the open export and sync bugs.", "fin_answer": "Total: 30 tickets\nBy category: billing=3, bug=8, how_to=8, feature_request=5, account_access=5, security=1\nOpen sev1: 2\nLongest open: T-8554 (82h)\nRisks: Two critical severity-1 account access lockouts remain unresolved and need immediate intervention.\nAction: Escalate the sev1 tickets to the security team immediately.", "gold": null, "fin_note": "All required counts and ticket IDs match the reference facts.", "v1_note": "Incorrect category counts (how_to=9, account_access=4) and wrong longest-open ticket (T-8547 instead of T-8554/T-8583).", "v2_note": "Incorrect category counts (how_to=9, account_access=4) and wrong longest-open ticket (T-8547 instead of T-8554/T-8583).", "baseline_note": "All counts, categories, and ticket details are accurate, complete, and policy-safe.", "v2_escalated": false, "cache": null }, { "id": "R080", "task": "summarize", "input": "Ticket T-7111 | Customer: Grace (Starter)\nGrace: Where is the template gallery? I only see 12 templates.\nAgent: Starter has the 12 built-in templates. Custom templates and the shared gallery are on Team and Business.\nGrace: Got it, thanks.\nStatus: resolved.", "baseline_pass": 1, "v1_pass": 0, "v2_pass": 0, "fin_pass": 1, "baseline_answer": "Hello! Here is a summary of ticket T-7111 for your reference.\n\n**Customer and context:** The ticket was raised by Grace, a customer on the Starter plan.\n\n**The customer's problem:** Grace asked where to find the template gallery. She said she could see only 12 templates and seemed to expect more.\n\n**What was tried:** No troubleshooting was needed. The agent explained the plan difference in a single reply. Starter inc", "v1_answer": "Issue: Starter plan customer Grace couldn't find the shared template gallery, restricted to Team and Business plans.\nStatus: Resolved; agent informed customer that Starter includes only 12 built-in templates.\nNext: None", "v2_answer": "Issue: Starter plan customer Grace couldn't find the shared template gallery, restricted to Team and Business plans.\nStatus: Resolved; agent informed customer that Starter includes only 12 built-in templates.\nNext: None", "fin_answer": "Issue: Grace on Starter asked about template gallery location, seeing only 12 templates. Cause: Starter includes 12 built-in templates, whereas custom templates and shared gallery are on Team and Business.\nStatus: Resolved. Agent clarified Starter limit of 12 built-in templates and higher-tier requirements.\nNext: None.", "gold": null, "fin_note": "Accurate, complete, and adheres to all rules.", "v1_note": "Omits that custom templates also require Team or Business plans.", "v2_note": "Omits that custom templates also require Team or Business plans.", "baseline_note": "Accurate, complete, and adheres to all guidelines.", "v2_escalated": false, "cache": null } ], "judge": "gemini-4-argon-high", "retrieval": { "questions": 36, "all_gold_in_top10_naive": 0.944, "all_gold_in_top3_reranked": 0.889, "any_gold_in_top3_reranked": 0.917 } }, "tokens_by_task": { "qa": { "baseline": { "prompt": 4363, "completion": 664 }, "trim": { "prompt": 2246, "completion": 463 }, "caps": { "prompt": 2198, "completion": 113 }, "v2": { "prompt": 1588, "completion": 70 } }, "classify": { "baseline": { "prompt": 3207, "completion": 647 }, "trim": { "prompt": 484, "completion": 366 }, "caps": { "prompt": 452, "completion": 4 }, "v2": { "prompt": 285, "completion": 2 } }, "summarize": { "baseline": { "prompt": 3331, "completion": 1082 }, "trim": { "prompt": 331, "completion": 801 }, "caps": { "prompt": 292, "completion": 131 }, "v2": { "prompt": 194, "completion": 63 } }, "report": { "baseline": { "prompt": 4100, "completion": 5092 }, "trim": { "prompt": 1082, "completion": 1068 }, "caps": { "prompt": 1031, "completion": 641 }, "v2": { "prompt": 831, "completion": 110 } } }, "estimator_basis": { "trim_prompt_cut": 0.655, "caps_output_cut": 0.751, "baseline_avg_prompt": 3842, "baseline_avg_output": 1158, "trim_avg_prompt": 1326, "caps_avg_output": 136, "note": "Measured across all 81 requests (token totals from the API usage field). The estimator's high scenario uses these; the low scenario is a cautious haircut." }, "dedupe": { "exact": 3, "normalized": 5, "requests": 81, "calls_after_dedupe": 77 }, "pricing": { "fetched": "2026-11-11", "sources": { "anthropic": "https://platform.claude.com/docs/en/about-claude/pricing", "openai": "https://developers.openai.com/api/docs/pricing", "google": "https://aistudio.google.com/docs/pricing" }, "families": { "anthropic": { "label": "Anthropic", "big": { "model": "Claude Sonnet 5.5", "input": 2.0, "cached_input": 0.1, "cache_write": 2.5, "output": 10.0 }, "small": { "model": "Claude Haiku 5.5 (prompts up to 100k)", "input": 0.1, "cached_input": 0.01, "cache_write": 0.125, "output": 0.5 } }, "openai": { "label": "OpenAI", "big": { "model": "gpt-6.1-sol", "input": 2.0, "cached_input": 0.1, "output": 10.0 }, "small": { "model": "gpt-6-luna", "input": 0.1, "cached_input": 0.01, "output": 0.5 } }, "google": { "label": "Google", "big": { "model": "gemini-4-carbon", "input": 2.0, "cached_input": 0.1, "output": 10.0 }, "small": { "model": "gemini-3.1-flash-lite", "input": 0.25, "cached_input": 0.025, "output": 1.5 } } }, "batch_factor": 0.5, "notes": [ "Prices are the public list prices read from the official pages on the fetched date. Cached-input = cache read / cache hit rate.", "Anthropic: Batch API is 50 percent off input and output (stated on the page). Cache write (5 minute) premium is 1.25x input.", "OpenAI: the page does not print a batch percentage; its Batch prices are half of Standard in the examples it lists (e.g. gpt-6-astra input 5.00 vs 10.00), so 0.5 is used. OpenAI has no cache-write fee.", "Google: Standard, context-cache and Batch prices are printed per model; batch is exactly half. Context-cache storage fees are not modelled. Gemini 4 Carbon list prices rise on 2026-12-01 to input 4.00 / cache 0.20 / output 20.00; the table uses the prices valid on the fetch date.", "Role mapping is an assumption: the experiment ran on a proxy, so each family's 'big' and 'small' tier is the commercial model of a comparable class. Token counts are the real counts from the proxy models; the same counts are priced under each family." ] } }