{ "schema_version": 6, "note": "Built by scripts/build_model_index.mjs from the published page payloads. No numbers of its own.", "counts": { "cybt_runs": 97, "botsv3_runs": 56, "models": 57 }, "pages": [ { "key": "home", "href": "index.html", "title": "Leaderboard" }, { "key": "open", "href": "open-source.html", "title": "Open weights" }, { "key": "astra", "href": "astra-cybt.html", "title": "Astra & Fable 5.1" }, { "key": "fable", "href": "cybt-ctf.html", "title": "Fable, unlocked" }, { "key": "anthropic5", "href": "cybt-ctf-anthropic5.html", "title": "Fable / Opus / Sonnet 5" }, { "key": "claude55", "href": "cybt-ctf-claude55.html", "title": "Claude 5.5 family" }, { "key": "gpt56", "href": "cybt-ctf-gpt56.html", "title": "GPT-5.6 family" }, { "key": "glm53", "href": "cybt-ctf-glm53.html", "title": "GLM 5.3" }, { "key": "glm", "href": "cybt-ctf-glm.html", "title": "GLM 5.2" }, { "key": "refusals", "href": "refusals.html", "title": "Refusals" }, { "key": "hygiene", "href": "contamination.html", "title": "BOTSv3 hygiene" }, { "key": "cheating", "href": "cheating.html", "title": "Cheating & hacking" }, { "key": "ttp", "href": "ttp.html", "title": "Cheats & attacks" }, { "key": "agreement", "href": "agreement-audit.html", "title": "Agreement audit" } ], "models": [ { "key": "gpt-5.6-sol", "name": "GPT-5.6 Sol", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-07-09", "year": 2026, "year_basis": "release", "card": { "src": "assets/gpt56-field-card.webp", "title": null, "family": true }, "discussed": [ "open", "gpt56" ], "listed": [ "home", "astra", "anthropic5", "claude55", "glm53", "refusals", "hygiene", "agreement" ], "ranks": { "cybt": { "place": 1, "tied": false, "of": 49, "score": 45 }, "cybt_open": null, "cybt_proprietary": { "place": 1, "tied": false, "of": 21, "score": 45 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GPT-5.6 Sol" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": "medium", "setting": "OpenAI · search MCP", "solved": 45, "total": 59, "rank": 1, "place": "1", "tied": false, "held": false, "reported": true, "cost": 79.56, "cost_mark": null, "cost_note": null, "cost_label": "$79.56", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.768, "solved_sec": 8820, "median_sec": 196, "run_date": null, "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 34, "total": 59, "rank": 20, "place": "20", "tied": true, "held": false, "reported": false, "cost": 50.5723, "cost_mark": null, "cost_note": null, "cost_label": "$50.57", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.4874, "solved_sec": 3378.7, "median_sec": 82.3, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "low", "setting": "OpenAI · low effort", "solved": 19, "total": 59, "rank": 65, "place": "65", "tied": true, "held": false, "reported": false, "cost": 34.391024, "cost_mark": null, "cost_note": null, "cost_label": "$34.39", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.810054, "solved_sec": 1071.821, "median_sec": 52.519, "run_date": "2026-07-15", "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "max", "setting": "OpenAI · max effort", "solved": 18, "total": 59, "rank": 69, "place": "69", "tied": true, "held": false, "reported": false, "cost": 33.183788, "cost_mark": null, "cost_note": null, "cost_label": "$33.18", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.843544, "solved_sec": 1007.714, "median_sec": 47.981, "run_date": "2026-07-10", "effort_label": "max", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "none", "setting": "OpenAI · explicit none effort", "solved": 17, "total": 59, "rank": 73, "place": "73", "tied": true, "held": false, "reported": false, "cost": 43.384404, "cost_mark": null, "cost_note": null, "cost_label": "$43.38", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.552024, "solved_sec": 1219.612, "median_sec": 64.233, "run_date": "2026-07-15", "effort_label": "none", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "medium", "setting": "OpenAI · medium effort", "solved": 16, "total": 59, "rank": 77, "place": "77", "tied": true, "held": false, "reported": false, "cost": 34.66731, "cost_mark": null, "cost_note": null, "cost_label": "$34.67", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.166707, "solved_sec": 740.292, "median_sec": 34.38, "run_date": "2026-07-10", "effort_label": "medium", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-5.6 Sol", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": 0.01694915254237288, "reviewed": true, "note": "Named this dataset's own hostname with no tools at all (1 of 59). Its other no-tools answers are public documentation and are not counted.", "no_tools_rate": 0.033898, "with_tools_rate": null }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "none effort", "tools": "with tools", "refusals": 0, "abstained": 19, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "medium effort", "tools": "with tools", "refusals": 0, "abstained": 19, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "high effort", "tools": "with tools", "refusals": 0, "abstained": 16, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "max effort", "tools": "with tools", "refusals": 0, "abstained": 16, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "low effort", "tools": "with tools", "refusals": 0, "abstained": 15, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Louie.ai", "variant": "medium · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "newer" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-6-astra", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "opus-4.8", "name": "Claude Opus 4.8", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-05-28", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "gpt56", "glm53", "agreement" ], "listed": [ "home", "open", "astra", "fable", "anthropic5", "claude55", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 2, "tied": false, "of": 49, "score": 44 }, "cybt_open": null, "cybt_proprietary": { "place": 2, "tied": false, "of": 21, "score": 44 }, "botsv3": { "place": 6, "tied": true, "of": 33, "score": 38 }, "botsv3_open": null, "botsv3_proprietary": { "place": 6, "tied": false, "of": 16, "score": 38 } }, "aliases": [ "Claude Opus 4.8", "Claude Opus 4.8 (opencode)", "Opus 4.8" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": "Standard", "setting": "Bedrock / search MCP", "solved": 44, "total": 59, "rank": 2, "place": "2", "tied": false, "held": false, "reported": true, "cost": 132.59, "cost_mark": null, "cost_note": null, "cost_label": "$132.59", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 3.013409, "solved_sec": 4972, "median_sec": 113, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk curl", "solved": 40, "total": 59, "rank": 5, "place": "5", "tied": true, "held": false, "reported": false, "cost": 33.415119, "cost_mark": null, "cost_note": null, "cost_label": "$33.42", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.835378, "solved_sec": 3069.586, "median_sec": 57.543, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock · Splunk curl", "solved": 31, "total": 59, "rank": 34, "place": "34", "tied": true, "held": false, "reported": false, "cost": 41.401538, "cost_mark": null, "cost_note": null, "cost_label": "$41.40", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.335533, "solved_sec": 2861.487, "median_sec": 83.772, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "Standard", "setting": "Anthropic.com / search curl", "solved": 31, "total": 59, "rank": 34, "place": "34", "tied": true, "held": false, "reported": false, "cost": 35.717449, "cost_mark": null, "cost_note": null, "cost_label": "$35.72", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.152176, "solved_sec": 3394.01, "median_sec": 90.965, "run_date": "2026-06-01", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Bedrock / search MCP", "solved": 29, "total": 59, "rank": 41, "place": "41", "tied": true, "held": false, "reported": false, "cost": 42.261907, "cost_mark": null, "cost_note": null, "cost_label": "$42.26", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.457307, "solved_sec": 2761.572, "median_sec": 85.405, "run_date": "2026-06-01", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": null, "setting": null, "solved": 27, "total": 59, "rank": 47, "place": "47", "tied": true, "held": false, "reported": true, "cost": 35, "cost_mark": null, "cost_note": null, "cost_label": "$35.00", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.296296, "solved_sec": 3882, "median_sec": 234, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": null, "setting": "Splunk MCP", "solved": 26, "total": 59, "rank": 51, "place": "51", "tied": true, "held": false, "reported": false, "cost": 43.13794, "cost_mark": null, "cost_note": null, "cost_label": "$43.14", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.659152, "solved_sec": 1303.807, "median_sec": 43.004, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "Claude Code", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 38, "total": 59, "rank": 7, "place": "7", "tied": false, "held": false, "reported": false, "cost": 14.934354, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.393009, "solved_sec": 2463.194, "median_sec": 35.6625, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 29, "total": 59, "rank": 36, "place": "36", "tied": false, "held": false, "reported": false, "cost": 10.480031, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.36138, "solved_sec": 1638.48, "median_sec": 38.379, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Claude Opus 4.8", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": 0.01694915254237288, "reviewed": true, "note": "Named this dataset's own hostname with no tools at all (1 of 59). Its other no-tools answers are public documentation and are not counted.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.576271186440678 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Louie.ai", "variant": "Standard · Bedrock / Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Louie.ai", "variant": null, "tools": null, "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "Standard · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "Standard · Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Anthropic / Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "opus-4.7", "run": 0, "reasons": [ "previous" ] }, { "key": "opus-5", "run": 0, "reasons": [ "newer" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western", "cheaper", "better_for_cost" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.5", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-6-astra", "name": "GPT-6 Astra", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-09-03", "year": 2026, "year_basis": "release", "card": { "src": "assets/gpt-6-astra-field-card.webp", "title": "the Star Archivist", "family": false }, "discussed": [ "home", "astra" ], "listed": [ "open", "claude55" ], "ranks": { "cybt": { "place": 3, "tied": true, "of": 49, "score": 41 }, "cybt_open": null, "cybt_proprietary": { "place": 3, "tied": true, "of": 21, "score": 41 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GPT-6 Astra" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 41, "total": 59, "rank": 3, "place": "3", "tied": true, "held": false, "reported": false, "cost": 95.8664, "cost_mark": null, "cost_note": null, "cost_label": "$95.87", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.3382, "solved_sec": 2641.5, "median_sec": 51.7, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "Splunk MCP", "solved": 39, "total": 59, "rank": 7, "place": "7", "tied": true, "held": false, "reported": false, "cost": 73.6105, "cost_mark": null, "cost_note": null, "cost_label": "$73.61", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.8874, "solved_sec": 3357.8, "median_sec": 81.2, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "low", "setting": "OpenAI · Splunk MCP", "solved": 31, "total": 59, "rank": 34, "place": "34", "tied": true, "held": false, "reported": false, "cost": 63.5034, "cost_mark": null, "cost_note": null, "cost_label": "$63.50", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.0485, "solved_sec": 2020.9, "median_sec": 41.2, "run_date": null, "effort_label": "low", "effort_note": "Effort setting recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary", "top_western", "better_for_cost" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "cheaper" ] }, { "key": "gpt-5.5", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-6.1-sol", "name": "GPT-6.1 Sol", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-09-29", "year": 2026, "year_basis": "release", "card": { "src": "assets/gpt-6-1-sol-field-card.webp", "title": "the Sunlit Sleuth", "family": false }, "discussed": [], "listed": [ "home", "claude55" ], "ranks": { "cybt": { "place": 3, "tied": true, "of": 49, "score": 41 }, "cybt_open": null, "cybt_proprietary": { "place": 3, "tied": true, "of": 21, "score": 41 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GPT-6.1 Sol" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 41, "total": 59, "rank": 3, "place": "3", "tied": true, "held": false, "reported": false, "cost": 14.6679952, "cost_mark": null, "cost_note": null, "cost_label": "$14.67", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.3577559804878049, "solved_sec": 3571.525, "median_sec": 70.162, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "previous", "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-6-astra", "run": 0, "reasons": [ "faster" ] }, { "key": "sonnet-5.5", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "reflection-beam-1.0", "name": "Reflection Beam 1.0", "maker": "Reflection AI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2026-10-16", "year": 2026, "year_basis": "release", "card": { "src": "assets/reflection-beam-1-0-field-card.webp", "title": "the Mirror Knight", "family": false }, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 5, "tied": false, "of": 49, "score": 40 }, "cybt_open": { "place": 1, "tied": false, "of": 28, "score": 40 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Reflection Beam 1.0" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "W&B / Splunk MCP, history kept", "solved": 40, "total": 59, "rank": 5, "place": "5", "tied": true, "held": false, "reported": false, "cost": 6.673662, "cost_mark": null, "cost_note": null, "cost_label": "$6.67", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.166842, "solved_sec": 2427.256, "median_sec": 51.863, "run_date": "2026-10-20", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · W&B / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "sonnet-5.5", "name": "Claude Sonnet 5.5", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-09-28", "year": 2026, "year_basis": "release", "card": { "src": "assets/claude-sonnet-5-5-field-card.webp", "title": "the Measured Bard", "family": false }, "discussed": [ "claude55" ], "listed": [ "home", "hygiene" ], "ranks": { "cybt": { "place": 6, "tied": true, "of": 49, "score": 39 }, "cybt_open": null, "cybt_proprietary": { "place": 5, "tied": true, "of": 21, "score": 39 }, "botsv3": { "place": 2, "tied": true, "of": 33, "score": 41 }, "botsv3_open": null, "botsv3_proprietary": { "place": 2, "tied": true, "of": 16, "score": 41 } }, "aliases": [ "Claude Sonnet 5.5", "Claude Sonnet 5.5 (opencode)", "Sonnet 5.5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "medium", "setting": "Bedrock · Splunk MCP", "solved": 39, "total": 59, "rank": 7, "place": "7", "tied": true, "held": false, "reported": false, "cost": 10.06, "cost_mark": null, "cost_note": null, "cost_label": "$10.06", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.26, "solved_sec": 1361.708, "median_sec": 26.198, "run_date": null, "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "low", "setting": "Bedrock · Splunk MCP", "solved": 37, "total": 59, "rank": 11, "place": "11", "tied": true, "held": false, "reported": false, "cost": 9.23, "cost_mark": null, "cost_note": null, "cost_label": "$9.23", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.25, "solved_sec": 1407.67, "median_sec": 24.761, "run_date": null, "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "Bedrock · Splunk MCP", "solved": 37, "total": 59, "rank": 11, "place": "11", "tied": true, "held": false, "reported": false, "cost": 14.32, "cost_mark": null, "cost_note": null, "cost_label": "$14.32", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.39, "solved_sec": 2055.549, "median_sec": 28.987, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "medium", "setting": "Splunk MCP", "solved": 31, "total": 59, "rank": 34, "place": "34", "tied": true, "held": false, "reported": false, "cost": 9.18, "cost_mark": null, "cost_note": null, "cost_label": "$9.18", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.3, "solved_sec": 884.096, "median_sec": 22.186, "run_date": null, "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "low", "setting": "Splunk MCP", "solved": 23, "total": 59, "rank": 57, "place": "57", "tied": true, "held": false, "reported": false, "cost": 8.92, "cost_mark": null, "cost_note": null, "cost_label": "$8.92", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.39, "solved_sec": 677.956, "median_sec": 21.771, "run_date": null, "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Splunk MCP", "solved": 23, "total": 59, "rank": 57, "place": "57", "tied": true, "held": false, "reported": false, "cost": 12.5, "cost_mark": null, "cost_note": null, "cost_label": "$12.50", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.54, "solved_sec": 854.434, "median_sec": 19.355, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "High", "setting": "MCP / 300s", "solved": 41, "total": 59, "rank": 3, "place": "3", "tied": false, "held": false, "reported": false, "cost": 6.391961, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.15590148780487806, "solved_sec": 1274.391, "median_sec": 17.402, "run_date": "2026-10-17", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "High", "setting": "MCP / Literal / 300s", "solved": 35, "total": 59, "rank": 19, "place": "19", "tied": false, "held": false, "reported": false, "cost": 6.054763, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.17299322857142857, "solved_sec": 1193.747, "median_sec": 22.901, "run_date": "2026-10-02", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Sonnet 5.5", "verdict": "Needs review", "tone": "partial", "confirmed_rate": null, "reviewed": false, "note": "Answered 6 of 59 with no tools. A per-question review found strong instance-specific signals in 4 of those answers. That is an answer-level signal, not a measured contamination rate.", "no_tools_rate": 0.1016949152542373, "with_tools_rate": 0.5932203389830508 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "sonnet-5", "run": 0, "reasons": [ "previous" ] }, { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "cheaper", "better_for_cost" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "sonnet-4.5", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-5.6-terra", "name": "GPT-5.6 Terra", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-07-09", "year": 2026, "year_basis": "release", "card": { "src": "assets/gpt56-field-card.webp", "title": null, "family": true }, "discussed": [ "gpt56", "agreement" ], "listed": [ "home", "astra", "anthropic5", "claude55", "glm53", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 6, "tied": true, "of": 49, "score": 39 }, "cybt_open": null, "cybt_proprietary": { "place": 5, "tied": true, "of": 21, "score": 39 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GPT-5.6 Terra" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 39, "total": 59, "rank": 7, "place": "7", "tied": true, "held": false, "reported": false, "cost": 22.4324, "cost_mark": null, "cost_note": null, "cost_label": "$22.43", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.5752, "solved_sec": 2654.9, "median_sec": 69.2, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": "medium", "setting": "OpenAI · search MCP", "solved": 34, "total": 59, "rank": 20, "place": "20", "tied": true, "held": false, "reported": true, "cost": 37.31, "cost_mark": null, "cost_note": null, "cost_label": "$37.31", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.097353, "solved_sec": 3638, "median_sec": 107, "run_date": null, "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "medium", "setting": "OpenAI · medium effort · adjusted (Splunk-lockout false positive cleared)", "solved": 32, "total": 59, "rank": 30, "place": "30", "tied": true, "held": false, "reported": false, "cost": 21.802629, "cost_mark": null, "cost_note": null, "cost_label": "$21.80", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.681332, "solved_sec": 2094.915, "median_sec": 52.665, "run_date": "2026-07-10", "effort_label": "medium", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-5.6 Terra", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 3 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.050847, "with_tools_rate": null }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "high effort", "tools": "with tools", "refusals": 0, "abstained": 5, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "medium effort", "tools": "with tools", "refusals": 0, "abstained": 2, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Louie.ai", "variant": "medium · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newer", "newest_western" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "cheaper" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "deepseek-v4-pro", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "glm-5.3-flash", "name": "GLM 5.3 Flash", "maker": "Z.ai", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-08-26", "year": 2026, "year_basis": "release", "card": { "src": "assets/glm-5-3-flash-field-card.webp", "title": "the Goblin Duelist", "family": false }, "discussed": [ "home", "open", "glm53", "glm", "agreement" ], "listed": [ "astra", "claude55", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 8, "tied": false, "of": 49, "score": 38 }, "cybt_open": { "place": 2, "tied": false, "of": 28, "score": 38 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GLM 5.3 Flash" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": null, "setting": "Splunk MCP", "solved": 38, "total": 59, "rank": 10, "place": "10", "tied": false, "held": false, "reported": true, "cost": 4.74, "cost_mark": null, "cost_note": null, "cost_label": "$4.74", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.1247, "solved_sec": 12654, "median_sec": 298, "run_date": "2026-08-31", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history kept", "solved": 32, "total": 59, "rank": 30, "place": "30", "tied": true, "held": false, "reported": false, "cost": 2.608328, "cost_mark": null, "cost_note": null, "cost_label": "$2.61", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.08151, "solved_sec": 3729.833, "median_sec": 112.558, "run_date": "2026-09-01", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 2, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GLM 5.3 Flash", "verdict": "Reviewed: nothing from memory", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 3 of 59 with no tools. Each was read against its question: one is a scorer artifact (the expected 8 matched inside 228), one is an advertising trope, one is a guess from a set of about six. None recalls this dataset.", "no_tools_rate": 0.05084745762711865, "with_tools_rate": 0.4407 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Splunk MCP · opencode 1.18.25", "tools": "with tools", "refusals": 0, "abstained": 0, "total": 59 }, { "benchmark": "BOTSv3", "harness": "opencode", "variant": "Splunk MCP · opencode 1.18.25", "tools": "with tools", "refusals": 0, "abstained": 0, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "glm-4.7-flash", "run": 0, "reasons": [ "previous" ] }, { "key": "glm-5.3-flash-(w4afp8)", "run": 0, "reasons": [ "other_build" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "glm-5.2", "name": "GLM 5.2", "maker": "Z.ai", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-06-16", "year": 2026, "year_basis": "release", "card": { "src": "assets/glm-5-2-field-card.webp", "title": null, "family": false }, "discussed": [ "home", "gpt56", "glm53", "glm", "agreement" ], "listed": [ "open", "anthropic5", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 9, "tied": false, "of": 49, "score": 37 }, "cybt_open": { "place": 3, "tied": false, "of": 28, "score": 37 }, "cybt_proprietary": null, "botsv3": { "place": 17, "tied": true, "of": 33, "score": 33 }, "botsv3_open": { "place": 5, "tied": true, "of": 17, "score": 33 }, "botsv3_proprietary": null }, "aliases": [ "GLM 5.2" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": null, "setting": "Splunk MCP", "solved": 37, "total": 59, "rank": 11, "place": "11", "tied": true, "held": false, "reported": true, "cost": 28.28, "cost_mark": null, "cost_note": null, "cost_label": "$28.28", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.7643, "solved_sec": 9842, "median_sec": 195, "run_date": "2026-08-24", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Fireworks search MCP", "solved": 33, "total": 59, "rank": 24, "place": "24", "tied": true, "held": false, "reported": false, "cost": 19.641381, "cost_mark": null, "cost_note": null, "cost_label": "$19.64", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.595193, "solved_sec": 3540.413, "median_sec": 108.613, "run_date": "2026-06-21", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Fireworks MCP / Literal / 360s", "solved": 33, "total": 59, "rank": 30, "place": "30", "tied": false, "held": false, "reported": false, "cost": 10.646785, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.32263, "solved_sec": 2519.926, "median_sec": 68.612, "run_date": "2026-06-21", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GLM 5.2", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.559322033898305 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "glm-5.1", "run": 0, "reasons": [ "previous" ] }, { "key": "glm-5.3", "run": 0, "reasons": [ "newer" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "maker_top", "top_eastern", "cheaper" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "sonnet-4.5", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-5.5", "name": "GPT-5.5", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-04-22", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "gpt56", "agreement" ], "listed": [ "home", "refusals", "hygiene", "cheating" ], "ranks": { "cybt": { "place": 10, "tied": false, "of": 49, "score": 36 }, "cybt_open": null, "cybt_proprietary": { "place": 7, "tied": false, "of": 21, "score": 36 }, "botsv3": { "place": 8, "tied": true, "of": 33, "score": 37 }, "botsv3_open": null, "botsv3_proprietary": { "place": 7, "tied": true, "of": 16, "score": 37 } }, "aliases": [ "GPT-5.5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "High", "setting": "OpenAI · search curl", "solved": 36, "total": 59, "rank": 14, "place": "14", "tied": false, "held": false, "reported": false, "cost": 62.967852, "cost_mark": null, "cost_note": null, "cost_label": "$62.97", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.749107, "solved_sec": 4623.262, "median_sec": 122.065, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 19, "total": 59, "rank": 65, "place": "65", "tied": true, "held": false, "reported": false, "cost": 69.858094, "cost_mark": "*", "cost_note": "Known subtotal, not the bill: 52 of 59 questions, at the standard rate. Codex does not show which requests passed the long-context threshold, so those 52 cost $69.86 to $134.64. Seven timed-out questions have no usage record.", "cost_label": "$69.86*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 3.676742, "solved_sec": 1904.568, "median_sec": 95.11, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "OpenAI / Splunk MCP", "solved": 13, "total": 59, "rank": 81, "place": "81", "tied": false, "held": false, "reported": false, "cost": 99.491278, "cost_mark": "*", "cost_note": "Known subtotal, not the bill. The traces record usage for every question, but 49 final requests have none. They add at least $3.13 (trace-based estimate about $5.10).", "cost_label": "$99.49*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 7.653175, "solved_sec": 1796.178, "median_sec": 131.314, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "High", "setting": "MCP / Literal / 360s", "solved": 37, "total": 59, "rank": 13, "place": "13", "tied": false, "held": false, "reported": false, "cost": 25.333655, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.6846933783783784, "solved_sec": 6382.963, "median_sec": 158.574, "run_date": "2026-05-06", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "Medium", "setting": "MCP / Literal / 360s", "solved": 37, "total": 59, "rank": 14, "place": "14", "tied": false, "held": false, "reported": false, "cost": 16.644872, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.44986140540540537, "solved_sec": 6568.132, "median_sec": 154.88, "run_date": "2026-05-06", "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 36, "total": 59, "rank": 18, "place": "18", "tied": false, "held": false, "reported": false, "cost": 21.019048, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.5838624444444445, "solved_sec": 6909.296, "median_sec": 190.505, "run_date": "2026-05-06", "effort_label": "low", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-5.5", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": 0.01694915254237288, "reviewed": true, "note": "Named this dataset's own hostname with no tools at all (1 of 59). Its other no-tools answers are public documentation and are not counted.", "no_tools_rate": 0.05084745762711865, "with_tools_rate": 0.6271186440677966 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "High · Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "High · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "High · OpenAI / Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "gpt-6-astra", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "fable-5-unlocked-(cvp)", "name": "Fable 5 unlocked (CVP)", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-07-06", "year": 2026, "year_basis": "release", "card": { "src": "assets/fable-5-field-card.webp", "title": null, "family": true }, "discussed": [ "gpt56" ], "listed": [ "home", "fable", "claude55", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 11, "tied": true, "of": 49, "score": 35 }, "cybt_open": null, "cybt_proprietary": { "place": 8, "tied": true, "of": 21, "score": 35 }, "botsv3": { "place": 4, "tied": false, "of": 33, "score": 40 }, "botsv3_open": null, "botsv3_proprietary": { "place": 4, "tied": false, "of": 16, "score": 40 } }, "aliases": [ "Fable 5 unlocked (CVP)" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "low", "setting": "search curl", "solved": 35, "total": 59, "rank": 15, "place": "15", "tied": true, "held": false, "reported": false, "cost": 39.649031, "cost_mark": null, "cost_note": null, "cost_label": "$39.65", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.132829, "solved_sec": 2523.585, "median_sec": 53.587, "run_date": "2026-07-06", "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Standard", "setting": "Anthropic.com CVP / with tools", "solved": 40, "total": 59, "rank": 5, "place": "5", "tied": false, "held": false, "reported": false, "cost": 25.747622, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.643691, "solved_sec": 5593.117, "median_sec": 59.505, "run_date": "2026-07-06", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Fable 5 unlocked (CVP)", "verdict": "No current no-tools lane", "tone": "review", "confirmed_rate": null, "reviewed": true, "note": "No full no-tools check exists for this model.", "no_tools_rate": null, "with_tools_rate": 0.6779661016949152 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "low · Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "opus-5.5", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "opus-4.5", "name": "Claude Opus 4.5", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2025-11-24", "year": 2025, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "fable", "anthropic5", "claude55", "gpt56", "glm53", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 11, "tied": true, "of": 49, "score": 35 }, "cybt_open": null, "cybt_proprietary": { "place": 8, "tied": true, "of": 21, "score": 35 }, "botsv3": { "place": 12, "tied": true, "of": 33, "score": 36 }, "botsv3_open": null, "botsv3_proprietary": { "place": 10, "tied": true, "of": 16, "score": 36 } }, "aliases": [ "Claude Opus 4.5", "Opus 4.5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": null, "setting": null, "solved": 35, "total": 59, "rank": 15, "place": "15", "tied": true, "held": false, "reported": true, "cost": 47.57, "cost_mark": null, "cost_note": null, "cost_label": "$47.57", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.359143, "solved_sec": 6728, "median_sec": 233, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 36, "total": 59, "rank": 16, "place": "16", "tied": false, "held": false, "reported": false, "cost": 16.040046, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.433515, "solved_sec": 2459.79, "median_sec": 25.381, "run_date": "2026-05-07", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 35, "total": 59, "rank": 23, "place": "23", "tied": false, "held": false, "reported": false, "cost": 24.10871, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.651587, "solved_sec": 1767.59, "median_sec": 24.683, "run_date": "2026-05-04", "effort_label": "low", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Opus 4.5", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "All 2 correct answer-only baseline answers are explainable from general knowledge. No clear known-answer pattern found.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.7457627118644068 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Louie.ai", "variant": null, "tools": null, "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "opus-4.7", "run": 0, "reasons": [ "newer" ] }, { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "opus-5.5", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "fable-5.1", "name": "Claude Fable 5.1", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-09-01", "year": 2026, "year_basis": "release", "card": { "src": "assets/fable-5-1-field-card.webp", "title": "the Mimic Oracle", "family": false }, "discussed": [ "home", "astra" ], "listed": [ "open" ], "ranks": { "cybt": { "place": 11, "tied": true, "of": 49, "score": 35 }, "cybt_open": null, "cybt_proprietary": { "place": 8, "tied": true, "of": 21, "score": 35 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Claude Fable 5.1" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "low", "setting": "Splunk MCP", "solved": 35, "total": 59, "rank": 15, "place": "15", "tied": true, "held": false, "reported": false, "cost": 36.2462, "cost_mark": null, "cost_note": null, "cost_label": "$36.25", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.0356, "solved_sec": 3403.4, "median_sec": 65.5, "run_date": null, "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Splunk MCP", "solved": 35, "total": 59, "rank": 15, "place": "15", "tied": true, "held": false, "reported": false, "cost": 41.6934, "cost_mark": null, "cost_note": null, "cost_label": "$41.69", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.1912, "solved_sec": 3716, "median_sec": 85.6, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "Bedrock · Splunk MCP", "solved": 30, "total": 59, "rank": 39, "place": "39", "tied": true, "held": false, "reported": false, "cost": 51.4326, "cost_mark": null, "cost_note": null, "cost_label": "$51.43", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.7144, "solved_sec": 2389.3, "median_sec": 69, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "opus-5.5", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "glm-5.3-flash-(w4afp8)", "name": "GLM 5.3 Flash (W4AFP8)", "maker": "Z.ai", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-08-26", "year": 2026, "year_basis": "release", "card": { "src": "assets/glm-5-3-flash-w4afp8-field-card.webp", "title": "the Goblin Artificer", "family": false }, "discussed": [], "listed": [ "home", "open", "glm53", "glm" ], "ranks": { "cybt": { "place": 11, "tied": true, "of": 49, "score": 35 }, "cybt_open": { "place": 4, "tied": false, "of": 28, "score": 35 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GLM 5.3 Flash (W4AFP8)" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": null, "setting": "Splunk MCP · self-hosted, 2×H200", "solved": 35, "total": 59, "rank": 15, "place": "15", "tied": true, "held": false, "reported": true, "cost": 12.18, "cost_mark": null, "cost_note": null, "cost_label": "~$12", "cost_approximate": true, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Total time across every case, not solved cases only: this run was reported as one distribution.", "median_note": "Median across every case, not solved cases only: this run was reported as one distribution.", "cost_per_solved": 0.348, "solved_sec": 9516.7, "median_sec": 163.5, "run_date": "2026-09-16", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "glm-5.3-flash", "run": 0, "reasons": [ "other_build", "maker_top", "top_eastern", "cheaper" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western", "better_for_cost" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "sonnet-5.5", "run": 5, "reasons": [ "similar_cost" ] } ] } }, { "key": "deepseek-v4.1-flash", "name": "DeepSeek-V4.1-Flash", "maker": "DeepSeek", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-09-10", "year": 2026, "year_basis": "release", "card": { "src": "assets/deepseek-v4-1-flash-field-card.webp", "title": "the Thrifty Heir", "family": false }, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 15, "tied": true, "of": 49, "score": 34 }, "cybt_open": { "place": 5, "tied": false, "of": 28, "score": 34 }, "cybt_proprietary": null, "botsv3": { "place": 8, "tied": true, "of": 33, "score": 37 }, "botsv3_open": { "place": 2, "tied": false, "of": 17, "score": 37 }, "botsv3_proprietary": null }, "aliases": [ "DeepSeek V4.1 Flash", "DeepSeek-V4.1-Flash" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Fireworks / search MCP, history kept", "solved": 34, "total": 59, "rank": 20, "place": "20", "tied": true, "held": false, "reported": false, "cost": 3.871898, "cost_mark": null, "cost_note": null, "cost_label": "$3.87", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.113879, "solved_sec": 4699.616, "median_sec": 134.35, "run_date": "2026-10-08", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 1, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 37, "total": 59, "rank": 10, "place": "10", "tied": false, "held": false, "reported": false, "cost": 1.894086, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.05119151351351351, "solved_sec": 2632.45, "median_sec": 32.003, "run_date": "2026-10-08", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "DeepSeek V4.1 Flash", "verdict": "Not reviewed", "tone": "pending", "confirmed_rate": null, "reviewed": false, "note": "Answered 3.4% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.6271186440677966 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "deepseek-v4-flash", "run": 0, "reasons": [ "previous" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "glm-4.7-flash", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-5.6-luna", "name": "GPT-5.6 Luna", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-07-09", "year": 2026, "year_basis": "release", "card": { "src": "assets/gpt56-field-card.webp", "title": null, "family": true }, "discussed": [ "gpt56", "agreement" ], "listed": [ "home", "open", "astra", "anthropic5", "claude55", "glm53", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 15, "tied": true, "of": 49, "score": 34 }, "cybt_open": null, "cybt_proprietary": { "place": 11, "tied": false, "of": 21, "score": 34 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GPT-5.6 Luna" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "high", "setting": "OpenAI · high effort", "solved": 34, "total": 59, "rank": 20, "place": "20", "tied": true, "held": false, "reported": false, "cost": 2.218376, "cost_mark": null, "cost_note": null, "cost_label": "$2.22", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.065246, "solved_sec": 2163.823, "median_sec": 65.864, "run_date": "2026-07-10", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": "high", "setting": "OpenAI · search MCP", "solved": 31, "total": 59, "rank": 34, "place": "34", "tied": true, "held": false, "reported": true, "cost": 19.77, "cost_mark": null, "cost_note": null, "cost_label": "$19.77", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.637742, "solved_sec": 5704, "median_sec": 184, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "medium", "setting": "OpenAI · medium effort", "solved": 29, "total": 59, "rank": 41, "place": "41", "tied": true, "held": false, "reported": false, "cost": 2.669746, "cost_mark": null, "cost_note": null, "cost_label": "$2.67", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.09206, "solved_sec": 2043.837, "median_sec": 59.059, "run_date": "2026-07-10", "effort_label": "medium", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-5.6 Luna", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.016949, "with_tools_rate": null }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "medium effort", "tools": "with tools", "refusals": 0, "abstained": 4, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "high · Splunk MCP", "tools": "with tools", "refusals": 0, "abstained": 3, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Louie.ai", "variant": "high · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-6-luna", "run": 0, "reasons": [ "newer" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-oss-120b", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "opus-4.7", "name": "Claude Opus 4.7", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-04-20", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "agreement" ], "listed": [ "home", "fable", "claude55", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 17, "tied": true, "of": 49, "score": 33 }, "cybt_open": null, "cybt_proprietary": { "place": 12, "tied": true, "of": 21, "score": 33 }, "botsv3": { "place": 2, "tied": true, "of": 33, "score": 41 }, "botsv3_open": null, "botsv3_proprietary": { "place": 2, "tied": true, "of": 16, "score": 41 } }, "aliases": [ "Claude Opus 4.7", "Opus 4.7" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "Standard", "setting": "Anthropic.com / search curl", "solved": 33, "total": 59, "rank": 24, "place": "24", "tied": true, "held": false, "reported": false, "cost": 54.229761, "cost_mark": null, "cost_note": null, "cost_label": "$54.23", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.643326, "solved_sec": 3027.25, "median_sec": 67.479, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "Claude Code", "effort": "Standard", "setting": "No-tools control", "solved": 1, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 41, "total": 59, "rank": 4, "place": "4", "tied": false, "held": false, "reported": false, "cost": 17.68344, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.411243, "solved_sec": 2269.577, "median_sec": 30.435, "run_date": "2026-05-03", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "High", "setting": "MCP / Literal / 360s", "solved": 35, "total": 59, "rank": 22, "place": "22", "tied": false, "held": false, "reported": false, "cost": 25.041033, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.715458, "solved_sec": 1692.397, "median_sec": 28.403, "run_date": "2026-05-05", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 26, "total": 59, "rank": 41, "place": "41", "tied": false, "held": false, "reported": false, "cost": 13.51369, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.519757, "solved_sec": 857.82, "median_sec": 21.363, "run_date": "2026-05-07", "effort_label": "low", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Opus 4.7", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": 0.01694915254237288, "reviewed": true, "note": "Named this dataset's own hostname with no tools at all (1 of 59). Its other no-tools answers are public documentation and are not counted.", "no_tools_rate": 0.05084745762711865, "with_tools_rate": 0.7288135593220338 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "Standard · Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "opus-4.5", "run": 0, "reasons": [ "previous" ] }, { "key": "opus-4.8", "run": 0, "reasons": [ "newer", "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern", "faster" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "opus-5", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "kimi-k3", "name": "Kimi-K3", "maker": "Moonshot", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-08-05", "year": 2026, "year_basis": "release", "card": { "src": "assets/kimi-k3-field-card.webp", "title": "the Coin-Eating Dragon", "family": false }, "discussed": [ "home" ], "listed": [ "open", "gpt56", "glm53", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 17, "tied": true, "of": 49, "score": 33 }, "cybt_open": { "place": 6, "tied": true, "of": 28, "score": 33 }, "cybt_proprietary": null, "botsv3": { "place": 6, "tied": true, "of": 33, "score": 38 }, "botsv3_open": { "place": 1, "tied": false, "of": 17, "score": 38 }, "botsv3_proprietary": null }, "aliases": [ "Kimi K3", "Kimi-K3" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP", "solved": 33, "total": 59, "rank": 24, "place": "24", "tied": true, "held": false, "reported": false, "cost": 27.853568, "cost_mark": null, "cost_note": null, "cost_label": "$27.85", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.844048, "solved_sec": 3851.324, "median_sec": 122.443, "run_date": "2026-08-06", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Provider default (not explicitly set)", "setting": "No-tools control", "solved": 2, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "provider default (not explicitly set)", "effort_note": "The control did not set an effort level; the provider's effective setting is unknown." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 38, "total": 59, "rank": 8, "place": "8", "tied": false, "held": false, "reported": false, "cost": 15.619585, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.411042, "solved_sec": 3348.675, "median_sec": 75.37899999999999, "run_date": "2026-08-05", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Kimi K3", "verdict": "Not reviewed", "tone": "pending", "confirmed_rate": null, "reviewed": false, "note": "Answered 1.7% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": null }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "glm-5.2", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "glm-5.1", "name": "GLM 5.1", "maker": "Z.ai", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-04-03", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "open", "gpt56", "glm", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 17, "tied": true, "of": 49, "score": 33 }, "cybt_open": { "place": 6, "tied": true, "of": 28, "score": 33 }, "cybt_proprietary": null, "botsv3": { "place": 19, "tied": true, "of": 33, "score": 31 }, "botsv3_open": { "place": 7, "tied": true, "of": 17, "score": 31 }, "botsv3_proprietary": null }, "aliases": [ "GLM 5.1" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history kept", "solved": 33, "total": 59, "rank": 24, "place": "24", "tied": true, "held": false, "reported": false, "cost": 17.843828, "cost_mark": null, "cost_note": null, "cost_label": "$17.84", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.540722, "solved_sec": 2583.154, "median_sec": 56.785, "run_date": "2026-06-23", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 2, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 31, "total": 59, "rank": 32, "place": "32", "tied": false, "held": false, "reported": false, "cost": 12.742023, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.411033, "solved_sec": 1717.5629999999999, "median_sec": 41.009, "run_date": "2026-06-22", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GLM 5.1", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": null, "reviewed": false, "note": "Answered 0.0% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0, "with_tools_rate": 0.5254237288135594 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "glm-4.7", "run": 0, "reasons": [ "previous" ] }, { "key": "glm-5.2", "run": 0, "reasons": [ "newer" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "maker_top", "top_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "glm-5.2", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "opus-5", "name": "Claude Opus 5", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-07-24", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "anthropic5", "agreement" ], "listed": [ "home", "open", "astra", "fable", "claude55", "gpt56", "glm53", "refusals", "hygiene", "cheating" ], "ranks": { "cybt": { "place": 17, "tied": true, "of": 49, "score": 33 }, "cybt_open": null, "cybt_proprietary": { "place": 12, "tied": true, "of": 21, "score": 33 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Claude Opus 5", "Opus 5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Anthropic.com / search curl", "solved": 33, "total": 59, "rank": 24, "place": "24", "tied": true, "held": false, "reported": false, "cost": 37.810062, "cost_mark": null, "cost_note": null, "cost_label": "$37.81", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.145759, "solved_sec": 3087.628, "median_sec": 76.961, "run_date": "2026-07-27", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Splunk MCP", "solved": 9, "total": 59, "rank": 85, "place": "85", "tied": true, "held": false, "reported": false, "cost": 54.423656, "cost_mark": null, "cost_note": null, "cost_label": "$54.42", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 6.047073, "solved_sec": 663.51, "median_sec": 78.742, "run_date": "2026-07-24", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Opus 5", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": 0.03389830508474576, "reviewed": null, "note": "Named this dataset's own usernames and hostname with no tools at all (2 of 59). A third no-tools answer is general AWS knowledge and is not counted.", "no_tools_rate": 0.05084745762711865, "with_tools_rate": null }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "high · Splunk MCP", "tools": "with tools", "refusals": 0, "abstained": 38, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "high · Splunk curl", "tools": "with tools", "refusals": 0, "abstained": 0, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "opus-4.8", "run": 0, "reasons": [ "previous", "maker_top" ] }, { "key": "opus-5.5", "run": 0, "reasons": [ "newer" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern", "faster" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "gpt-5.6-terra", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "qwen3.7-plus", "name": "Qwen3.7 Plus", "maker": "Alibaba", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "proprietary", "released_at": "2026-06-23", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "open", "gpt56", "agreement" ], "listed": [ "home", "glm53", "refusals" ], "ranks": { "cybt": { "place": 17, "tied": true, "of": 49, "score": 33 }, "cybt_open": null, "cybt_proprietary": { "place": 12, "tied": true, "of": 21, "score": 33 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Qwen3.7 Plus" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Fireworks search MCP", "solved": 33, "total": 59, "rank": 24, "place": "24", "tied": true, "held": false, "reported": false, "cost": 5.223346, "cost_mark": null, "cost_note": null, "cost_label": "$5.22", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.158283, "solved_sec": 1130.047, "median_sec": 24.436, "run_date": "2026-06-24", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "muse-glimmer-30b", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "sonnet-4.5", "name": "Claude Sonnet 4.5", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2025-09-29", "year": 2025, "year_basis": "release", "card": null, "discussed": [ "home", "open" ], "listed": [ "fable", "anthropic5", "claude55", "gpt56", "refusals", "hygiene", "agreement" ], "ranks": { "cybt": { "place": 22, "tied": true, "of": 49, "score": 32 }, "cybt_open": null, "cybt_proprietary": { "place": 15, "tied": false, "of": 21, "score": 32 }, "botsv3": { "place": 8, "tied": true, "of": 33, "score": 37 }, "botsv3_open": null, "botsv3_proprietary": { "place": 7, "tied": true, "of": 16, "score": 37 } }, "aliases": [ "Claude Sonnet 4.5", "Claude Sonnet 4.5 (opencode)", "Sonnet 4.5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Bedrock / search MCP", "solved": 32, "total": 59, "rank": 30, "place": "30", "tied": true, "held": false, "reported": false, "cost": 25.192427, "cost_mark": null, "cost_note": null, "cost_label": "$25.19", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.787263, "solved_sec": 2513.976, "median_sec": 73.572, "run_date": "2026-06-01", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk curl", "solved": 28, "total": 59, "rank": 45, "place": "45", "tied": true, "held": false, "reported": false, "cost": 28.517875, "cost_mark": null, "cost_note": null, "cost_label": "$28.52", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.018496, "solved_sec": 1855.324, "median_sec": 62.848, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Louie.ai", "effort": null, "setting": null, "solved": 26, "total": 59, "rank": 51, "place": "51", "tied": true, "held": false, "reported": true, "cost": 9.67, "cost_mark": null, "cost_note": null, "cost_label": "$9.67", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.371923, "solved_sec": 7549, "median_sec": 315, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Splunk MCP", "solved": 12, "total": 59, "rank": 82, "place": "82", "tied": false, "held": false, "reported": false, "cost": 19.91311, "cost_mark": null, "cost_note": null, "cost_label": "$19.91", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.659426, "solved_sec": 1743.34, "median_sec": 139.062, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "Claude Code", "effort": "High", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 37, "total": 59, "rank": 9, "place": "9", "tied": false, "held": false, "reported": false, "cost": 21.571569, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.583015, "solved_sec": 2600.186, "median_sec": 49.713, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "High", "setting": "MCP / Literal / 360s", "solved": 37, "total": 59, "rank": 12, "place": "12", "tied": false, "held": false, "reported": false, "cost": 16.638036, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.437843, "solved_sec": 3076.89, "median_sec": 38.719, "run_date": "2026-05-08", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Medium", "setting": "MCP / Literal / 360s", "solved": 36, "total": 59, "rank": 17, "place": "17", "tied": false, "held": false, "reported": false, "cost": 16.612388, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.448983, "solved_sec": 2677.701, "median_sec": 41.038, "run_date": "2026-05-08", "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 34, "total": 59, "rank": 28, "place": "28", "tied": false, "held": false, "reported": false, "cost": 15.982065, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.45663, "solved_sec": 2154.334, "median_sec": 39.733, "run_date": "2026-05-08", "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 30, "total": 59, "rank": 34, "place": "34", "tied": false, "held": false, "reported": false, "cost": 17.300004, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.576667, "solved_sec": 1726.4759999999999, "median_sec": 30.7855, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Claude Sonnet 4.5", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "All 2 correct answer-only baseline answers are explainable from general knowledge. No clear known-answer pattern found.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.6440677966101694 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Louie.ai", "variant": null, "tools": null, "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "High · Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "High · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Anthropic / Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "sonnet-5", "run": 0, "reasons": [ "newer" ] }, { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern", "faster" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "kimi-k2-thinking", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "mistral-large-4", "name": "Mistral Large 4", "maker": "Mistral AI", "country": "France", "country_code": "FR", "flag": "🇫🇷", "region": "western", "kind": "open", "released_at": "2026-10-06", "year": 2026, "year_basis": "release", "card": { "src": "assets/mistral-large-4-field-card.webp", "title": "the Grand Coq", "family": false }, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 22, "tied": true, "of": 49, "score": 32 }, "cybt_open": { "place": 8, "tied": false, "of": 28, "score": 32 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Mistral Large 4" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Mistral API (released, Nov 1) / Splunk MCP, history kept", "solved": 32, "total": 59, "rank": 30, "place": "30", "tied": true, "held": false, "reported": false, "cost": 8.106237, "cost_mark": "*", "cost_note": "Priced at Mistral's listed rate. Title calls are not recorded and no Mistral bill has been read.", "cost_label": "$8.11*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.25332, "solved_sec": 3392.445, "median_sec": 94.853, "run_date": "2026-11-01", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Mistral API (preview, Oct 23–26) / Splunk MCP, history kept", "solved": 28, "total": 59, "rank": 45, "place": "45", "tied": true, "held": false, "reported": false, "cost": 31.220162, "cost_mark": "*", "cost_note": "Priced at Mistral's listed rate. Title calls are not recorded and no Mistral bill has been read.", "cost_label": "$31.22*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.115006, "solved_sec": 3315.743, "median_sec": 101.323, "run_date": "2026-10-24", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Mistral API (preview, Oct 23–26) / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Mistral API (released, Nov 1) / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "mistral-small-4", "run": 0, "reasons": [ "other_size" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western", "better_for_cost" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "faster" ] }, { "key": "qwen3.8-27b", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "sonnet-5", "name": "Claude Sonnet 5", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-06-30", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "open", "anthropic5", "claude55", "gpt56", "agreement" ], "listed": [ "home", "fable", "glm53", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 24, "tied": false, "of": 49, "score": 30 }, "cybt_open": null, "cybt_proprietary": { "place": 16, "tied": false, "of": 21, "score": 30 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Claude Sonnet 5", "Sonnet 5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "medium", "setting": "Anthropic.com / search MCP", "solved": 30, "total": 59, "rank": 39, "place": "39", "tied": true, "held": false, "reported": false, "cost": 36.01, "cost_mark": null, "cost_note": null, "cost_label": "$36.01", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.200333, "solved_sec": 2370.736, "median_sec": 76.172, "run_date": "2026-07-17", "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Splunk MCP", "solved": 25, "total": 59, "rank": 55, "place": "55", "tied": true, "held": false, "reported": false, "cost": 42.95, "cost_mark": null, "cost_note": null, "cost_label": "$42.95", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.718, "solved_sec": 1932.712, "median_sec": 64.778, "run_date": "2026-07-17", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Sonnet 5", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": null, "reviewed": null, "note": "Answered 0.0% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0, "with_tools_rate": null }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "medium · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "high · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "sonnet-4.5", "run": 0, "reasons": [ "previous" ] }, { "key": "sonnet-5.5", "run": 0, "reasons": [ "newer" ] }, { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "fable-5.1", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "glm-5.3", "name": "GLM 5.3", "maker": "Z.ai", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-08-14", "year": 2026, "year_basis": "release", "card": { "src": "assets/glm-5-3-field-card.webp", "title": "the Goblin Brute", "family": false }, "discussed": [ "home", "open", "glm53", "glm", "agreement" ], "listed": [ "astra", "claude55", "gpt56", "refusals", "hygiene", "ttp" ], "ranks": { "cybt": { "place": 25, "tied": true, "of": 49, "score": 29 }, "cybt_open": { "place": 9, "tied": false, "of": 28, "score": 29 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GLM 5.3" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history kept", "solved": 29, "total": 59, "rank": 41, "place": "41", "tied": true, "held": false, "reported": false, "cost": 36.518258, "cost_mark": null, "cost_note": null, "cost_label": "$36.52", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.25925, "solved_sec": 3808.833, "median_sec": 124.776, "run_date": "2026-08-31", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 2, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GLM 5.3", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered nothing with no tools.", "no_tools_rate": 0, "with_tools_rate": 0.5593 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "BOTSv3", "harness": "opencode", "variant": "Splunk MCP · opencode 1.18.25", "tools": "with tools", "refusals": 0, "abstained": 1, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Splunk MCP · opencode 1.18.25", "tools": "with tools", "refusals": 0, "abstained": 0, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "glm-5.2", "run": 0, "reasons": [ "previous" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "maker_top", "top_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "opus-5.5", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "qwen3.8-flash", "name": "Qwen3.8 Flash", "maker": "Alibaba", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "proprietary", "released_at": "2026-08-26", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 25, "tied": true, "of": 49, "score": 29 }, "cybt_open": null, "cybt_proprietary": { "place": 17, "tied": false, "of": 21, "score": 29 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Qwen3.8 Flash" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Together / Splunk MCP, history kept", "solved": 29, "total": 59, "rank": 41, "place": "41", "tied": true, "held": false, "reported": false, "cost": 6.766192, "cost_mark": "*", "cost_note": "Priced at Together's listed rate. Title calls are not recorded and no Together bill has been read.", "cost_label": "$6.77*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.233317, "solved_sec": 3187.969, "median_sec": 89.398, "run_date": "2026-10-26", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Together / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "qwen3.7-plus", "run": 0, "reasons": [ "maker_top", "top_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "better_for_cost", "similar_cost" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] } ] } }, { "key": "opus-5.5", "name": "Claude Opus 5.5", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-09-22", "year": 2026, "year_basis": "release", "card": { "src": "assets/claude-opus-5-5-field-card.webp", "title": "the Gilded Archmage", "family": false }, "discussed": [ "home", "claude55" ], "listed": [ "astra", "fable", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 27, "tied": true, "of": 49, "score": 27 }, "cybt_open": null, "cybt_proprietary": { "place": 18, "tied": true, "of": 21, "score": 27 }, "botsv3": { "place": 1, "tied": false, "of": 33, "score": 49 }, "botsv3_open": null, "botsv3_proprietary": { "place": 1, "tied": false, "of": 16, "score": 49 } }, "aliases": [ "Claude Opus 5.5", "Claude Opus 5.5 (opencode)", "Opus 5.5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Splunk MCP", "solved": 27, "total": 59, "rank": 47, "place": "47", "tied": true, "held": false, "reported": false, "cost": 41.160256, "cost_mark": null, "cost_note": null, "cost_label": "$41.16", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.524454, "solved_sec": 3777.446, "median_sec": 130.895, "run_date": "2026-10-14", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "medium", "setting": "Bedrock · Splunk MCP", "solved": 20, "total": 59, "rank": 61, "place": "61", "tied": true, "held": false, "reported": false, "cost": 49.41, "cost_mark": null, "cost_note": null, "cost_label": "$49.41", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.47, "solved_sec": 2969.045, "median_sec": 125.255, "run_date": null, "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "medium", "setting": "Splunk MCP", "solved": 18, "total": 59, "rank": 69, "place": "69", "tied": true, "held": false, "reported": false, "cost": 36.33, "cost_mark": null, "cost_note": null, "cost_label": "$36.33", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.02, "solved_sec": 2596.746, "median_sec": 160.606, "run_date": null, "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "Bedrock · Splunk MCP", "solved": 17, "total": 59, "rank": 73, "place": "73", "tied": true, "held": false, "reported": false, "cost": 53.82, "cost_mark": null, "cost_note": null, "cost_label": "$53.82", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 3.17, "solved_sec": 2518.107, "median_sec": 136.527, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "low", "setting": "Bedrock · Splunk MCP", "solved": 9, "total": 59, "rank": 85, "place": "85", "tied": true, "held": false, "reported": false, "cost": 37.19, "cost_mark": null, "cost_note": null, "cost_label": "$37.19", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 4.13, "solved_sec": 773.896, "median_sec": 34.053, "run_date": null, "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "low", "setting": "Splunk MCP", "solved": 9, "total": 59, "rank": 85, "place": "85", "tied": true, "held": false, "reported": false, "cost": 23.19, "cost_mark": null, "cost_note": null, "cost_label": "$23.19", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.58, "solved_sec": 818.113, "median_sec": 87.653, "run_date": null, "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "High", "setting": "MCP / Literal / 300s", "solved": 49, "total": 59, "rank": 1, "place": "1", "tied": false, "held": false, "reported": false, "cost": 8.174972, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.16683616326530612, "solved_sec": 2316.313, "median_sec": 34.604, "run_date": "2026-09-23", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "High", "setting": "MCP / 300s", "solved": 43, "total": 59, "rank": 2, "place": "2", "tied": false, "held": false, "reported": false, "cost": 9.46578, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.2201344186046512, "solved_sec": 1317.143, "median_sec": 24.515, "run_date": "2026-10-17", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Opus 5.5", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": null, "reviewed": false, "note": "Answered 17 of 59 with no tools. A per-question review found strong instance-specific signals in 10 of those answers. That is an answer-level signal, not a measured contamination rate.", "no_tools_rate": 0.288135593220339, "with_tools_rate": 0.8305084745762712 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Claude Code", "variant": "high · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "opus-5", "run": 0, "reasons": [ "previous" ] }, { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "deepseek-v4-flash", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "opus-4.8", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "deepseek-v4-flash", "name": "DeepSeek-V4-Flash", "maker": "DeepSeek", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-08-06", "year": 2026, "year_basis": "release", "card": { "src": "assets/deepseek-v4-flash-field-card.webp", "title": "the Pauper Prince", "family": false }, "discussed": [ "home", "open" ], "listed": [ "claude55", "gpt56", "glm53", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 27, "tied": true, "of": 49, "score": 27 }, "cybt_open": { "place": 10, "tied": false, "of": 28, "score": 27 }, "cybt_proprietary": null, "botsv3": { "place": 19, "tied": true, "of": 33, "score": 31 }, "botsv3_open": { "place": 7, "tied": true, "of": 17, "score": 31 }, "botsv3_proprietary": null }, "aliases": [ "DeepSeek V4 Flash", "DeepSeek-V4-Flash" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP", "solved": 27, "total": 59, "rank": 47, "place": "47", "tied": true, "held": false, "reported": false, "cost": 2.123815, "cost_mark": null, "cost_note": null, "cost_label": "$2.12", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.07866, "solved_sec": 3685.941, "median_sec": 122.371, "run_date": "2026-08-06", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 31, "total": 59, "rank": 33, "place": "33", "tied": false, "held": false, "reported": false, "cost": 1.062266, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.034267, "solved_sec": 3857.534, "median_sec": 80.779, "run_date": "2026-08-06", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "DeepSeek V4 Flash", "verdict": "Never checked", "tone": "pending", "confirmed_rate": null, "reviewed": false, "note": "No answer-only baseline has been committed for this model line yet.", "no_tools_rate": null, "with_tools_rate": 0.5254237288135594 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newer", "maker_top", "newest_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-6.1-terra", "name": "GPT-6.1 Terra", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-10-26", "year": 2026, "year_basis": "release", "card": { "src": "assets/gpt-6-1-terra-field-card.webp", "title": "the Stonebound Surveyor", "family": false }, "discussed": [], "listed": [ "home", "claude55", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 27, "tied": true, "of": 49, "score": 27 }, "cybt_open": null, "cybt_proprietary": { "place": 18, "tied": true, "of": 21, "score": 27 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GPT-6.1 Terra" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 27, "total": 59, "rank": 47, "place": "47", "tied": true, "held": false, "reported": false, "cost": 4.396366, "cost_mark": "*", "cost_note": "Priced at OpenAI's standard API rate. The run used a ChatGPT plan, whose charge is unknown.", "cost_label": "$4.40*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.162828, "solved_sec": 2766, "median_sec": 86, "run_date": "2026-10-27", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "Codex", "effort": "high", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-6.1 Terra", "verdict": "Not reviewed", "tone": "pending", "confirmed_rate": null, "reviewed": false, "note": "Answered 3.4% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.711864406779661 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "high · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-terra", "run": 0, "reasons": [ "previous" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "deepseek-v4-flash", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "minimax-m3", "name": "MiniMax M3", "maker": "MiniMax", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-06-23", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "open", "claude55", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 30, "tied": true, "of": 49, "score": 26 }, "cybt_open": { "place": 11, "tied": false, "of": 28, "score": 26 }, "cybt_proprietary": null, "botsv3": { "place": 14, "tied": true, "of": 33, "score": 35 }, "botsv3_open": { "place": 3, "tied": false, "of": 17, "score": 35 }, "botsv3_proprietary": null }, "aliases": [ "MiniMax M3" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history kept", "solved": 26, "total": 59, "rank": 51, "place": "51", "tied": true, "held": false, "reported": false, "cost": 9.015876, "cost_mark": null, "cost_note": null, "cost_label": "$9.02", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.346764, "solved_sec": 2122.549, "median_sec": 70.402, "run_date": "2026-06-23", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 2, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 35, "total": 59, "rank": 21, "place": "21", "tied": false, "held": false, "reported": false, "cost": 4.047526, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.11564360000000001, "solved_sec": 1606.603, "median_sec": 21.4295, "run_date": "2026-06-23", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "MiniMax M3", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": null, "reviewed": false, "note": "Answered 0.0% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0, "with_tools_rate": 0.5932203389830508 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western", "better_for_cost" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "haiku-5.5", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-6-luna", "name": "GPT-6 Luna", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-09-22", "year": 2026, "year_basis": "release", "card": { "src": "assets/gpt-6-luna-field-card.webp", "title": "the Moonlit Pathfinder", "family": false }, "discussed": [], "listed": [ "home", "claude55" ], "ranks": { "cybt": { "place": 30, "tied": true, "of": 49, "score": 26 }, "cybt_open": null, "cybt_proprietary": { "place": 20, "tied": false, "of": 21, "score": 26 }, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GPT-6 Luna" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 26, "total": 59, "rank": 51, "place": "51", "tied": true, "held": false, "reported": false, "cost": 1.4847360649999999, "cost_mark": null, "cost_note": null, "cost_label": "$1.48", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.05710523326923077, "solved_sec": 3389.877, "median_sec": 137.37, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "previous" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "qwen3.8-27b", "name": "Qwen3.8 27B", "maker": "Alibaba", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-08-05", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "open" ], "listed": [ "home", "claude55", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 32, "tied": false, "of": 49, "score": 25 }, "cybt_open": { "place": 12, "tied": false, "of": 28, "score": 25 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Qwen3.8 27B" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "W&B / Splunk MCP, history kept", "solved": 25, "total": 59, "rank": 55, "place": "55", "tied": true, "held": false, "reported": false, "cost": 8.878042, "cost_mark": null, "cost_note": null, "cost_label": "$8.88", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.355122, "solved_sec": 3075.623, "median_sec": 96.595, "run_date": "2026-10-25", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · W&B / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "qwen3.7-plus", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western", "better_for_cost" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "muse-glimmer-30b", "run": 0, "reasons": [ "similar_size" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 3, "reasons": [ "faster" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "similar_cost" ] } ] } }, { "key": "nemotron-3-ultra-nvfp4", "name": "Nemotron 3 Ultra NVFP4", "maker": "NVIDIA", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2026-06-04", "year": 2026, "year_basis": "release", "card": { "src": "assets/nemotron-3-ultra-nvfp4-field-card.webp", "title": "the Clockwork Titan", "family": false }, "discussed": [ "open" ], "listed": [ "home", "claude55", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 33, "tied": false, "of": 49, "score": 23 }, "cybt_open": { "place": 13, "tied": false, "of": 28, "score": 23 }, "cybt_proprietary": null, "botsv3": { "place": 23, "tied": false, "of": 33, "score": 28 }, "botsv3_open": { "place": 10, "tied": false, "of": 17, "score": 28 }, "botsv3_proprietary": null }, "aliases": [ "Nemotron 3 Ultra NVFP4" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Fireworks search MCP, history pruned", "solved": 23, "total": 59, "rank": 57, "place": "57", "tied": true, "held": false, "reported": false, "cost": 44.928514, "cost_mark": null, "cost_note": null, "cost_label": "$44.93", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.953414, "solved_sec": 1684.287, "median_sec": 66.434, "run_date": "2026-09-03", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks / Splunk MCP, history kept", "solved": 20, "total": 59, "rank": 61, "place": "61", "tied": true, "held": false, "reported": false, "cost": 32.53902, "cost_mark": null, "cost_note": null, "cost_label": "$32.54", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.626951, "solved_sec": 1205.215, "median_sec": 40.022, "run_date": "2026-10-18", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / 300s", "solved": 28, "total": 59, "rank": 38, "place": "38", "tied": false, "held": false, "reported": false, "cost": 23.117516, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.8256255714285714, "solved_sec": 1142.354, "median_sec": 28.301000000000002, "run_date": "2026-10-22", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Nemotron 3 Ultra NVFP4", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": null, "reviewed": false, "note": "Answered 0.0% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0, "with_tools_rate": 0.4745762711864407 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history pruned", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "gpt-5.6-sol", "run": 4, "reasons": [ "similar_cost" ] } ] } }, { "key": "deepseek-v4-pro", "name": "DeepSeek-V4-Pro", "maker": "DeepSeek", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-06-23", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "open", "claude55", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 34, "tied": false, "of": 49, "score": 21 }, "cybt_open": { "place": 14, "tied": false, "of": 28, "score": 21 }, "cybt_proprietary": null, "botsv3": { "place": 17, "tied": true, "of": 33, "score": 33 }, "botsv3_open": { "place": 5, "tied": true, "of": 17, "score": 33 }, "botsv3_proprietary": null }, "aliases": [ "DeepSeek V4 Pro", "DeepSeek-V4-Pro" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history kept", "solved": 21, "total": 59, "rank": 60, "place": "60", "tied": false, "held": false, "reported": false, "cost": 22.878745, "cost_mark": null, "cost_note": null, "cost_label": "$22.88", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.089464, "solved_sec": 2887.258, "median_sec": 121.101, "run_date": "2026-06-23", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 4, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 33, "total": 59, "rank": 31, "place": "31", "tied": false, "held": false, "reported": false, "cost": 11.212345, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.33976803030303027, "solved_sec": 3304.887, "median_sec": 65.69749999999999, "run_date": "2026-06-23", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "DeepSeek V4 Pro", "verdict": "Not reviewed", "tone": "pending", "confirmed_rate": null, "reviewed": false, "note": "Answered 1.7% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.559322033898305 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "maker_top", "newest_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "opus-5.5", "run": 5, "reasons": [ "similar_cost" ] } ] } }, { "key": "muse-glimmer-30b", "name": "Muse Glimmer 30B", "maker": "Meta", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2026-08-10", "year": 2026, "year_basis": "release", "card": { "src": "assets/muse-glimmer-30b-field-card.webp", "title": "the Glimmer Sprite", "family": false }, "discussed": [ "open" ], "listed": [ "home", "claude55", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 35, "tied": true, "of": 49, "score": 20 }, "cybt_open": { "place": 15, "tied": true, "of": 28, "score": 20 }, "cybt_proprietary": null, "botsv3": { "place": 25, "tied": false, "of": 33, "score": 26 }, "botsv3_open": { "place": 12, "tied": false, "of": 17, "score": 26 }, "botsv3_proprietary": null }, "aliases": [ "Muse Glimmer 30B" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "low", "setting": "Fireworks · Together / Splunk MCP, history kept", "solved": 20, "total": 59, "rank": 61, "place": "61", "tied": true, "held": false, "reported": false, "cost": 5.112923, "cost_mark": null, "cost_note": null, "cost_label": "$5.11", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.255646, "solved_sec": 2198.891, "median_sec": 97.559, "run_date": "2026-10-07", "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history pruned", "solved": 15, "total": 59, "rank": 79, "place": "79", "tied": false, "held": false, "reported": false, "cost": 11.592677, "cost_mark": null, "cost_note": null, "cost_label": "$11.59", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.772845, "solved_sec": 3166.123, "median_sec": 212.291, "run_date": "2026-09-03", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "low", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 26, "total": 59, "rank": 42, "place": "42", "tied": false, "held": false, "reported": false, "cost": 4.073836, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.156686, "solved_sec": 1738.907, "median_sec": 48.208, "run_date": "2026-10-07", "effort_label": "low", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Muse Glimmer 30B", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": null, "reviewed": false, "note": "Answered 0.0% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0, "with_tools_rate": 0.4406779661016949 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history pruned", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "low · Together / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern", "better_for_cost" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "qwen3.8-27b", "run": 0, "reasons": [ "similar_size" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "minimax-2.5", "name": "MiniMax-2.5", "maker": "MiniMax", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2026-02-12", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "open", "claude55", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 35, "tied": true, "of": 49, "score": 20 }, "cybt_open": { "place": 15, "tied": true, "of": 28, "score": 20 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "MiniMax-2.5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks · Bedrock / Splunk curl", "solved": 20, "total": 59, "rank": 61, "place": "61", "tied": true, "held": false, "reported": false, "cost": 9.38, "cost_mark": null, "cost_note": null, "cost_label": "$9.38", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.469, "solved_sec": 1588.215, "median_sec": 65.822, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "minimax-m3", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western", "better_for_cost" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "sonnet-5.5", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "inkling", "name": "Inkling", "maker": "Thinking Machines", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2026-07-15", "year": 2026, "year_basis": "release", "card": { "src": "assets/inkling-field-card.webp", "title": "the Grand Ink Scribe", "family": false }, "discussed": [ "hygiene" ], "listed": [ "home", "open", "claude55", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 37, "tied": true, "of": 49, "score": 19 }, "cybt_open": { "place": 17, "tied": true, "of": 28, "score": 19 }, "cybt_proprietary": null, "botsv3": { "place": 16, "tied": false, "of": 33, "score": 34 }, "botsv3_open": { "place": 4, "tied": false, "of": 17, "score": 34 }, "botsv3_proprietary": null }, "aliases": [ "Inkling" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history kept", "solved": 19, "total": 59, "rank": 65, "place": "65", "tied": true, "held": false, "reported": false, "cost": 24.322766, "cost_mark": null, "cost_note": null, "cost_label": "$24.32", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.280146, "solved_sec": 1272.951, "median_sec": 56.068, "run_date": "2026-08-07", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 2, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 34, "total": 59, "rank": 27, "place": "27", "tied": false, "held": false, "reported": false, "cost": 17.740888, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.5217908235294119, "solved_sec": 2039.057, "median_sec": 44.195, "run_date": "2026-08-07", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Inkling", "verdict": "Never checked", "tone": "pending", "confirmed_rate": null, "reviewed": false, "note": "No no-tools control was run for Inkling on BOTSv3.", "no_tools_rate": null, "with_tools_rate": 0.576271186440678 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "kimi-k2-thinking", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "gemma-4-31b", "name": "Gemma 4 31B", "maker": "Google", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2026-04-02", "year": 2026, "year_basis": "release", "card": { "src": "assets/gemma-4-31b-field-card.webp", "title": "the Crystal Guardian", "family": false }, "discussed": [ "open" ], "listed": [ "home", "claude55", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 37, "tied": true, "of": 49, "score": 19 }, "cybt_open": { "place": 17, "tied": true, "of": 28, "score": 19 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Gemma 4 31B" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock · W&B / Splunk MCP, history kept", "solved": 19, "total": 59, "rank": 65, "place": "65", "tied": true, "held": false, "reported": false, "cost": 5.760798, "cost_mark": null, "cost_note": null, "cost_label": "$5.76", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.3032, "solved_sec": 905.779, "median_sec": 21.344, "run_date": "2026-10-20", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · W&B / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern", "better_for_cost" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "qwen3.8-27b", "run": 0, "reasons": [ "similar_size" ] }, { "key": "gpt-6-luna", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "haiku-5.5", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "nemotron-lightning-3.5-30b-a3b", "name": "Nemotron Lightning 3.5 30B-A3B", "maker": "NVIDIA", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2026-08-11", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "open" ], "listed": [ "home", "claude55", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 39, "tied": false, "of": 49, "score": 18 }, "cybt_open": { "place": 19, "tied": false, "of": 28, "score": 18 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Nemotron Lightning 3.5 30B-A3B" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history kept", "solved": 18, "total": 59, "rank": 69, "place": "69", "tied": true, "held": false, "reported": false, "cost": 1.443772, "cost_mark": null, "cost_note": null, "cost_label": "$1.44", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.08021, "solved_sec": 838.256, "median_sec": 42.894, "run_date": "2026-09-03", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks search MCP, history pruned", "solved": 9, "total": 59, "rank": 85, "place": "85", "tied": true, "held": false, "reported": false, "cost": 1.914192, "cost_mark": null, "cost_note": null, "cost_label": "$1.91", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.212688, "solved_sec": 809.938, "median_sec": 62.764, "run_date": "2026-09-03", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Fireworks search MCP, history pruned", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "nemotron-3-ultra-nvfp4", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "qwen3.8-27b", "run": 0, "reasons": [ "similar_size" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "mistral-small-4", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-oss-120b", "name": "GPT-OSS-120B", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2025-08-05", "year": 2025, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "open", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 40, "tied": true, "of": 49, "score": 17 }, "cybt_open": { "place": 20, "tied": true, "of": 28, "score": 17 }, "cybt_proprietary": null, "botsv3": { "place": 28, "tied": false, "of": 33, "score": 20 }, "botsv3_open": { "place": 14, "tied": false, "of": 17, "score": 20 }, "botsv3_proprietary": null }, "aliases": [ "GPT-OSS 120B", "GPT-OSS-120B" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk MCP", "solved": 17, "total": 59, "rank": 73, "place": "73", "tied": true, "held": false, "reported": false, "cost": 2.270997, "cost_mark": null, "cost_note": null, "cost_label": "$2.27", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.133588, "solved_sec": 932.901, "median_sec": 30.318, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / 300s", "solved": 20, "total": 59, "rank": 46, "place": "46", "tied": false, "held": false, "reported": false, "cost": 2.461151, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.12305755, "solved_sec": 749.861, "median_sec": 23.4185, "run_date": "2026-10-20", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 11, "total": 59, "rank": 49, "place": "49", "tied": false, "held": false, "reported": false, "cost": 1.404124, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.127648, "solved_sec": 598.743, "median_sec": 28.697, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GPT-OSS 120B", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.3389830508474576 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-oss-20b", "run": 0, "reasons": [ "other_size" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "nemotron-super-3-120b", "run": 0, "reasons": [ "similar_size" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "better_for_cost", "similar_cost" ] } ] } }, { "key": "deepseek-v3.2", "name": "DeepSeek-V3.2", "maker": "DeepSeek", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2025-12-01", "year": 2025, "year_basis": "release", "card": null, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 40, "tied": true, "of": 49, "score": 17 }, "cybt_open": { "place": 20, "tied": true, "of": 28, "score": 17 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "DeepSeek-V3.2" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks · Bedrock / Splunk curl", "solved": 17, "total": 59, "rank": 73, "place": "73", "tied": true, "held": false, "reported": false, "cost": 13.326783, "cost_mark": "*", "cost_note": "Known subtotal, not the bill. This lane never recorded each question's final request. From the traces those add at least $1.98. AWS's bill for that day, less the model's other runs, caps the total at $16.00.", "cost_label": "$13.33*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.783928, "solved_sec": 1753.464, "median_sec": 87.478, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 1, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "maker_top", "newest_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western", "better_for_cost" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "sonnet-5.5", "run": 5, "reasons": [ "similar_cost" ] } ] } }, { "key": "glm-4.7", "name": "GLM 4.7", "maker": "Z.ai", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2025-12-01", "year": 2025, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "gpt56", "glm", "hygiene", "agreement" ], "ranks": { "cybt": { "place": 42, "tied": false, "of": 49, "score": 14 }, "cybt_open": { "place": 22, "tied": false, "of": 28, "score": 14 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "GLM 4.7" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / search MCP", "solved": 14, "total": 59, "rank": 80, "place": "80", "tied": false, "held": false, "reported": false, "cost": 34.221545, "cost_mark": null, "cost_note": null, "cost_label": "$34.22", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 2.444396, "solved_sec": 2457.628, "median_sec": 145.166, "run_date": "2026-06-21", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GLM 4.7", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answer-only baseline is 0.0%; no publishable BOTSv3 tool-assisted run is attached yet.", "no_tools_rate": 0, "with_tools_rate": null }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "glm-5.1", "run": 0, "reasons": [ "newer" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "maker_top", "top_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "gpt-5.6-sol", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "haiku-5.5", "name": "Claude Haiku 5.5", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-10-07", "year": 2026, "year_basis": "release", "card": { "src": "assets/claude-haiku-5-5-field-card-r2.webp", "title": "the Pocket Scribe", "family": false }, "discussed": [], "listed": [ "home", "claude55", "hygiene" ], "ranks": { "cybt": { "place": 43, "tied": true, "of": 49, "score": 11 }, "cybt_open": null, "cybt_proprietary": { "place": 21, "tied": false, "of": 21, "score": 11 }, "botsv3": { "place": 12, "tied": true, "of": 33, "score": 36 }, "botsv3_open": null, "botsv3_proprietary": { "place": 10, "tied": true, "of": 16, "score": 36 } }, "aliases": [ "Claude Haiku 5.5", "Claude Haiku 5.5 (opencode)", "Haiku 5.5" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "high", "setting": "Splunk MCP", "solved": 11, "total": 59, "rank": 83, "place": "83", "tied": true, "held": false, "reported": false, "cost": 5.69, "cost_mark": null, "cost_note": null, "cost_label": "$5.69", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.52, "solved_sec": 1134.327, "median_sec": 113.668, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "Bedrock · Splunk MCP", "solved": 9, "total": 59, "rank": 85, "place": "85", "tied": true, "held": false, "reported": false, "cost": 9.04, "cost_mark": null, "cost_note": null, "cost_label": "$9.04", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1, "solved_sec": 1102.927, "median_sec": 108.671, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "Claude Code", "effort": "medium", "setting": "Splunk MCP", "solved": 7, "total": 59, "rank": 90, "place": "90", "tied": true, "held": false, "reported": false, "cost": 3.61, "cost_mark": null, "cost_note": null, "cost_label": "$3.61", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.52, "solved_sec": 609.661, "median_sec": 58.241, "run_date": null, "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "High", "setting": "MCP / Literal / 300s", "solved": 36, "total": 59, "rank": 15, "place": "15", "tied": false, "held": false, "reported": false, "cost": 0.616171, "cost_mark": "*", "cost_note": "Known subtotal, not the full cost. One question's usage record is incomplete, so 58 of 59 are priced here. Known usage including that question comes to at least $0.77; the full cost is not certified.", "cost_per_solved": 0.017115861111111113, "solved_sec": 1727.405, "median_sec": 29.7195, "run_date": "2026-10-08", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "High", "setting": "MCP / 300s", "solved": 33, "total": 59, "rank": 29, "place": "29", "tied": false, "held": false, "reported": false, "cost": 1.755141, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.05318609090909091, "solved_sec": 1767.778, "median_sec": 34.048, "run_date": "2026-10-11", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Haiku 5.5", "verdict": "Not reviewed", "tone": "pending", "confirmed_rate": null, "reviewed": false, "note": "Answered 1.7% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.6101694915254238 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "opus-4.8", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open" ] }, { "key": "gpt-6.1-terra", "run": 0, "reasons": [ "newest_western" ] }, { "key": "qwen3.7-plus", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "qwen3.8-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "gemma-4-31b", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "nemotron-super-3-120b", "name": "Nemotron Super 3 120B", "maker": "NVIDIA", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2026-03-16", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "open", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 43, "tied": true, "of": 49, "score": 11 }, "cybt_open": { "place": 23, "tied": false, "of": 28, "score": 11 }, "cybt_proprietary": null, "botsv3": { "place": 27, "tied": false, "of": 33, "score": 22 }, "botsv3_open": { "place": 13, "tied": false, "of": 17, "score": 22 }, "botsv3_proprietary": null }, "aliases": [ "Nemotron Super 3 120B" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": "high", "setting": "Bedrock / Splunk MCP, history kept", "solved": 11, "total": 59, "rank": 83, "place": "83", "tied": true, "held": false, "reported": false, "cost": 11.819719, "cost_mark": null, "cost_note": null, "cost_label": "$11.82", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.07452, "solved_sec": 1886.73, "median_sec": 192.965, "run_date": "2026-10-08", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk curl", "solved": 6, "total": 59, "rank": 92, "place": "92", "tied": false, "held": false, "reported": false, "cost": 2.858, "cost_mark": null, "cost_note": null, "cost_label": "$2.86", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.476333, "solved_sec": 219.833, "median_sec": 27.812, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "high", "setting": "No-tools control", "solved": 2, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "High", "setting": "MCP / Literal / 360s", "solved": 22, "total": 59, "rank": 45, "place": "45", "tied": false, "held": false, "reported": false, "cost": 4.604847, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.2093112272727273, "solved_sec": 1491.567, "median_sec": 46.079499999999996, "run_date": "2026-10-08", "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 18, "total": 59, "rank": 47, "place": "47", "tied": false, "held": false, "reported": false, "cost": 4.920117, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.27334, "solved_sec": 990.904, "median_sec": 20.735500000000002, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 15, "total": 59, "rank": 48, "place": "48", "tied": false, "held": false, "reported": false, "cost": 3.661587, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.244106, "solved_sec": 754.567, "median_sec": 36.391, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Nemotron Super 3 120B", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.3728813559322034 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 }, { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "high · Bedrock / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "nemotron-3-ultra-nvfp4", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western", "better_for_cost" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-oss-120b", "run": 0, "reasons": [ "similar_size" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 4, "reasons": [ "faster" ] }, { "key": "muse-glimmer-30b", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "kimi-k2-thinking", "name": "Kimi-K2-Thinking", "maker": "Moonshot", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2025-11-06", "year": 2025, "year_basis": "release", "card": null, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 45, "tied": false, "of": 49, "score": 7 }, "cybt_open": { "place": 24, "tied": false, "of": 28, "score": 7 }, "cybt_proprietary": null, "botsv3": { "place": 24, "tied": false, "of": 33, "score": 27 }, "botsv3_open": { "place": 11, "tied": false, "of": 17, "score": 27 }, "botsv3_proprietary": null }, "aliases": [ "Kimi K2 Thinking", "Kimi-K2-Thinking" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk curl", "solved": 7, "total": 59, "rank": 90, "place": "90", "tied": true, "held": false, "reported": false, "cost": 24.92355, "cost_mark": "*", "cost_note": "Known subtotal, not the bill. This lane never recorded each question's final request. From the traces those add at least $2.62. AWS's bill for that day, less the model's other runs, caps the total at $33.05.", "cost_label": "$24.92*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 3.560507, "solved_sec": 853.681, "median_sec": 102.657, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 27, "total": 59, "rank": 39, "place": "39", "tied": false, "held": false, "reported": false, "cost": 19.756351, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.731717, "solved_sec": 1160.453, "median_sec": 32.168, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 27, "total": 59, "rank": 40, "place": "40", "tied": false, "held": false, "reported": false, "cost": 15.846431, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.586905, "solved_sec": 1792.2839999999999, "median_sec": 56.404, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Kimi K2 Thinking", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.4576271186440678 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "kimi-k3", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "cheaper" ] }, { "key": "haiku-5.5", "run": 2, "reasons": [ "faster" ] }, { "key": "gpt-6.1-sol", "run": 0, "reasons": [ "better_for_cost" ] }, { "key": "sonnet-4.5", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "mistral-small-4", "name": "Mistral Small 4", "maker": "Mistral AI", "country": "France", "country_code": "FR", "flag": "🇫🇷", "region": "western", "kind": "open", "released_at": "2026-03-16", "year": 2026, "year_basis": "release", "card": { "src": "assets/mistral-small-4-field-card.webp", "title": "Le Petit Coq", "family": false }, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals" ], "ranks": { "cybt": { "place": 46, "tied": false, "of": 49, "score": 5 }, "cybt_open": { "place": 25, "tied": false, "of": 28, "score": 5 }, "cybt_proprietary": null, "botsv3": null, "botsv3_open": null, "botsv3_proprietary": null }, "aliases": [ "Mistral Small 4" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Mistral API / Splunk MCP, history kept", "solved": 5, "total": 59, "rank": 93, "place": "93", "tied": false, "held": false, "reported": false, "cost": 1.430197, "cost_mark": "*", "cost_note": "Priced at Mistral's listed rate. Title calls are not recorded and no Mistral bill has been read.", "cost_label": "$1.43*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.286039, "solved_sec": 248.572, "median_sec": 28.834, "run_date": "2026-10-22", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": null, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Mistral API / Splunk MCP, history kept", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "mistral-large-4", "run": 0, "reasons": [ "other_size", "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "nemotron-super-3-120b", "run": 1, "reasons": [ "faster" ] }, { "key": "nemotron-lightning-3.5-30b-a3b", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-oss-20b", "name": "GPT-OSS-20B", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2025-08-05", "year": 2025, "year_basis": "release", "card": null, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 47, "tied": false, "of": 49, "score": 4 }, "cybt_open": { "place": 26, "tied": false, "of": 28, "score": 4 }, "cybt_proprietary": null, "botsv3": { "place": 29, "tied": true, "of": 33, "score": 9 }, "botsv3_open": { "place": 15, "tied": true, "of": 17, "score": 9 }, "botsv3_proprietary": null }, "aliases": [ "GPT-OSS 20B", "GPT-OSS-20B" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk curl", "solved": 4, "total": 59, "rank": 94, "place": "94", "tied": false, "held": false, "reported": false, "cost": 0.195444, "cost_mark": "*", "cost_note": "Known subtotal, not the bill. This lane never recorded each question's final request. From the traces those add at least $0.03. AWS's bill for that day caps the total at $0.39.", "cost_label": "$0.1954*", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.048861, "solved_sec": 545.172, "median_sec": 159.44, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 1, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 9, "total": 59, "rank": 50, "place": "50", "tied": false, "held": false, "reported": false, "cost": 0.3042, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.0338, "solved_sec": 287.41, "median_sec": 22.289, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GPT-OSS 20B", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.15254237288135594 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "gpt-oss-120b", "run": 0, "reasons": [ "other_size" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "maker_top", "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "qwen3.8-27b", "run": 0, "reasons": [ "similar_size" ] }, { "key": "nemotron-super-3-120b", "run": 1, "reasons": [ "faster" ] }, { "key": "llama-4-maverick", "run": 0, "reasons": [ "similar_cost" ] } ] } }, { "key": "glm-4.7-flash", "name": "GLM 4.7 Flash", "maker": "Z.ai", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2025-12-01", "year": 2025, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "gpt56", "glm", "hygiene", "agreement" ], "ranks": { "cybt": { "place": 48, "tied": false, "of": 49, "score": 3 }, "cybt_open": { "place": 27, "tied": false, "of": 28, "score": 3 }, "cybt_proprietary": null, "botsv3": { "place": 29, "tied": true, "of": 33, "score": 9 }, "botsv3_open": { "place": 15, "tied": true, "of": 17, "score": 9 }, "botsv3_proprietary": null }, "aliases": [ "GLM 4.7 Flash" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / search MCP", "solved": 3, "total": 59, "rank": 95, "place": "95", "tied": false, "held": false, "reported": false, "cost": 3.900756, "cost_mark": null, "cost_note": null, "cost_label": "$3.90", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.300252, "solved_sec": 476.494, "median_sec": 220.38, "run_date": "2026-06-21", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Bedrock MCP / Literal / 360s", "solved": 9, "total": 59, "rank": 51, "place": "51", "tied": false, "held": false, "reported": false, "cost": 2.981199, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.331244, "solved_sec": 369.07, "median_sec": 20.673, "run_date": "2026-06-21", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "GLM 4.7 Flash", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answer-only baseline is 0.0% versus 15.3% with tool assistance, so current aggregate evidence does not flag this model as benchmark-contaminated.", "no_tools_rate": 0, "with_tools_rate": 0.15254237288135594 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "cybt", "picks": [ { "key": "glm-5.3-flash", "run": 0, "reasons": [ "newer", "maker_top", "top_eastern" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern", "similar_cost" ] }, { "key": "gpt-oss-20b", "run": 0, "reasons": [ "cheaper" ] }, { "key": "nemotron-super-3-120b", "run": 1, "reasons": [ "faster" ] }, { "key": "gpt-5.6-luna", "run": 0, "reasons": [ "better_for_cost" ] } ] } }, { "key": "llama-4-maverick", "name": "Llama 4 Maverick", "maker": "Meta", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "open", "released_at": "2025-04-05", "year": 2025, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "open", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": { "place": 49, "tied": false, "of": 49, "score": 2 }, "cybt_open": { "place": 28, "tied": false, "of": 28, "score": 2 }, "cybt_proprietary": null, "botsv3": { "place": 32, "tied": false, "of": 33, "score": 5 }, "botsv3_open": { "place": 17, "tied": false, "of": 17, "score": 5 }, "botsv3_proprietary": null }, "aliases": [ "Llama 4 Maverick" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk curl", "solved": 2, "total": 59, "rank": 96, "place": "96", "tied": false, "held": false, "reported": false, "cost": 0.352, "cost_mark": null, "cost_note": null, "cost_label": "$0.3520", "cost_approximate": false, "cost_estimated": false, "score_note": "Solved cases out of the suite.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.176, "solved_sec": 15.479, "median_sec": 7.739, "run_date": null, "effort_label": "not adjustable", "effort_note": "This model has no adjustable reasoning effort on Bedrock." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not adjustable", "effort_note": "This model has no adjustable reasoning effort on Bedrock." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 5, "total": 59, "rank": 53, "place": "53", "tied": false, "held": false, "reported": false, "cost": 0.236173, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.047235, "solved_sec": 32.818, "median_sec": 5.863, "run_date": "2026-06-10", "effort_label": "not adjustable", "effort_note": "This model has no adjustable reasoning effort on Bedrock." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 3, "total": 59, "rank": 54, "place": "54", "tied": false, "held": false, "reported": false, "cost": 0.253735, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.084578, "solved_sec": 14.257000000000001, "median_sec": 4.75, "run_date": "2026-06-10", "effort_label": "not adjustable", "effort_note": "This model has no adjustable reasoning effort on Bedrock." } ], "hygiene": { "label": "Llama 4 Maverick", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": null, "reviewed": false, "note": "Answered 0.0% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0, "with_tools_rate": 0.0847457627118644 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "cybt", "picks": [ { "key": "muse-glimmer-30b", "run": 0, "reasons": [ "maker_top" ] }, { "key": "gpt-5.6-sol", "run": 0, "reasons": [ "top", "top_proprietary" ] }, { "key": "reflection-beam-1.0", "run": 0, "reasons": [ "top_open", "top_western", "newest_western" ] }, { "key": "glm-5.3-flash", "run": 0, "reasons": [ "top_eastern" ] }, { "key": "deepseek-v4.1-flash", "run": 0, "reasons": [ "newest_eastern" ] }, { "key": "gpt-oss-20b", "run": 0, "reasons": [ "cheaper", "better_for_cost", "similar_cost" ] } ] } }, { "key": "opus-4.6", "name": "Claude Opus 4.6", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-02-19", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "hygiene", "cheating" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 5, "tied": false, "of": 33, "score": 39 }, "botsv3_open": null, "botsv3_proprietary": { "place": 5, "tied": false, "of": 16, "score": 39 } }, "aliases": [ "Claude Opus 4.6" ], "runs": [ { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "High", "setting": "MCP / Literal / 360s", "solved": 39, "total": 59, "rank": 6, "place": "6", "tied": false, "held": false, "reported": false, "cost": 12.750688, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.310992, "solved_sec": 2561.411, "median_sec": 33.251, "run_date": "2026-05-05", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "Claude Opus 4.6", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": 0.15254237288135594, "reviewed": true, "note": "9 answers from the answer-only baseline matched known-answer patterns (9/59 = 15.3%). These answers may come from prior benchmark exposure.", "no_tools_rate": 0.1694915254237288, "with_tools_rate": 0.7288135593220338 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "botsv3", "picks": [ { "key": "opus-4.5", "run": 1, "reasons": [ "previous" ] }, { "key": "opus-4.7", "run": 2, "reasons": [ "newer" ] }, { "key": "opus-5.5", "run": 6, "reasons": [ "maker_top", "top", "top_proprietary", "top_western", "better_for_cost" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "newest_western" ] }, { "key": "sonnet-5.5", "run": 6, "reasons": [ "cheaper", "faster" ] }, { "key": "glm-5.1", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-5.3-codex", "name": "GPT-5.3-codex", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-01-15", "year": 2026, "year_basis": "release", "card": null, "discussed": [ "agreement" ], "listed": [ "home", "hygiene" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 8, "tied": true, "of": 33, "score": 37 }, "botsv3_open": null, "botsv3_proprietary": { "place": 7, "tied": true, "of": 16, "score": 37 } }, "aliases": [ "GPT-5.3-codex" ], "runs": [ { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 37, "total": 59, "rank": 11, "place": "11", "tied": false, "held": false, "reported": false, "cost": 59.36101, "cost_mark": null, "cost_note": null, "cost_per_solved": 1.5621318421052632, "solved_sec": 2650.126, "median_sec": 44.9195, "run_date": "2026-05-06", "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "High", "setting": "MCP / Literal / 360s", "solved": 35, "total": 59, "rank": 24, "place": "24", "tied": false, "held": false, "reported": false, "cost": 63.21529, "cost_mark": null, "cost_note": null, "cost_per_solved": 1.7559802777777778, "solved_sec": 2035.074, "median_sec": 38.6725, "run_date": "2026-05-08", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-5.3-codex", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 2 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.6440677966101694 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "botsv3", "picks": [ { "key": "opus-5.5", "run": 6, "reasons": [ "top", "top_proprietary", "top_western", "better_for_cost" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 2, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 6, "reasons": [ "faster" ] }, { "key": "gpt-5.4", "run": 1, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-5.4-mini", "name": "GPT-5.4-mini", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-02-15", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "hygiene", "cheating" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 14, "tied": true, "of": 33, "score": 35 }, "botsv3_open": null, "botsv3_proprietary": { "place": 12, "tied": false, "of": 16, "score": 35 } }, "aliases": [ "GPT-5.4-mini" ], "runs": [ { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "Medium", "setting": "MCP / Literal / 360s", "solved": 35, "total": 59, "rank": 20, "place": "20", "tied": false, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": 1523.35, "median_sec": 28.608, "run_date": "2026-05-08", "effort_label": "medium", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 34, "total": 59, "rank": 25, "place": "25", "tied": false, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": 1545.776, "median_sec": 30.194, "run_date": "2026-05-07", "effort_label": "low", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "High", "setting": "MCP / Literal / 360s", "solved": 34, "total": 59, "rank": 26, "place": "26", "tied": false, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": 1609.112, "median_sec": 28.958, "run_date": "2026-05-08", "effort_label": "high", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-5.4-mini", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.5932203389830508 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "botsv3", "picks": [ { "key": "gpt-5.3-codex", "run": 0, "reasons": [ "maker_top" ] }, { "key": "opus-5.5", "run": 6, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "newest_western" ] }, { "key": "sonnet-5.5", "run": 7, "reasons": [ "faster" ] } ] } }, { "key": "sonnet-4.6", "name": "Claude Sonnet 4.6", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-02-19", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "hygiene" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 21, "tied": true, "of": 33, "score": 29 }, "botsv3_open": null, "botsv3_proprietary": { "place": 13, "tied": false, "of": 16, "score": 29 } }, "aliases": [ "Claude Sonnet 4.6" ], "runs": [ { "bench": "botsv3", "tools": "with", "harness": "Claude Code", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 29, "total": 59, "rank": 35, "place": "35", "tied": false, "held": false, "reported": false, "cost": 7.1859, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.24779, "solved_sec": 1625.681, "median_sec": 22.721, "run_date": "2026-04-29", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Claude Sonnet 4.6", "verdict": "Contaminated", "tone": "warn", "confirmed_rate": 0.06779661016949153, "reviewed": true, "note": "4 answers from the answer-only baseline matched known-answer patterns (4/59 = 6.8%). These answers may come from prior benchmark exposure.", "no_tools_rate": 0.1016949152542373, "with_tools_rate": 0.6440677966101694 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "botsv3", "picks": [ { "key": "sonnet-4.5", "run": 6, "reasons": [ "previous" ] }, { "key": "sonnet-5.5", "run": 6, "reasons": [ "newer", "better_for_cost" ] }, { "key": "opus-5.5", "run": 6, "reasons": [ "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "newest_western", "cheaper" ] }, { "key": "sonnet-5.5", "run": 7, "reasons": [ "faster" ] }, { "key": "amazon-nova-pro", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "minimax-m2", "name": "MiniMax M2", "maker": "MiniMax", "country": "China", "country_code": "CN", "flag": "🇨🇳", "region": "eastern", "kind": "open", "released_at": "2025-10-28", "year": 2025, "year_basis": "release", "card": null, "discussed": [ "open" ], "listed": [ "home", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 21, "tied": true, "of": 33, "score": 29 }, "botsv3_open": { "place": 9, "tied": false, "of": 17, "score": 29 }, "botsv3_proprietary": null }, "aliases": [ "MiniMax M2" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Fireworks · Splunk MCP", "solved": 16, "total": 59, "rank": 77, "place": "77", "tied": true, "held": true, "reported": false, "cost": 6.966675, "cost_mark": null, "cost_note": null, "cost_label": "$6.97", "cost_approximate": false, "cost_estimated": false, "score_note": "Original scoring. The per-question record for this run was not found, so it has not been re-scored under the corrected rubric or the 5-minute cap.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 0.435417, "solved_sec": 1442.031, "median_sec": 132.726, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 29, "total": 59, "rank": 37, "place": "37", "tied": false, "held": false, "reported": false, "cost": 10.184605, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.351193, "solved_sec": 2075.962, "median_sec": 51.861, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 24, "total": 59, "rank": 44, "place": "44", "tied": false, "held": false, "reported": false, "cost": 7.900267, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.329178, "solved_sec": 1870.559, "median_sec": 53.034499999999994, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "MiniMax M2", "verdict": "Answered nothing without tools", "tone": "measured", "confirmed_rate": null, "reviewed": false, "note": "Answered 0.0% of questions with no tools. Per-question review has not run.", "no_tools_rate": 0, "with_tools_rate": 0.4915254237288136 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "botsv3", "picks": [ { "key": "minimax-m3", "run": 2, "reasons": [ "newer", "maker_top" ] }, { "key": "opus-5.5", "run": 6, "reasons": [ "top", "top_proprietary", "better_for_cost" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open", "top_eastern" ] }, { "key": "inkling", "run": 2, "reasons": [ "top_western" ] }, { "key": "muse-glimmer-30b", "run": 3, "reasons": [ "newest_western" ] }, { "key": "deepseek-v4.1-flash", "run": 2, "reasons": [ "newest_eastern" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 7, "reasons": [ "faster" ] }, { "key": "opus-4.8", "run": 10, "reasons": [ "similar_cost" ] } ] } }, { "key": "gpt-5.4", "name": "GPT-5.4", "maker": "OpenAI", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-02-15", "year": 2026, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "gpt56", "refusals", "hygiene", "cheating" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 26, "tied": false, "of": 33, "score": 24 }, "botsv3_open": null, "botsv3_proprietary": { "place": 14, "tied": false, "of": 16, "score": 24 } }, "aliases": [ "GPT-5.4" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "Codex", "effort": "high", "setting": "OpenAI · Splunk MCP", "solved": 18, "total": 59, "rank": 69, "place": "69", "tied": true, "held": true, "reported": false, "cost": 24.719422, "cost_mark": "*", "cost_note": "Known subtotal, not the bill: 45 of 59 questions, at the standard rate. Long-context pricing could raise those 45 to $46.41. Fourteen questions have no usage record.", "cost_label": "$24.72*", "cost_approximate": false, "cost_estimated": false, "score_note": "Original scoring. The per-question record for this run was not found, so it has not been re-scored under the corrected rubric or the 5-minute cap.", "time_note": "Sum of time on solved cases.", "median_note": "Median time on solved cases.", "cost_per_solved": 1.373301, "solved_sec": 1677.259, "median_sec": 129.032, "run_date": null, "effort_label": "high", "effort_note": "Effort setting recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "Codex", "effort": "Low", "setting": "MCP / Literal / 360s", "solved": 24, "total": 59, "rank": 43, "place": "43", "tied": false, "held": false, "reported": false, "cost": 53.931708, "cost_mark": null, "cost_note": null, "cost_per_solved": 2.2471545, "solved_sec": 1522.958, "median_sec": 47.1885, "run_date": "2026-05-07", "effort_label": "low", "effort_note": "Effort setting recorded for this run." } ], "hygiene": { "label": "GPT-5.4", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 2 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.4576271186440678 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "Codex", "variant": "High · Splunk MCP", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "botsv3", "picks": [ { "key": "gpt-5.5", "run": 3, "reasons": [ "newer" ] }, { "key": "gpt-5.3-codex", "run": 0, "reasons": [ "maker_top", "similar_cost" ] }, { "key": "opus-5.5", "run": 6, "reasons": [ "top", "top_proprietary", "top_western", "better_for_cost" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "newest_western", "cheaper" ] }, { "key": "opus-4.7", "run": 4, "reasons": [ "faster" ] } ] } }, { "key": "amazon-nova-pro", "name": "Amazon Nova Pro", "maker": "Amazon", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2024-12-03", "year": 2024, "year_basis": "release", "card": null, "discussed": [], "listed": [ "home", "gpt56", "refusals", "hygiene" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 31, "tied": false, "of": 33, "score": 8 }, "botsv3_open": null, "botsv3_proprietary": { "place": 15, "tied": false, "of": 16, "score": 8 } }, "aliases": [ "Amazon Nova Pro", "Nova Pro" ], "runs": [ { "bench": "cybt", "tools": "with", "harness": "opencode", "effort": null, "setting": "Bedrock / Splunk curl", "solved": 1, "total": 59, "rank": 97, "place": "97", "tied": false, "held": true, "reported": false, "cost": 3.703676, "cost_mark": null, "cost_note": null, "cost_label": "$3.70", "cost_approximate": false, "cost_estimated": false, "score_note": "Original scoring. The per-question record for this run was not found, so it has not been re-scored under the corrected rubric or the 5-minute cap.", "time_note": "No solved-time total was recorded for this run.", "median_note": "Median time on solved cases.", "cost_per_solved": 3.703676, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "cybt", "tools": "none", "harness": "opencode", "effort": "Standard", "setting": "No-tools control", "solved": 0, "total": 59, "rank": null, "place": null, "held": false, "reported": false, "cost": null, "cost_mark": null, "cost_note": null, "cost_per_solved": null, "solved_sec": null, "median_sec": null, "run_date": null, "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "MCP / Literal / 360s", "solved": 8, "total": 59, "rank": 52, "place": "52", "tied": false, "held": false, "reported": false, "cost": 7.696991, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.962124, "solved_sec": 93.037, "median_sec": 9.083, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." }, { "bench": "botsv3", "tools": "with", "harness": "opencode", "effort": "Standard", "setting": "Curl / Literal / 300s", "solved": 3, "total": 59, "rank": 55, "place": "55", "tied": false, "held": false, "reported": false, "cost": 31.923305, "cost_mark": null, "cost_note": null, "cost_per_solved": 10.641102, "solved_sec": 319.85699999999997, "median_sec": 12.827, "run_date": "2026-06-10", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Amazon Nova Pro", "verdict": "No contamination detected", "tone": "measured", "confirmed_rate": 0, "reviewed": true, "note": "Answered 1 of 59 with no tools, and a reviewer found every one of those answers in public documentation or too short to prove anything.", "no_tools_rate": 0.01694915254237288, "with_tools_rate": 0.13559322033898305 }, "cybt_hygiene": null, "refusals": [ { "benchmark": "CyBT-CTF", "harness": "opencode", "variant": "Standard · Bedrock / Splunk curl", "tools": "with tools", "refusals": null, "abstained": null, "total": 59 } ], "related": { "bench": "botsv3", "picks": [ { "key": "opus-5.5", "run": 6, "reasons": [ "top", "top_proprietary", "top_western" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "newest_western" ] }, { "key": "gpt-oss-20b", "run": 2, "reasons": [ "cheaper" ] }, { "key": "sonnet-5.5", "run": 6, "reasons": [ "better_for_cost" ] }, { "key": "minimax-m2", "run": 2, "reasons": [ "similar_cost" ] } ] } }, { "key": "fable-5-stock-route", "name": "Fable 5 stock route", "maker": "Anthropic", "country": "United States", "country_code": "US", "flag": "🇺🇸", "region": "western", "kind": "proprietary", "released_at": "2026-06-10", "year": 2026, "year_basis": "release", "card": { "src": "assets/fable-5-field-card.webp", "title": null, "family": true }, "discussed": [], "listed": [ "home", "hygiene" ], "ranks": { "cybt": null, "cybt_open": null, "cybt_proprietary": null, "botsv3": { "place": 33, "tied": false, "of": 33, "score": 2 }, "botsv3_open": null, "botsv3_proprietary": { "place": 16, "tied": false, "of": 16, "score": 2 } }, "aliases": [ "Fable 5 stock route" ], "runs": [ { "bench": "botsv3", "tools": "none", "harness": "Claude Code", "effort": "Standard", "setting": "Anthropic.com stock account / no tools", "solved": 2, "total": 59, "rank": 56, "place": "56", "tied": false, "held": false, "reported": false, "cost": 0.682136, "cost_mark": null, "cost_note": null, "cost_per_solved": 0.341068, "solved_sec": 298.194, "median_sec": 3.863, "run_date": "2026-06-11", "effort_label": "not recorded", "effort_note": "The effective effort setting is not recorded for this run." } ], "hygiene": { "label": "Fable 5 stock route", "verdict": "Needs review", "tone": "review", "confirmed_rate": 0, "reviewed": true, "note": "High hand-off rate: 51 / 59 attempts were handed to Opus, so the row is a stock-route diagnostic rather than a clean Fable-only score.", "no_tools_rate": 0.03389830508474576, "with_tools_rate": 0.03389830508474576 }, "cybt_hygiene": null, "refusals": [], "related": { "bench": "botsv3", "picks": [ { "key": "opus-5.5", "run": 6, "reasons": [ "maker_top", "top", "top_proprietary", "top_western" ] }, { "key": "kimi-k3", "run": 2, "reasons": [ "top_open" ] }, { "key": "haiku-5.5", "run": 3, "reasons": [ "newest_western", "better_for_cost", "similar_cost" ] }, { "key": "llama-4-maverick", "run": 2, "reasons": [ "cheaper" ] }, { "key": "llama-4-maverick", "run": 3, "reasons": [ "faster" ] } ] } } ] }