{
  "instances": {
    "total": 869,
    "userspace": 502,
    "v8": 181,
    "kernel": 186
  },
  "results": [
    {
      "model": "GPT-5.6 Sol (reasoning max)",
      "agent": "Codex CLI",
      "source": "OpenAI & ExploitGym Team",
      "source_url": "https://deploymentsafety.openai.com/gpt-5-6",
      "date": "2026-07-13",
      "eval_note": "6h timeout",
      "userspace": 149,
      "v8": 71,
      "kernel": 73,
      "on_target": 293,
      "mitigated": {
        "userspace": 60,
        "v8": 32,
        "kernel": 44
      },
      "metrics": {
        "cost": 203.24,
        "out_tokens": 690824,
        "time_min": 201.58,
        "llm_calls": 839.08
      },
      "metrics_success": {
        "cost": 70.77,
        "out_tokens": 278092,
        "time_min": 86.53,
        "llm_calls": 370.18
      },
      "version": "v1",
      "subRows": [
        {
          "eval_note": "2h timeout cutoff",
          "userspace": 119,
          "v8": 59,
          "kernel": 38,
          "on_target": 216,
          "mitigated": {
            "userspace": 41,
            "v8": 19,
            "kernel": 24
          },
          "metrics": {
            "cost": 83.59,
            "out_tokens": 302953,
            "time_min": 87.93,
            "llm_calls": 372.09
          },
          "metrics_success": {
            "cost": 32.85,
            "out_tokens": 141874,
            "time_min": 40.19,
            "llm_calls": 175.53
          }
        }
      ]
    },
    {
      "model": "Muse Spark 1.1 (helpful-only versoin)",
      "agent": "Meta Agent (off-the-shelf agent harness)",
      "source": "Meta AI",
      "source_url": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report",
      "date": "2026-07-09",
      "eval_note": "4h timeout",
      "userspace": 4,
      "v8": 3,
      "kernel": 0,
      "on_target": 7,
      "mitigated": {
        "userspace": 0,
        "v8": 0,
        "kernel": 0
      },
      "version": "v1"
    },
    {
      "model": "Claude Mythos Preview (Results obtained in collaboration with Anthropic.)",
      "agent": "Claude Code",
      "source": "ExploitGym Team",
      "source_url": null,
      "date": "2026-05-13",
      "eval_note": "2h timeout",
      "userspace": 107,
      "v8": 38,
      "kernel": 12,
      "on_target": 157,
      "flag_captured": 226,
      "mitigated": {
        "userspace": 25,
        "v8": 17,
        "kernel": 3
      },
      "metrics": {
        "cost": null,
        "out_tokens": null,
        "time_min": 106.4,
        "llm_calls": 695.3
      },
      "metrics_success": {
        "cost": null,
        "out_tokens": null,
        "time_min": 48.9,
        "llm_calls": 454.4
      },
      "version": "v0"
    },
    {
      "model": "GPT-5.5 (OpenAI's default safety filters block all GPT-5.5 exploit attempts under default prompting.)",
      "agent": "Codex CLI",
      "source": "ExploitGym Team",
      "source_url": null,
      "date": "2026-06-16",
      "eval_note": "2h timeout",
      "userspace": 70,
      "v8": 38,
      "kernel": 21,
      "on_target": 129,
      "flag_captured": 208,
      "mitigated": {
        "userspace": 10,
        "v8": 4,
        "kernel": 8
      },
      "metrics": {
        "cost": 35.43,
        "out_tokens": 168862,
        "time_min": 68.35,
        "llm_calls": 365
      },
      "metrics_success": {
        "cost": 20.73,
        "out_tokens": 115908,
        "time_min": 48.0,
        "llm_calls": 232.45
      },
      "version": "v1"
    },
    {
      "model": "GPT-5.4",
      "agent": "Codex CLI",
      "source": "ExploitGym Team",
      "source_url": null,
      "date": "2026-06-16",
      "eval_note": "2h timeout",
      "userspace": 38,
      "v8": 22,
      "kernel": 1,
      "on_target": 61,
      "mitigated": {
        "userspace": 2,
        "v8": 0,
        "kernel": 1
      },
      "metrics": {
        "cost": 25.25,
        "out_tokens": 312596,
        "time_min": 103.0,
        "llm_calls": 440.2
      },
      "metrics_success": {
        "cost": 12.15,
        "out_tokens": 165308,
        "time_min": 50.6,
        "llm_calls": 217.49
      },
      "version": "v1"
    },
    {
      "model": "Claude Opus 4.6 (Results obtained in collaboration with Anthropic.)",
      "agent": "Claude Code",
      "source": "ExploitGym Team",
      "source_url": null,
      "date": "2026-06-16",
      "eval_note": "2h timeout",
      "userspace": 12,
      "v8": 3,
      "kernel": 1,
      "on_target": 16,
      "mitigated": {
        "userspace": 0,
        "v8": 0,
        "kernel": 0
      },
      "metrics": {
        "cost": 21.76,
        "out_tokens": null,
        "time_min": 66.7,
        "llm_calls": 285.9
      },
      "metrics_success": {
        "cost": 8.56,
        "out_tokens": null,
        "time_min": 23.3,
        "llm_calls": 104.8
      },
      "version": "v1"
    },
    {
      "model": "Gemini 3.1 Pro",
      "agent": "Gemini CLI",
      "source": "ExploitGym Team",
      "source_url": null,
      "date": "2026-05-13",
      "eval_note": "2h timeout",
      "userspace": 10,
      "v8": 2,
      "kernel": 0,
      "on_target": 12,
      "mitigated": {
        "userspace": 0,
        "v8": 0,
        "kernel": 0
      },
      "metrics": {
        "cost": 8.97,
        "out_tokens": 114103,
        "time_min": 75.4,
        "llm_calls": 174.7
      },
      "metrics_success": {
        "cost": 9.02,
        "out_tokens": 94717,
        "time_min": 47.9,
        "llm_calls": 163.9
      },
      "version": "v0"
    },
    {
      "model": "Claude Opus 4.7",
      "agent": "Claude Code",
      "source": "ExploitGym Team",
      "source_url": null,
      "date": "2026-06-16",
      "eval_note": "2h timeout",
      "userspace": 4,
      "v8": 8,
      "kernel": 0,
      "on_target": 12,
      "mitigated": {
        "userspace": 0,
        "v8": 0,
        "kernel": 0
      },
      "metrics": {
        "cost": 4.82,
        "out_tokens": 41767,
        "time_min": 16.75,
        "llm_calls": 60.2
      },
      "metrics_success": {
        "cost": 24.05,
        "out_tokens": 189662,
        "time_min": 57.44,
        "llm_calls": 168.75
      },
      "version": "v1"
    },
    {
      "model": "GLM-5.1",
      "agent": "Claude Code",
      "source": "ExploitGym Team",
      "source_url": null,
      "date": "2026-06-16",
      "eval_note": "2h timeout",
      "userspace": 4,
      "v8": 0,
      "kernel": 0,
      "on_target": 4,
      "mitigated": {
        "userspace": 0,
        "v8": 0,
        "kernel": 0
      },
      "metrics": {
        "cost": 6.39,
        "out_tokens": 224573,
        "time_min": 118.0,
        "llm_calls": 246.0
      },
      "metrics_success": {
        "cost": 4.18,
        "out_tokens": 131744,
        "time_min": 82.2,
        "llm_calls": 151.2
      },
      "version": "v1"
    }
  ]
}