{
  "schemaVersion": "eve-engineering-benchmark.v1",
  "documentType": "candidate-catalog",
  "benchmarkVersion": "1.0.0",
  "frozenAt": "2026-09-12T17:00:11Z",
  "visibility": "candidate-readable",
  "split": "held-out",
  "contextPolicy": {
    "materialization": "git-archive",
    "repositoryHistory": "base-snapshot-without-git-directory",
    "network": "deny",
    "caseExposure": "one-selected-case",
    "candidateIncludes": ["case-metadata", "task-brief", "constraints", "base-workspace-tree"],
    "candidateExcludes": [
      "oracle-commit",
      "acceptance-criteria",
      "verification-commands",
      "source-provenance",
      "evaluator-canaries",
      "current-or-future-git-history",
      "other-case-prompts"
    ],
    "onViolation": "refuse-and-discard-run"
  },
  "cases": [
    {
      "id": "eng-v1-001",
      "taskClass": "diagnosis-only",
      "variant": "repository",
      "split": "held-out",
      "sourceSnapshot": "22f25c733a19c62345d1a77761613adb89a7d6f1",
      "taskBrief": "Audit the human-video phase for capability claims that are not backed by the implementation. Produce a source-linked diagnosis that distinguishes real bindings, deliberate refusals, test-only doubles, fabricated metrics, and any remaining contract weakness.",
      "constraints": [
        "Do not implement or silently repair the capability gaps you find.",
        "Do not claim that a source seam or mock proves a deployed runtime.",
        "Keep every unresolved weakness explicit and evidence-linked."
      ],
      "expectedDeliverable": "A diagnosis-only audit record and truthful tracker updates.",
      "riskTier": "medium",
      "taskSize": "medium",
      "runtimeRequirements": ["repository"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The brief paraphrases engineering findings and excludes commit authors, sessions, raw logs, and external identifiers.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-002",
      "taskClass": "bug-fix",
      "variant": "repository",
      "split": "held-out",
      "sourceSnapshot": "ec052572a2d2ff892984040fa668080e52132ec8",
      "taskBrief": "Fix the MP4 probe so track-header dimensions are decoded correctly for both version-0 and version-1 boxes, without weakening codec or duration parsing. Add a regression that would fail if offsets were still tied to the shorter header layout.",
      "constraints": [
        "Use the box structure rather than a hard-coded version-specific coincidence.",
        "Preserve the existing null-on-malformed-input behavior.",
        "Run the focused media-probe tests."
      ],
      "expectedDeliverable": "A narrow parser repair with a version-1 regression test.",
      "riskTier": "medium",
      "taskSize": "small",
      "runtimeRequirements": ["repository", "node"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The brief carries only synthetic container-format behavior and no media payloads or user metadata.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-003",
      "taskClass": "feature",
      "variant": "repository",
      "split": "held-out",
      "sourceSnapshot": "d14c248e76ec0b97d2950cbdb1a3c99174565435",
      "taskBrief": "Add release-aware expectation selection to the assistant evaluation deck. The current and native-app releases must retain the honest deferred-room behavior, the later room-opening release must restore the saved expectations, and malformed or unsupported release inputs must fail closed before provider work begins.",
      "constraints": [
        "Read the shipping release from the authoritative navigation contract.",
        "Do not change ordinary evaluation cases or silently default unknown releases.",
        "Keep the transitional advisory metadata until a separate measurement decision."
      ],
      "expectedDeliverable": "A typed release selector wired into the real evaluation runner with focused tests.",
      "riskTier": "medium",
      "taskSize": "medium",
      "runtimeRequirements": ["repository", "node"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The task contains release semantics only; provider transcripts, tokens, and member content are excluded.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-004",
      "taskClass": "refactor",
      "variant": "repository",
      "split": "held-out",
      "sourceSnapshot": "268e31a8b1d534ab12db9d81a9a33dbb4322c152",
      "taskBrief": "Retire the obsolete AI-video package while preserving its provider-independent lens-physics capability in the package that owns frame enhancement. Remove dead provider surfaces and update consumers, exports, dependency rules, tests, and documentation without changing the optics behavior.",
      "constraints": [
        "Preserve the lens-physics API and numerical behavior through the move.",
        "Remove obsolete package and generated-provider references completely.",
        "Demonstrate that the receiving package owns and tests the relocated module."
      ],
      "expectedDeliverable": "A behavior-preserving package retirement and module relocation.",
      "riskTier": "medium",
      "taskSize": "large",
      "runtimeRequirements": ["repository", "node"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "Only package architecture and synthetic optics behavior are retained; no generated media or provider payload is included.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-005",
      "taskClass": "migration-contract",
      "variant": "service",
      "split": "held-out",
      "sourceSnapshot": "ea17a2a69777f4746462fa7f4d8f05a38094ff7b",
      "taskBrief": "Implement the approved persistence migration for the settled Metis aggregates as a resumable expand, backfill, verify, cutover, and contract sequence. Bind the generated database migration, service contract, API surface, and real-database tests while refusing unresolved aggregate designs.",
      "constraints": [
        "The phase order and source-state-bound cutover permit are safety properties.",
        "Backfill must be checkpointed and idempotent, and destructive contraction must remain last.",
        "Prove tenant isolation, restart behavior, drift detection, and rollback on a real database."
      ],
      "expectedDeliverable": "A versioned migration contract, generated schema change, service path, and database verification.",
      "riskTier": "high",
      "taskSize": "large",
      "runtimeRequirements": ["repository", "python", "postgresql"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The benchmark uses synthetic tenant and aggregate fixtures; connection values and database contents are never copied into the corpus.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-006",
      "taskClass": "security-repair",
      "variant": "service",
      "split": "held-out",
      "sourceSnapshot": "36de995efb2949f9cf58a991c1ba0fcf6174c70f",
      "taskBrief": "Harden operator memory against prompt injection, sensitive-data admission, unauthorized model-triggered writes, and deletion resurrection. Preserve explicit operator control and same-subject isolation, add durable erasure fencing, and extend adversarial and persistence coverage.",
      "constraints": [
        "Memory text remains untrusted data and cannot grant authority or trigger tools.",
        "Account erasure must serialize with writers and remain fenced after restart.",
        "Do not store raw subjects in the deletion fence or add a semantic/vector write path."
      ],
      "expectedDeliverable": "A fail-closed memory security repair with adversarial, route, and database regressions.",
      "riskTier": "high",
      "taskSize": "large",
      "runtimeRequirements": ["repository", "node", "postgresql"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "All attack examples are synthetic and the task excludes real subjects, credentials, conversations, and retained model text.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-007",
      "taskClass": "docs-product-graph",
      "variant": "repository",
      "split": "held-out",
      "sourceSnapshot": "59ccc55a36333cdec8a9fe7b52fffce35906f225",
      "taskBrief": "Close the final documentation and curation phase of the product-graph restructure. Record the shipped architecture, keep compiler failures honest, regenerate the content-addressed graph, and reconcile the phase checklist without weakening graph totality or projection boundaries.",
      "constraints": [
        "A missing optional tree may be empty, but permission and I/O failures must not silently shrink the corpus.",
        "Generated graph sections must remain content-addressed and reproducible.",
        "Document artifact truth separately from disposable database projections."
      ],
      "expectedDeliverable": "A source-linked closeout, hardened corpus traversal, and regenerated product graph.",
      "riskTier": "medium",
      "taskSize": "large",
      "runtimeRequirements": ["repository", "node"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The case retains repository documentation structure and content hashes but excludes user documents and search queries.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-008",
      "taskClass": "ui-playwright",
      "variant": "browser",
      "split": "held-out",
      "sourceSnapshot": "2f4fa19a4f7626c48a1a0f1e366ac894711d0b1a",
      "taskBrief": "Repair the Metis authoring feedback regions so screen readers announce state changes, then add keyboard-only browser coverage for reachability, document-order traversal, visible focus, trap freedom, and form completion. Keep the existing accessibility scan active.",
      "constraints": [
        "Live regions must exist before their messages are inserted.",
        "The browser journey must use keyboard input rather than pointer shortcuts.",
        "Distinguish product controls from development-only browser chrome."
      ],
      "expectedDeliverable": "Accessible UI behavior with source-level regression and Playwright keyboard coverage.",
      "riskTier": "medium",
      "taskSize": "medium",
      "runtimeRequirements": ["repository", "node", "browser", "accessibility-api"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The browser fixtures use synthetic form values and retain no screenshots, credentials, or learner records.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-009",
      "taskClass": "service-integration",
      "variant": "service",
      "split": "held-out",
      "sourceSnapshot": "cc734aa82d1a5a37051ba87e0a51a6d8036cc121",
      "taskBrief": "Complete the LTI 1.3 browser launch handshake against a real local learning platform: accept GET and form-post login initiation, issue the correct redirect semantics, receive the signed callback, enforce registration and key trust, update the contract, and retain a sanitized driver and receipt.",
      "constraints": [
        "Plain HTTP key retrieval is allowed only for literal loopback hosts; deceptive hostnames remain refused.",
        "The browser redirect after a posted initiation must switch to GET.",
        "Do not retain launch tokens, passwords, cookies, or learner identity in benchmark evidence."
      ],
      "expectedDeliverable": "A standards-bound service integration with contract tests and a real local-platform receipt.",
      "riskTier": "high",
      "taskSize": "large",
      "runtimeRequirements": ["repository", "python", "service"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The task describes protocol shapes only; real launch tokens, credentials, cookies, names, and email addresses are excluded.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-010",
      "taskClass": "dcc-native",
      "variant": "dcc",
      "split": "held-out",
      "sourceSnapshot": "5028324227195fcde48476f2787f0761b98e8ef1",
      "taskBrief": "Pin and smoke the actual Blender executable outside the model loop through the shipped stdin RPC bridge. Exercise handshake, health, a safe scene edit, native project save/readback, two export formats, and an absent-executable refusal with an honest host-scoped record.",
      "constraints": [
        "Use the real Blender executable selected by the production candidate order.",
        "Verify saved and exported bytes independently rather than trusting bridge success text.",
        "Do not generalize one headless Linux observation to other hosts or transports."
      ],
      "expectedDeliverable": "A real-DCC integration smoke and bounded runtime evidence record.",
      "riskTier": "high",
      "taskSize": "medium",
      "runtimeRequirements": ["repository", "node", "blender"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "Only synthetic scene geometry and executable metadata are retained; generated project files remain ephemeral.",
        "retention": "versioned-internal-benchmark"
      }
    },
    {
      "id": "eng-v1-011",
      "taskClass": "dcc-native",
      "variant": "native-desktop",
      "split": "held-out",
      "sourceSnapshot": "6065e42ad81a572eb06c7d915981fda9f2591e60",
      "taskBrief": "Build a browser-independent native desktop fixture that proves real capture and input across a known application, two displays, HiDPI coordinates, and the accessibility API. Refuse occluded or unverified targets, redact screenshots before retention, and clean up every process and display resource.",
      "constraints": [
        "Playwright evidence cannot substitute for the native desktop boundary.",
        "Input must be gated on the expected visible target and verified end state.",
        "Screenshots are redacted, mode-restricted, short-lived, and never retained raw."
      ],
      "expectedDeliverable": "A native desktop fixture, cross-platform bindings, and privacy-safe execution evidence.",
      "riskTier": "high",
      "taskSize": "large",
      "runtimeRequirements": ["repository", "node", "native-desktop", "accessibility-api"],
      "dataGovernance": {
        "sourceRights": "repository-owned-development-work",
        "containsPersonalData": false,
        "containsCredentialValues": false,
        "containsRawConversations": false,
        "redaction": "The fixture uses synthetic pixels and text; raw screen captures are excluded and sanitized captures have bounded retention.",
        "retention": "versioned-internal-benchmark"
      }
    }
  ]
}