{
  "example_kind": "original synthetic manual planning case",
  "slug": "reviewer-agreement-case-log",
  "title": "Inspect reviewer agreement and actual disagreements",
  "scenario": "Synthetic planning case: Two reviewers independently apply the same toy acceptance rubric to five cases. They agree on A,B,C and disagree on D,E.",
  "layout": "evaluation",
  "dataset": {
    "heading": "Inspect the invented case records",
    "headers": [
      "Case",
      "Reviewer 1",
      "Reviewer 2",
      "Agreement"
    ],
    "rows": [
      [
        "A",
        "Pass",
        "Pass",
        "Yes"
      ],
      [
        "B",
        "Hold",
        "Hold",
        "Yes"
      ],
      [
        "C",
        "Pass",
        "Pass",
        "Yes"
      ],
      [
        "D",
        "Pass",
        "Hold",
        "No"
      ],
      [
        "E",
        "Hold",
        "Pass",
        "No"
      ]
    ]
  },
  "derivation": "Agreement count=3; total=5;3/5=60%. This is a descriptive exact-label match, not a chance-adjusted reliability coefficient.",
  "result": "Observed agreement is three of five, or 60%. Retain the two disputed cases and their criteria; the percentage alone cannot establish reviewer reliability or decide which verdict is correct.",
  "boundary": "The artifact examines reviewer-label agreement with inspectable case-level disagreement.",
  "limitations": "A five-case synthetic log does not validate a rubric or reviewer competence. Report actual disagreements and context.",
  "sources": [
    "original"
  ],
  "tasks": [
    [
      "Preserve independent verdicts",
      "Review coordinator",
      "Initial labels were recorded before a consensus discussion."
    ],
    [
      "Locate the disputed criteria",
      "Rubric owner",
      "D and E include the evidence and rule each reviewer used."
    ],
    [
      "Calibrate the next review",
      "Acceptance lead",
      "A clarified rubric is tested on fresh cases without rewriting the initial disagreement log."
    ]
  ],
  "faqs": [
    [
      "Does agreement prove both reviewers are right?",
      "No. Two people can share the same mistaken interpretation."
    ],
    [
      "Can I average pass and hold into acceptance?",
      "Not automatically. Resolve the governing criterion and required evidence."
    ]
  ],
  "method_references": [
    {
      "id": "original",
      "title": "Original worked-case definitions",
      "scope": "Definitions, policy choices, records and calculations are authored for this worksheet. No external standard, statistical validation or live product measurement is claimed."
    }
  ]
}