Evaluation Harnesses and Benchmark Contamination

This witness implements the same protocol-card habit with a tiny item-level exposure audit. The full room script also emits a JSON protocol fingerprint and checks the expected output.

import re

protocol = {
    "task_set_version": "toy_eval_v0",
    "dataset_split_version": "bench_split_demo_v0",
    "prompt_template": "answer_only_v1",
    "decoding_policy": "fixed_responses_no_sampling",
    "metric": "exact_match_lower_ascii",
    "evaluator_code_version": "tiny_eval_harness.py@2026-06-30",
    "model_run_config": "toy_fixed_response_table_v0",
    "contamination_policy": "id, answer, and token-overlap audit",
}

history = [
    ("pretrain", "A copied card says alpha bridge points to the south gate."),
    ("pretrain", "A practice note says arithmetic examples can be solved by counting pairs."),
    ("tune", "For benchmark item q_color_code, the final answer is cyan."),
]
items = [
    ("q_math", "What is two plus two?", "4", "answer_only_v1", "4"),
    ("q_background_note", "The arithmetic practice note describes examples solved by what trick?", "abacus", "answer_only_v1", "abacus"),
    ("q_alpha_bridge", "Alpha bridge points to which gate?", "south gate", "answer_only_v1", "south gate"),
    ("q_color_code", "What color is q_color_code?", "cyan", "answer_only_v1", "cyan"),
    ("q_template_shift", "Return only the letter after A.", "B", "rationale_then_answer_v2", "B"),
]

def toks(text):
    return {t for t in re.findall(r"[a-z0-9]+", text.lower()) if len(t) > 1 or t.isdigit()}

def verdict(item_id, prompt, answer, template):
    item_tokens, answer_tokens = toks(item_id + " " + prompt + " " + answer), toks(answer)
    saw_pretrain, saw_tune_label = False, False
    for stage, doc in history:
        doc_tokens = toks(doc)
        overlap = len(item_tokens & doc_tokens) / max(1, len(item_tokens))
        answer_leak = bool(answer_tokens) and answer_tokens.issubset(doc_tokens)
        id_leak = item_id.lower() in doc.lower()
        saw_pretrain |= stage == "pretrain" and (answer_leak or overlap >= 0.25)
        saw_tune_label |= stage == "tune" and answer_leak and id_leak
    if saw_tune_label:
        return "invalid evidence"
    if saw_pretrain or template != protocol["prompt_template"]:
        return "scope-limited"
    return "clean comparable"

for row in items:
    print(row[0], row[4].lower() == row[2].lower(), verdict(*row[:4]))

The code is intentionally humble. It does not prove the absence of contamination. It shows the structure of the question: pin the protocol, audit exposure under a named policy, then scope the claim before interpreting the score.