Process Reward Models: Step-Level Verifiers for Reasoning

This witness uses four fixed traces for the equation $2(x+3)=14$. The outcome verifier prefers a trace with the correct final answer but invalid reasoning. The accurate process verifier prefers the clean trace. If we inject a false-positive process error, the process verifier can be fooled too.

import math
import numpy as np

def logit(p):
    p = min(max(p, 1e-6), 1 - 1e-6)
    return math.log(p / (1 - p))

def policy(scores, ref, beta=0.7):
    logits = np.log(ref) + np.asarray(scores) / beta
    logits = logits - logits.max()
    weights = np.exp(logits)
    return weights / weights.sum()

traces = ["clean", "lucky_final", "slip", "wrong"]
ref = np.array([0.30, 0.22, 0.28, 0.20])
outcome_p = np.array([0.86, 0.92, 0.16, 0.12])
step_p = [
    [0.93, 0.91],
    [0.16, 0.24],
    [0.93, 0.22],
    [0.20, 0.35],
]

def process_scores(step_probs):
    return [sum(logit(p) for p in probs) for probs in step_probs]

outcome_winner = traces[int(policy([logit(p) for p in outcome_p], ref).argmax())]
process_winner = traces[int(policy(process_scores(step_p), ref).argmax())]

hacked_steps = [list(probs) for probs in step_p]
hacked_steps[1] = [0.98, 0.97]
hacked_winner = traces[int(policy(process_scores(hacked_steps), ref).argmax())]

print(outcome_winner, process_winner, hacked_winner)
assert outcome_winner == "lucky_final"
assert process_winner == "clean"
assert hacked_winner == "lucky_final"