mtfyun/ds4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #319 from rinaldofesta/fix/eval-grader-false-negatives

Browse Source 
ds4-eval: fix four answer-grader false negatives + golden self-tests
This commit is contained in:
Salvatore Sanfilippo authored and GitHub committed 2026-06-17 08:25:30 +02:00
2 parents cafc134f78 b2533d12b1
commit 80ebbc396a
1 file changed
+234 -6
Download Patch File Download Diff File Expand all files Collapse all files
ds4_eval.c
+234 -6
View File
@@ -2697,6 +2697,62 @@ static char *find_last_answer_marker(const char *visible) {
return last ? last : strcasestr_local(visible, "answer");
}
/* True when the in-range capital at `letter` is the object of an explicit
* rejection earlier on the same answer line -- "not B", "isn't B",
* "rules out C", "eliminate E" -- so it is a distractor the model is
* discarding, not its pick. This rewrites only the lexical "first valid
* letter wins" default for clear elimination phrasing; the cue set is kept
* small and high-precision on purpose, and there is no general
* selection-vs-rejection sentence parsing (issue #321). It never looks before
* `start` or across a newline, so "D, not B" still grades D: the pick is
* reached and accepted before the rejected distractor is ever inspected. */
static bool mc_letter_is_negated(const char *start, const char *letter) {
const char *p = letter;
/* Step back over the gap to the previous word: spaces and light separating
* punctuation only, and never across a line break. */
while (p > start) {
char c = p[-1];
if (c == '\n') return false;
if (c == ' ' || c == '\t' || c == ',' || c == ';') p--;
else break;
}
/* Read the immediately preceding word (letters/apostrophe), lowercased. */
const char *wend = p;
while (p > start && (isalpha((unsigned char)p[-1]) || p[-1] == '\'')) p--;
size_t wlen = (size_t)(wend - p);
if (wlen == 0 || wlen >= 16) return false;
char w[16];
for (size_t i = 0; i < wlen; i++) w[i] = (char)tolower((unsigned char)p[i]);
w[wlen] = '\0';
/* Contraction form: isn't / aren't / wasn't / doesn't / won't / can't. */
if (wlen >= 3 && strcmp(w + wlen - 3, "n't") == 0) return true;
static const char *cues[] = {
"not", "except", "excluding", "exclude", "excludes",
"eliminate", "eliminates", "eliminated",
"reject", "rejects", "rejected", "rejecting", NULL
};
for (int i = 0; cues[i]; i++) if (strcmp(w, cues[i]) == 0) return true;
/* Two-word cue: "rule out" / "rules out" / "ruled out". */
if (strcmp(w, "out") == 0) {
const char *q = p;
while (q > start && (q[-1] == ' ' || q[-1] == '\t')) q--;
const char *rend = q;
while (q > start && isalpha((unsigned char)q[-1])) q--;
size_t rl = (size_t)(rend - q);
if (rl && rl < 8) {
char r[8];
for (size_t i = 0; i < rl; i++) r[i] = (char)tolower((unsigned char)q[i]);
r[rl] = '\0';
if (!strcmp(r, "rule") || !strcmp(r, "rules") || !strcmp(r, "ruled"))
return true;
}
}
return false;
}
static char find_answer_letter(const char *generated, int nchoices) {
if (nchoices <= 0) return '?';
const char *visible = strstr(generated, "</think>");
@@ -2712,7 +2768,22 @@ static char find_answer_letter(const char *generated, int nchoices) {
if (c >= 'A' && c <= max_answer) {
char before = p == visible ? ' ' : p[-1];
char after = p[1];
if (is_letter_boundary(before, after)) return c;
if (!is_letter_boundary(before, after)) continue;
/* A standalone capital that begins a same-line English word
* ("A careful ...") or a contraction ("I'll ...") is prose,
* not the model's pick: the real letter comes later on the
* line. Skip it; the reverse scan below still recovers it if
* it was the only candidate (e.g. "Answer: A is correct"). */
if (after == '\'') continue;
if (after == ' ' || after == '\t') {
const char *w = p + 1;
while (*w == ' ' || *w == '\t') w++;
if (islower((unsigned char)*w)) continue;
}
/* A distractor explicitly rejected before the pick on the same
* line ("not B, ... D") must not win over the real choice. */
if (mc_letter_is_negated(answer, p)) continue;
return c;
}
}
}
@@ -2765,7 +2836,17 @@ static void find_integer_answer(const char *generated, char *dst, size_t dstlen)
if (answer) {
const char *end = answer + strlen(answer);
if (strlen(answer) > 160) end = answer + 160;
if (scan_first_integer(answer, end, dst, dstlen)) return;
/* Restrict to the final answer line: digits after it (continued
* reasoning, footnotes, years) must not override the answer. */
const char *nl = memchr(answer, '\n', (size_t)(end - answer));
if (nl) end = nl;
/* When the line shows arithmetic ("m+n = 256+37 = 293") the stated
* result is the right-hand side of the LAST '='. Otherwise the first
* integer on the line is the answer (keeps "Final answer: 082"). */
const char *eq = NULL;
for (const char *r = answer; r < end; r++) if (*r == '=') eq = r;
if (scan_first_integer(eq ? eq + 1 : answer, end, dst, dstlen)) return;
if (eq && scan_first_integer(answer, end, dst, dstlen)) return;
}
const char *last_start = NULL;
@@ -3132,6 +3213,32 @@ static char *trace_copy_model_output(const char *case_start, const char *case_en
return out;
}
typedef enum {
REGRADE_NOT_GRADED,
REGRADE_PASSED,
REGRADE_FAILED,
} regrade_outcome;
/* Decide how one trace case regrades. Only PASSED/FAILED traces were graded by
* a completed run; STOPPED/SKIPPED/SWITCHED/ERROR cases carry partial or no
* model output and must be reported as not-graded rather than counted, since
* grading them inflates the totals and raises spurious "changed" drift. An
* empty status keeps legacy traces (written before the status line) working. */
static regrade_outcome regrade_case_outcome(const eval_case *tc,
const char *traced_status,
const char *model_output,
char *got, size_t gotlen) {
bool gradeable = traced_status[0] == '\0' ||
!strcmp(traced_status, "PASSED") ||
!strcmp(traced_status, "FAILED");
if (!gradeable) {
if (gotlen) got[0] = '\0';
return REGRADE_NOT_GRADED;
}
find_case_answer(tc, model_output, got, gotlen);
return answer_matches(tc, got) ? REGRADE_PASSED : REGRADE_FAILED;
}
static int regrade_trace_file(const char *path) {
size_t len = 0;
char *text = read_text_file(path, &len);
@@ -3145,6 +3252,7 @@ static int regrade_trace_file(const char *path) {
int changed = 0;
int unknown = 0;
int parse_errors = 0;
int not_graded = 0;
while (true) {
const char *case_start = trace_find_next_case(start, end);
@@ -3186,8 +3294,14 @@ static int regrade_trace_file(const char *path) {
}
char got[EVAL_ANSWER_MAX];
find_case_answer(tc, model_output, got, sizeof(got));
bool ok = answer_matches(tc, got);
regrade_outcome outcome =
regrade_case_outcome(tc, traced_status, model_output, got, sizeof(got));
if (outcome == REGRADE_NOT_GRADED) {
not_graded++;
free(model_output);
continue;
}
bool ok = (outcome == REGRADE_PASSED);
if (ok) passed++;
else failed++;
@@ -3204,8 +3318,8 @@ static int regrade_trace_file(const char *path) {
free(model_output);
}
printf("ds4-eval: regraded %d cases from %s: passed=%d failed=%d changed=%d unknown=%d parse_errors=%d\n",
total, path, passed, failed, changed, unknown, parse_errors);
printf("ds4-eval: regraded %d cases from %s: passed=%d failed=%d changed=%d not_graded=%d unknown=%d parse_errors=%d\n",
total, path, passed, failed, changed, not_graded, unknown, parse_errors);
free(text);
return (unknown || parse_errors || total == 0) ? 1 : 0;
}
@@ -3273,10 +3387,37 @@ static int trace_copy_self_test_case(void) {
return 0;
}
static int regrade_status_self_test_case(void) {
int failed = 0;
const eval_case integer = {.source = "AIME2025", .answer = "82"};
char got[EVAL_ANSWER_MAX];
if (regrade_case_outcome(&integer, "PASSED", "</think>Answer: 82",
got, sizeof(got)) != REGRADE_PASSED) {
fprintf(stderr, "ds4-eval: regrade self-test failed: PASSED trace not regraded\n");
failed++;
}
/* An interrupted run whose partial output happens to look correct must not
* be counted or flagged as drift. */
if (regrade_case_outcome(&integer, "STOPPED", "</think>Answer: 82",
got, sizeof(got)) != REGRADE_NOT_GRADED) {
fprintf(stderr, "ds4-eval: regrade self-test failed: STOPPED trace was graded\n");
failed++;
}
/* Legacy traces without a status line stay gradeable. */
if (regrade_case_outcome(&integer, "", "</think>Answer: 82",
got, sizeof(got)) != REGRADE_PASSED) {
fprintf(stderr, "ds4-eval: regrade self-test failed: legacy empty status not graded\n");
failed++;
}
return failed;
}
static int run_extractor_self_tests(void) {
int failed = 0;
failed += trace_copy_self_test_case();
failed += regrade_status_self_test_case();
const eval_case mc = {
.source = "SuperGPQA",
@@ -3337,6 +3478,93 @@ static int run_extractor_self_tests(void) {
"Answer: 10",
"10");
/* --- Regression cases for answer-extractor false negatives. These guard
* against the grader under-reporting model accuracy on well-formed
* final-answer lines. --- */
/* Multiple choice: a standalone in-range capital that merely begins an
* English word ("I think", "A careful") or a contraction ("I'll") must
* not shadow the choice the model actually states later on the line.
* Only reachable when the stray letter is itself a valid option, i.e.
* 10-choice (A-J) cases, of which the embedded set has 24. */
const eval_case mc_c = {
.source = "SuperGPQA",
.choice = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"},
.answer = "C",
};
failed += extractor_self_test_case(
"MC: leading pronoun 'I' does not shadow the chosen letter",
&mc_c, "</think>Answer: I think it is C", "C");
failed += extractor_self_test_case(
"MC: contraction I'll does not shadow the chosen letter",
&mc_c, "</think>Answer: I'll go with C.", "C");
failed += extractor_self_test_case(
"MC: leading article 'A' does not shadow the chosen letter",
&mc_c, "</think>Answer: A careful reading shows C.", "C");
const eval_case mc_i = {
.source = "SuperGPQA",
.choice = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"},
.answer = "I",
};
failed += extractor_self_test_case(
"MC: a genuine standalone 'I' answer is still picked",
&mc_i, "</think>Answer: I.", "I");
const eval_case mc4_d = {
.source = "SuperGPQA",
.choice = {"A", "B", "C", "D"},
.answer = "D",
};
failed += extractor_self_test_case(
"MC: out-of-range pronoun is harmless on 4-choice cases",
&mc4_d, "</think>Answer: I think it is D", "D");
/* A distractor the model explicitly rejects *before* stating its pick on
* the same line must not shadow the real choice ("not B, ... D" /
* "rules out C, leaving D"). The bare first-valid-letter rule grabbed the
* rejected letter; a small, high-precision negation-cue skip fixes it.
* "D, not B" is the guard against the naive "take the last letter" fix:
* the pick is reached and accepted before the rejected distractor. #321 */
const eval_case mc_d = {
.source = "SuperGPQA",
.choice = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"},
.answer = "D",
};
failed += extractor_self_test_case(
"MC: 'not B' distractor before the pick does not shadow it",
&mc_d, "</think>Answer: It is not B, the answer is D", "D");
failed += extractor_self_test_case(
"MC: 'rules out C, leaving D' grades the surviving choice",
&mc_d, "</think>Answer: rules out C, leaving D", "D");
failed += extractor_self_test_case(
"MC: contraction negation (isn't B) before the pick is skipped",
&mc_d, "</think>Answer: It isn't B, so D", "D");
failed += extractor_self_test_case(
"MC: a leading pick followed by 'not B' is still graded as the pick",
&mc_d, "</think>Answer: D, not B", "D");
/* Integer: when the answer line shows the arithmetic, the graded value
* must be the stated result (right of the last '='), not the first
* summand/factor; digits on later lines must not leak in either. Many
* embedded AIME2025 cases ask for a derived sum (m+n, a+b+c, ...). */
const eval_case int_293 = {.source = "AIME2025", .answer = "293"};
failed += extractor_self_test_case(
"integer: sum line grades the total, not the first summand",
&int_293, "</think>Answer: m+n = 256+37 = 293", "293");
const eval_case int_62 = {.source = "AIME2025", .answer = "62"};
failed += extractor_self_test_case(
"integer: three-term sum line grades the total",
&int_62, "</think>Answer: a+b+c = 12+25+25 = 62", "62");
const eval_case int_81 = {.source = "AIME2025", .answer = "81"};
failed += extractor_self_test_case(
"integer: product-sum line grades the result, not the first factor",
&int_81, "</think>Answer: 2*7 + 3*6 + 5*4 + 7*5 = 81", "81");
const eval_case int_82 = {.source = "AIME2025", .answer = "82"};
failed += extractor_self_test_case(
"integer: digits on a later line do not override the answer line",
&int_82, "</think>Answer: 82\nThe value 2025 is just the year.", "82");
if (failed) return 1;
printf("ds4-eval: answer extractor self-tests passed\n");
return 0;