Match random fallback labeling protocol

Author: pengyangda7

RUN_STEPS.md

@@ -124,7 +124,10 @@ deterministic responses in the output directory's `cache/` subdirectory.
 The runtime refactor preserves the standard single-item and batched prompt
 templates used by the previous release; snapshot tests cover all four paths.
 For stochastic runs, set `VERIBENCH_JUDGE_TEMPERATURE=1` and
-`VERIBENCH_DISABLE_JUDGE_CACHE=1`.
+`VERIBENCH_DISABLE_JUDGE_CACHE=1`. Judgments that remain unparsable after the
+domain parser and correction rules receive uniformly random positive or
+negative labels and remain in the denominator. Set `VERIBENCH_FALLBACK_SEED` to
+make fallback labels deterministic.
 
 DeepResearch:
 

code/run_judges/README.md

@@ -16,8 +16,10 @@ The two runner files are the execution entrypoints. The `agenticcoding/`
 directory contains only the result-comparison utility.
 The standard DeepResearch and AgenticCoding prompt templates are unchanged
 from the previous release and are protected by prompt snapshot tests.
-As in the released experimental runner, missing or unparsable judgments are
-retained in the output and assigned the negative label.
+As in the released experimental runner, judgments that remain missing or
+unparsable after parsing and correction are retained in the output and assigned
+uniformly random positive or negative labels. Set `VERIBENCH_FALLBACK_SEED` to
+make these fallback labels deterministic for a rerun.
 
 Convenience wrappers:
 
@@ -55,6 +57,7 @@ For stochastic self-voting runs, you may also set:
 ```bash
 export VERIBENCH_JUDGE_TEMPERATURE=1
 export VERIBENCH_DISABLE_JUDGE_CACHE=1
+export VERIBENCH_FALLBACK_SEED=1234
 ```
 
 ---

code/run_judges/agenticcoding_runner.py

@@ -16,7 +16,7 @@
     sys.path.insert(0, str(ROOT))
 
 from prompting import agenticcoding_prompt
-from runtime import APIModel
+from runtime import APIModel, fallback_binary_label
 
 
 def read_jsonl(path: str | Path) -> list[dict[str, Any]]:
@@ -111,12 +111,24 @@ def normalize_result(value: Any) -> str:
     raise ValueError(f"invalid result value: {value!r}")
 
 
-def failed_check(check_id: str, reason: str) -> dict[str, str]:
-    return {"check_id": check_id, "result": "fail", "reasoning": reason}
+def fallback_check(check_id: str, reason: str, key: str = "") -> dict[str, str]:
+    result = "success" if fallback_binary_label(key or check_id) else "fail"
+    return {
+        "check_id": check_id,
+        "result": result,
+        "reasoning": f"{reason}; assigned uniformly random {result} label",
+    }
 
 
-def parse_checks(raw: str | None, expected: list[dict[str, Any]]) -> list[dict[str, str]]:
+def parse_checks(
+    raw: str | None,
+    expected: list[dict[str, Any]],
+    fallback_prefix: str = "",
+) -> list[dict[str, str]]:
     expected_ids = [str(item["check_id"]) for item in expected]
+    def fallback(check_id: str, reason: str) -> dict[str, str]:
+        key = f"{fallback_prefix}::{check_id}" if fallback_prefix else check_id
+        return fallback_check(check_id, reason, key)
     try:
         parsed = extract_json(raw or "")
         values = parsed if isinstance(parsed, list) else [parsed]
@@ -133,16 +145,16 @@ def parse_checks(raw: str | None, expected: list[dict[str, Any]]) -> list[dict[s
         output = []
         for check_id in expected_ids:
             if check_id in duplicate:
-                output.append(failed_check(check_id, "duplicate check_id in judge output"))
+                output.append(fallback(check_id, "duplicate check_id in judge output"))
                 continue
             value = by_id.get(check_id)
             if value is None:
-                output.append(failed_check(check_id, "judge output omitted this check_id"))
+                output.append(fallback(check_id, "judge output omitted this check_id"))
                 continue
             try:
                 result = normalize_result(value.get("result"))
             except ValueError as exc:
-                output.append(failed_check(check_id, str(exc)))
+                output.append(fallback(check_id, str(exc)))
                 continue
             output.append({
                 "check_id": check_id,
@@ -151,7 +163,7 @@ def parse_checks(raw: str | None, expected: list[dict[str, Any]]) -> list[dict[s
             })
         return output
     except Exception as exc:
-        return [failed_check(check_id, f"judge output could not be parsed: {exc}") for check_id in expected_ids]
+        return [fallback(check_id, f"judge output could not be parsed: {exc}") for check_id in expected_ids]
 
 
 def evaluate_case(case: dict[str, Any], trajectory: dict[str, Any], api: APIModel, batch_size: int, style: str, max_chars: int) -> dict[str, Any]:
@@ -165,9 +177,16 @@ def evaluate_case(case: dict[str, Any], trajectory: dict[str, Any], api: APIMode
             raw = api.generate_chat([{"role": "user", "content": prompt}], max_tokens=1024, temperature=0.0)
         except Exception as exc:
             raw = None
-            parsed = [failed_check(str(item["check_id"]), f"judge request failed: {exc}") for item in batch]
+            parsed = [
+                fallback_check(
+                    str(item["check_id"]),
+                    f"judge request failed: {exc}",
+                    f"{case['instance_id']}::{item['check_id']}",
+                )
+                for item in batch
+            ]
         else:
-            parsed = parse_checks(raw, batch)
+            parsed = parse_checks(raw, batch, fallback_prefix=str(case["instance_id"]))
         evaluated.extend(parsed)
 
     by_category: dict[str, dict[str, Any]] = {}

code/run_judges/deepresearch_runner.py

@@ -15,7 +15,7 @@
     sys.path.insert(0, str(ROOT))
 
 from prompting import deepresearch_batch_prompt, deepresearch_single_prompt
-from runtime import APIModel
+from runtime import APIModel, fallback_binary_label
 
 
 def read_json(path: str | Path) -> Any:
@@ -115,10 +115,18 @@ def evaluate_item(
             index = start + offset
             candidate = by_index.get(index)
             try:
-                covered = normalize_answer(candidate.get("covered")) if candidate else False
-            except (AttributeError, ValueError):
-                covered = False
-            justification = str(candidate.get("justification", "")) if candidate else parse_error or "missing point_index"
+                covered = normalize_answer(candidate.get("covered")) if candidate else None
+            except (AttributeError, ValueError) as exc:
+                covered = None
+                parse_error = parse_error or f"invalid coverage value: {exc}"
+            if covered is None:
+                covered = fallback_binary_label(f"{item['id']}::{index}")
+                justification = (
+                    f"{parse_error or 'missing point_index'}; assigned uniformly random "
+                    f"{'positive' if covered else 'negative'} label"
+                )
+            else:
+                justification = str(candidate.get("justification", ""))
             weight = point.get("weight", 1)
             outputs.append({"point": point["point"], "weight": weight, "covered": covered, "justification": justification})
     total_weight = sum(point.get("weight", 1) for point in points)

code/run_judges/runtime.py

@@ -10,6 +10,7 @@
 import hashlib
 import json
 import os
+import secrets
 import threading
 import time
 from pathlib import Path
@@ -48,6 +49,15 @@ def _env_float(name: str, default: float) -> float:
     return float(raw)
 
 
+def fallback_binary_label(key: str) -> bool:
+    """Return the protocol's random binary label for an unparsable judgment."""
+    seed = os.getenv("VERIBENCH_FALLBACK_SEED", "").strip()
+    if seed:
+        digest = hashlib.sha256(f"{seed}\0{key}".encode("utf-8")).digest()
+        return bool(digest[0] & 1)
+    return bool(secrets.randbits(1))
+
+
 class JsonCache:
     """Process-safe-enough JSON cache for deterministic judge calls."""
 

tests/test_judge_runners.py

@@ -2,13 +2,16 @@
 import json
 import sys
 import unittest
+from unittest.mock import patch
 from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "code" / "run_judges"))
 sys.path.insert(0, str(ROOT / "code" / "main_leaderboard"))
 
+import agenticcoding_runner
+import deepresearch_runner
 from agenticcoding_runner import evaluate_case, parse_checks
 from compute_main_leaderboard import iter_ac_checks
 from deepresearch_runner import evaluate_item
@@ -50,6 +53,16 @@ def save_cache(self):
         pass
 
 
+class UnparseableJudge:
+    model_name = "unparseable-judge"
+
+    def generate_chat(self, messages, **kwargs):
+        return "not a valid judgment"
+
+    def save_cache(self):
+        pass
+
+
 class RunnerTests(unittest.TestCase):
     def test_run_steps_packaged_result_paths_exist(self):
         run_steps = (ROOT / "RUN_STEPS.md").read_text(encoding="utf-8")
@@ -126,9 +139,32 @@ def test_agenticcoding_output_keeps_category_schema(self):
         self.assertEqual(result["Rules"]["checks"][0]["check_id"], "r1")
         self.assertEqual(len(iter_ac_checks({"results": [result]})), 1)
 
-    def test_agenticcoding_missing_check_is_fail(self):
+    def test_unparsable_labels_use_random_fallback(self):
+        item = {"id": "DRB2_1", "question": "Q", "response": "A"}
+        rubrics = {"rubric": [{"point": "first"}, {"point": "second"}]}
+        with patch.object(deepresearch_runner, "fallback_binary_label", return_value=True):
+            result = evaluate_item(
+                item,
+                rubrics,
+                UnparseableJudge(),
+                batch_size=2,
+                style="standard",
+                model_name="fixture",
+            )
+        coverage = result["result"]["coverage_results"]
+        self.assertEqual([row["covered"] for row in coverage], [True, True])
+        self.assertTrue(all("uniformly random positive label" in row["justification"] for row in coverage))
+
+        expected = [{"check_id": "r1"}, {"check_id": "r2"}]
+        with patch.object(agenticcoding_runner, "fallback_binary_label", return_value=True):
+            parsed = parse_checks("not valid JSON", expected, fallback_prefix="case-1")
+        self.assertEqual([item["result"] for item in parsed], ["success", "success"])
+        self.assertTrue(all("uniformly random success label" in item["reasoning"] for item in parsed))
+
+    def test_agenticcoding_missing_check_can_be_negative_fallback(self):
         expected = [{"check_id": "r1"}, {"check_id": "r2"}]
-        parsed = parse_checks('[{"check_id": "r1", "result": "success"}]', expected)
+        with patch.object(agenticcoding_runner, "fallback_binary_label", return_value=False):
+            parsed = parse_checks('[{"check_id": "r1", "result": "success"}]', expected)
         self.assertEqual([item["result"] for item in parsed], ["success", "fail"])
         self.assertEqual(parsed[1]["check_id"], "r2")