ew23_classify.py

#!/usr/bin/env python3
"""For the 10 ew23 persistent-fail instances, classify localization vs logic:
- did the agent produce a non-empty patch? how many files / lines?
- which test FILES failed, and did the agent's patch touch any of those source areas?
Run on the FIX-arm copy of each instance (all arms fail these anyway)."""
import json, os, glob, re, hashlib

BASE = "/home/ubuntu/projects/mem-comp-26/harness"
FIX = BASE + "/results/kimi_ew23_memvaware_et2_rep1_20260630"

PERSIST_SIGS = {
    "b4a909f3f8", "82c804076e", "a4e2410d18", "0603da0851", "54c61efd75",
    "0d73017d71", "71f0f0d96e", "0861e193b7", "ea1276c6e0", "0831d92221",
}


def sig(inst_dir):
    with open(os.path.join(inst_dir, "instance.json"), encoding="utf-8", errors="replace") as f:
        ps = (json.load(f).get("problem_statement") or "").strip()
    return hashlib.sha1(ps[:200].encode()).hexdigest()[:10], ps[:50].replace("\n", " ")


def patch_files(inst_dir):
    pd = os.path.join(inst_dir, "patch.diff")
    if not os.path.exists(pd):
        return [], 0, 0
    files, added = [], 0
    with open(pd, encoding="utf-8", errors="replace") as f:
        for ln in f:
            m = re.match(r"^\+\+\+ b/(.+)$", ln)
            if m:
                files.append(m.group(1))
            elif ln.startswith("+") and not ln.startswith("+++"):
                added += 1
    size = os.path.getsize(pd)
    return files, added, size


def verdict(inst_dir):
    vv = os.path.join(inst_dir, "_harness", "verdict_val.json")
    if not os.path.exists(vv):
        return None
    with open(vv, encoding="utf-8", errors="replace") as f:
        return json.load(f)


def basename_noext(path):
    b = os.path.basename(path)
    return re.sub(r"\.(tsx?|jsx?)$", "", b)


print(f"{'sig':11s} {'files':>5s} {'+ln':>4s} {'patchB':>7s}  loc?  title")
for inst_dir in sorted(glob.glob(os.path.join(FIX, "p00i*"))):
    try:
        k, title = sig(inst_dir)
    except Exception:
        continue
    if k not in PERSIST_SIGS:
        continue
    files, added, size = patch_files(inst_dir)
    v = verdict(inst_dir) or {}
    passed = set(v.get("passed_tests") or [])
    all_tests = v.get("all_tests") or []
    # the required tests that did NOT pass = the failing required set
    failed = [t for t in all_tests if t not in passed]
    # implied source stems the agent edited
    edited_stems = {basename_noext(f) for f in files}
    # implied test target stems (strip -test suffix)
    def test_stem(t):
        first = t.split("|")[0].strip()
        return re.sub(r"-test$", "", basename_noext(first))
    fail_stems = {test_stem(t) for t in failed}
    overlap = edited_stems & fail_stems if fail_stems else set()
    if size == 0 or not files:
        loc = "EMPTY"
    elif fail_stems and overlap:
        loc = "LOGIC"      # edited a file whose test still fails
    elif fail_stems and not overlap:
        loc = "MISS?"      # failing tests are for files the agent didn't touch
    else:
        loc = "edited"     # non-empty patch, no failed-test list to compare
    print(f"{k}  {len(files):5d} {added:4d} {size:7d}  {loc:5s} {title}")
    if files:
        print(f"             touched: {', '.join(files[:6])}{' ...' if len(files)>6 else ''}")
    if fail_stems:
        print(f"             failtests stems: {', '.join(sorted(fail_stems)[:8])}")