{ "description": "Per-run facts from the luna / haiku / mercury treeward eval.", "models": { "luna": { "label": "GPT-6 Luna", "id": "openrouter/openai/gpt-6-luna" }, "haiku": { "label": "Claude Haiku 5.5", "id": "openrouter/anthropic/claude-haiku-5.5" }, "mercury": { "label": "Mercury 3", "id": "openrouter/inception/mercury-3" } }, "cases": [ { "id": "c01-loglevel", "title": "--log-level flag, CLI beats RUST_LOG", "commit": "54bf091951ea97aec5a9e268e175173c4b785c42", "parent": "0949eaa21236a9743528a8c7116481cdaf00eb6c", "pr_title": "Add --log-level; make it and -v override RUST_LOG env var (#89)", "candidateLetters": { "A": "haiku", "B": "mercury", "C": "luna" } }, { "id": "c02-fp-repeated-edits", "title": "fingerprint misses repeated edits", "commit": "4320ef611378f3718214674f7602128467d4851f", "parent": "c8b818956e799527a0905ac2eefbe990a5ad35b1", "pr_title": "bugfix: fingerprint validation misses repeated edits (#83)", "candidateLetters": { "A": "mercury", "B": "haiku", "C": "luna" } }, { "id": "c03-partial-writes", "title": "partial init/update leaves misleading wards", "commit": "695ce8d98efddbccae0ed8a1260398eafa63896a", "parent": "f5f2c3d58ae1f64408ec9531b7bbbfa1e16fce70", "pr_title": "fix: write ward files deepest-first and report partial updates (#147)", "candidateLetters": { "A": "mercury", "B": "luna", "C": "haiku" } }, { "id": "c04-list-read-race", "title": "file changes between listing and checksum", "commit": "7ff26be585866a7ed5b75c7f89f220e50085d8d7", "parent": "ff59c34e6a6c81a87f666c9759f0f4a4587351b9", "pr_title": "fix: abort when a file changes between listing and checksumming (#149)", "candidateLetters": { "A": "mercury", "B": "haiku", "C": "luna" } }, { "id": "c05-dir-mtime", "title": "pre-1970 directory mtimes rejected", "commit": "f5f2c3d58ae1f64408ec9531b7bbbfa1e16fce70", "parent": "95e43407f9515bede59f45bcb7f21f1e0874bd8b", "pr_title": "fix: stop rejecting pre-epoch and far-future directory mtimes (#146)", "candidateLetters": { "A": "mercury", "B": "luna", "C": "haiku" } }, { "id": "c06-stderr-escape", "title": "control characters leak to stderr", "commit": "58651e173275f8b3da86beef7670f10daa03ead6", "parent": "37dfeb4990857f591aff39d8f0bedca0cbdd971b", "pr_title": "fix: escape control characters in stderr diagnostics (#125)", "candidateLetters": { "A": "mercury", "B": "luna", "C": "haiku" } }, { "id": "c07-symlink-utf8", "title": "non-UTF-8 symlink targets", "commit": "c130ad22f421db1501a5630bcb2b86f2587cf314", "parent": "bf03f80f3b3ae80093bec5ce00b2151376e7419d", "pr_title": "fix: reject non-UTF-8 symlink targets when listing a directory (#144)", "candidateLetters": { "A": "mercury", "B": "luna", "C": "haiku" } }, { "id": "c08-root-ward-type", "title": "root .treeward that is not a regular file", "commit": "ff59c34e6a6c81a87f666c9759f0f4a4587351b9", "parent": "695ce8d98efddbccae0ed8a1260398eafa63896a", "pr_title": "fix: refuse a root .treeward that is not a regular file (#148)", "candidateLetters": { "A": "mercury", "B": "luna", "C": "haiku" } }, { "id": "c09-sha256-validate", "title": "malformed sha256 in ward files", "commit": "26f113bf430fa97f0b38d171a9c976ca80ba12c1", "parent": "6ddf457eb7566f7f5dd87582c8d9fe3a4a5c4870", "pr_title": "fix: reject ward files with malformed sha256 fields at load time (#116)", "candidateLetters": { "A": "haiku", "B": "luna", "C": "mercury" } }, { "id": "c10-status-diff", "title": "status --diff (large feature)", "commit": "c8b818956e799527a0905ac2eefbe990a5ad35b1", "parent": "9b21ea87f0ef647585d9bf4dd745298dbdcca289", "pr_title": "status --diff (#82)", "candidateLetters": { "A": "haiku", "B": "mercury", "C": "luna" } } ], "runs": [ { "case": "c01-loglevel", "model": "haiku", "acceptanceTests": { "hardPass": 13, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Conflict between -v and --log-level not detected when they are split across the subcommand", "verified": true, "verifiedHow": "Built a copy and ran it; exit 0 with DEBUG output. Same-side placements exit 2. The reference uses the same conflicts_with + global mechanism, so this is shared." }, { "severity": "minor", "summary": "Docstring and summary claim clap guarantees the two flags never both appear", "verified": true, "verifiedHow": "Same repro as above; the comment's invariant does not hold." } ], "ownTestSuite": { "passed": 176, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 9, "diff": { "files": 5, "added": 220, "deleted": 11 }, "usage": { "input": 50, "output": 47761, "cacheRead": 1119809, "cacheWrite": 67134, "cost": 0.04347534, "calls": 24 }, "toolCalls": 36, "wallSeconds": 340.719, "reviewLetter": "A" }, { "case": "c01-loglevel", "model": "mercury", "acceptanceTests": { "hardPass": 13, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Conflict between -v and --log-level not detected when they are split across the subcommand", "verified": true, "verifiedHow": "Built a copy and ran it; exit 0 with DEBUG output. Shared with A, C and the reference mechanism." } ], "ownTestSuite": { "passed": 172, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 5, "diff": { "files": 5, "added": 147, "deleted": 12 }, "usage": { "input": 155897, "output": 20277, "cacheRead": 2024655, "cacheWrite": 0, "cost": 0.042933200000000026, "calls": 54 }, "toolCalls": 70, "wallSeconds": 145.948, "reviewLetter": "B" }, { "case": "c01-loglevel", "model": "luna", "acceptanceTests": { "hardPass": 13, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Conflict between -v and --log-level not detected when they are split across the subcommand", "verified": true, "verifiedHow": "Built a copy and ran it; exit 0 with DEBUG output. The extra conflicts_with on -v does not help. Shared with A, B and the reference mechanism." } ], "ownTestSuite": { "passed": 174, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 7, "diff": { "files": 4, "added": 201, "deleted": 13 }, "usage": { "input": 78, "output": 20659, "cacheRead": 945716, "cacheWrite": 57499, "cost": 0.026981835, "calls": 26 }, "toolCalls": 57, "wallSeconds": 338.804, "reviewLetter": "C" }, { "case": "c02-fp-repeated-edits", "model": "mercury", "acceptanceTests": { "hardPass": 13, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Default `status` now reads/checksums every added and M? file, so it is no longer metadata-only and fails on unreadable changed files", "verified": true, "verifiedHow": "Built copy, ran scenario S1/S2 in scratch dir" }, { "severity": "minor", "summary": "User-facing docs still promise a metadata-only default status after the change made it read content", "verified": true, "verifiedHow": "grep of the finished tree for 'metadata-only'" } ], "ownTestSuite": { "passed": 173, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": true, "testFunctionsAdded": 14, "diff": { "files": 7, "added": 904, "deleted": 121 }, "usage": { "input": 210222, "output": 89029, "cacheRead": 13384243, "cacheWrite": 0, "cost": 0.18602477999999995, "calls": 103 }, "toolCalls": 134, "wallSeconds": 406.235, "reviewLetter": "A" }, { "case": "c02-fp-repeated-edits", "model": "haiku", "acceptanceTests": { "hardPass": 13, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Removed entries hash ObservedEntry::Absent only, so the prior ward state is not bound", "verified": true, "verifiedHow": "Scenario S4 on built copy" }, { "severity": "minor", "summary": "README edit is not dprint-formatted (CI requires dprint fmt)", "verified": true, "verifiedHow": "Ran dprint check on the copy" } ], "ownTestSuite": { "passed": 163, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [ "README.md" ], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 4, "diff": { "files": 4, "added": 363, "deleted": 37 }, "usage": { "input": 166, "output": 362061, "cacheRead": 4225054, "cacheWrite": 379746, "cost": 0.2707658900000001, "calls": 80 }, "toolCalls": 93, "wallSeconds": 1962.472, "reviewLetter": "B" }, { "case": "c02-fp-repeated-edits", "model": "luna", "acceptanceTests": { "hardPass": 13, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Default `status` now checksums added and M? files; unreadable changed files make `status` fail", "verified": true, "verifiedHow": "Scenario S1/S2 on built copy; the cost is documented in help and README" }, { "severity": "minor", "summary": "README rewrite dropped the 'two purposes' (TOCTOU / idempotency) list from 'Fingerprinting in depth'", "verified": true, "verifiedHow": "Read the diff" } ], "ownTestSuite": { "passed": 166, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 7, "diff": { "files": 5, "added": 593, "deleted": 97 }, "usage": { "input": 252, "output": 68340, "cacheRead": 9241679, "cacheWrite": 176699, "cost": 0.14869936500000003, "calls": 84 }, "toolCalls": 137, "wallSeconds": 1124.508, "reviewLetter": "C" }, { "case": "c03-partial-writes", "model": "mercury", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "critical", "summary": "Rollback deletes a newly created child ward whose pre-existing parent ward was already rewritten to list it", "verified": true, "verifiedHow": "Built a copy of the tree and ran both scenarios; p/.treeward contains [entries.n] type=dir while p/n/.treeward is absent; status before showed 'A p/n', after only 'A p/n/f.txt'." }, { "severity": "minor", "summary": "-v 'Updated N ward files' listing order changed (sorted by .treeward path, not directory)", "verified": true, "verifiedHow": "Ran -v init and -v init --dry-run on such a tree." } ], "ownTestSuite": { "passed": 225, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 3, "diff": { "files": 3, "added": 286, "deleted": 22 }, "usage": { "input": 228909, "output": 55425, "cacheRead": 6033019, "cacheWrite": 0, "cost": 0.10261984000000003, "calls": 85 }, "toolCalls": 97, "wallSeconds": 279.537, "reviewLetter": "A" }, { "case": "c03-partial-writes", "model": "luna", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [], "ownTestSuite": { "passed": 223, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": true, "claudeMdEdited": false, "testFunctionsAdded": 1, "diff": { "files": 7, "added": 277, "deleted": 65 }, "usage": { "input": 207, "output": 55226, "cacheRead": 6511543, "cacheWrite": 146276, "cost": 0.11103363000000002, "calls": 69 }, "toolCalls": 110, "wallSeconds": 981.2, "reviewLetter": "B" }, { "case": "c03-partial-writes", "model": "haiku", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "major", "summary": "Staging keeps one open NamedTempFile per changed ward, so large runs exhaust file descriptors", "verified": true, "verifiedHow": "prlimit --nofile=1024 treeward -C init on 1100 dirs (also 400 dirs at nofile=256)." }, { "severity": "minor", "summary": "-v listing of a successful run is now in deepest-first commit order and disagrees with --dry-run", "verified": true, "verifiedHow": "Ran -v init and -v init --dry-run." }, { "severity": "minor", "summary": "Recovery advice names `treeward update` even when init failed", "verified": true, "verifiedHow": "Ran init with sub chmod 555, then update." } ], "ownTestSuite": { "passed": 223, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 1, "diff": { "files": 3, "added": 261, "deleted": 33 }, "usage": { "input": 164, "output": 163154, "cacheRead": 4485819, "cacheWrite": 274096, "cost": 0.1607135900000001, "calls": 79 }, "toolCalls": 105, "wallSeconds": 1022.724, "reviewLetter": "C" }, { "case": "c04-list-read-race", "model": "mercury", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "checksum_file docstring truncated mid-sentence", "verified": true, "verifiedHow": "Read src/checksum.rs lines 54-60 in the candidate tree" }, { "severity": "minor", "summary": "Production doc links to a #[cfg(test)]-only item", "verified": false, "verifiedHow": "Reasoning from the diff; cargo doc not run" }, { "severity": "minor", "summary": "Race integration tests are sleep-timed and probabilistic", "verified": false, "verifiedHow": "Ran both tests twice in a copy: passed in 7-8 s each run; flakiness inferred from design" } ], "ownTestSuite": { "passed": 234, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 6, "diff": { "files": 9, "added": 278, "deleted": 9 }, "usage": { "input": 152359, "output": 117265, "cacheRead": 8030903, "cacheWrite": 0, "cost": 0.13658768, "calls": 93 }, "toolCalls": 108, "wallSeconds": 789.242, "reviewLetter": "A" }, { "case": "c04-list-read-race", "model": "haiku", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [], "ownTestSuite": { "passed": 230, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 2, "diff": { "files": 9, "added": 295, "deleted": 54 }, "usage": { "input": 102, "output": 118806, "cacheRead": 2461296, "cacheWrite": 213362, "cost": 0.11069641000000004, "calls": 47 }, "toolCalls": 72, "wallSeconds": 883.188, "reviewLetter": "B" }, { "case": "c04-list-read-race", "model": "luna", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Detailed Behavior/Errors docs for the checksum function deleted, not moved", "verified": true, "verifiedHow": "Read src/checksum.rs lines 38-75 in the candidate tree" } ], "ownTestSuite": { "passed": 235, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 7, "diff": { "files": 5, "added": 302, "deleted": 27 }, "usage": { "input": 117, "output": 37205, "cacheRead": 3244681, "cacheWrite": 125378, "cost": 0.06673326, "calls": 39 }, "toolCalls": 78, "wallSeconds": 612.587, "reviewLetter": "C" }, { "case": "c05-dir-mtime", "model": "mercury", "acceptanceTests": { "hardPass": 6, "hardFail": 0, "softPass": 0, "softFail": 1 }, "defects": [ { "severity": "minor", "summary": "Out-of-range dir mtimes collapse to a single None value in the fingerprint, so binding is inconsistent with in-range dirs", "verified": true, "verifiedHow": "probe2.ts against A binary: update --dry-run --fingerprint after pre-epoch->pre-epoch change exit 0; B and C exit 255" } ], "ownTestSuite": { "passed": 228, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 9, "diff": { "files": 8, "added": 279, "deleted": 17 }, "usage": { "input": 109623, "output": 34942, "cacheRead": 3181545, "cacheWrite": 0, "cost": 0.05500745, "calls": 47 }, "toolCalls": 84, "wallSeconds": 154.687, "reviewLetter": "A" }, { "case": "c05-dir-mtime", "model": "luna", "acceptanceTests": { "hardPass": 6, "hardFail": 0, "softPass": 0, "softFail": 1 }, "defects": [], "ownTestSuite": { "passed": 221, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": true, "claudeMdEdited": false, "testFunctionsAdded": 2, "diff": { "files": 5, "added": 141, "deleted": 15 }, "usage": { "input": 111, "output": 26665, "cacheRead": 2399973, "cacheWrite": 91390, "cost": 0.048767079999999983, "calls": 37 }, "toolCalls": 62, "wallSeconds": 677.229, "reviewLetter": "B" }, { "case": "c05-dir-mtime", "model": "haiku", "acceptanceTests": { "hardPass": 6, "hardFail": 0, "softPass": 0, "softFail": 1 }, "defects": [], "ownTestSuite": { "passed": 223, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 4, "diff": { "files": 5, "added": 160, "deleted": 15 }, "usage": { "input": 70, "output": 60169, "cacheRead": 1692625, "cacheWrite": 141362, "cost": 0.064688, "calls": 33 }, "toolCalls": 47, "wallSeconds": 337.51, "reviewLetter": "C" }, { "case": "c06-stderr-escape", "model": "mercury", "acceptanceTests": { "hardPass": 9, "hardFail": 0, "softPass": 1, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "SPEC/summary claim stderr escapes match the stdout listing and are unambiguous; actual ESC/C1/DEL render double-escaped and collide with literal text", "verified": true, "verifiedHow": "Built a copy, mkfifo with that name, ran init, inspected stderr with cat -A" } ], "ownTestSuite": { "passed": 205, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 3, "diff": { "files": 6, "added": 166, "deleted": 43 }, "usage": { "input": 125189, "output": 156425, "cacheRead": 2759546, "cacheWrite": 0, "cost": 0.09486310999999997, "calls": 58 }, "toolCalls": 70, "wallSeconds": 289.782, "reviewLetter": "A" }, { "case": "c06-stderr-escape", "model": "luna", "acceptanceTests": { "hardPass": 9, "hardFail": 0, "softPass": 1, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "No backslash doubling, so escaped stderr output is ambiguous with literal backslash text in names", "verified": true, "verifiedHow": "Built a copy, mkfifo with mixed real/literal escapes, ran init, inspected stderr with cat -A" } ], "ownTestSuite": { "passed": 204, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 2, "diff": { "files": 3, "added": 101, "deleted": 3 }, "usage": { "input": 135, "output": 42409, "cacheRead": 3274887, "cacheWrite": 107582, "cost": 0.06741462000000002, "calls": 45 }, "toolCalls": 74, "wallSeconds": 599.683, "reviewLetter": "B" }, { "case": "c06-stderr-escape", "model": "haiku", "acceptanceTests": { "hardPass": 9, "hardFail": 0, "softPass": 1, "softFail": 0 }, "defects": [], "ownTestSuite": { "passed": 208, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 9, "diff": { "files": 6, "added": 271, "deleted": 62 }, "usage": { "input": 172, "output": 129775, "cacheRead": 4674654, "cacheWrite": 289115, "cost": 0.14779061500000001, "calls": 83 }, "toolCalls": 105, "wallSeconds": 820.075, "reviewLetter": "C" }, { "case": "c07-symlink-utf8", "model": "mercury", "acceptanceTests": { "hardPass": 6, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [], "ownTestSuite": { "passed": 227, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": true, "claudeMdEdited": false, "testFunctionsAdded": 12, "diff": { "files": 5, "added": 417, "deleted": 0 }, "usage": { "input": 107720, "output": 98655, "cacheRead": 3688466, "cacheWrite": 0, "cost": 0.08218590999999999, "calls": 56 }, "toolCalls": 80, "wallSeconds": 192.607, "reviewLetter": "A" }, { "case": "c07-symlink-utf8", "model": "luna", "acceptanceTests": { "hardPass": 6, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "major", "summary": "Moving the NotInitialized/AlreadyInitialized checks after compute_status(WardUpdate) makes those cheap precondition failures checksum the whole tree first", "verified": true, "verifiedHow": "Built a copy of the tree and timed both invocations against A and C on the same scratch trees" }, { "severity": "minor", "summary": "Error precedence change is undocumented: unrelated scan errors now mask NotInitialized", "verified": true, "verifiedHow": "chmod 000 a file in an uninitialized tree and ran update on all three builds" } ], "ownTestSuite": { "passed": 218, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 3, "diff": { "files": 4, "added": 157, "deleted": 8 }, "usage": { "input": 108, "output": 35613, "cacheRead": 2364443, "cacheWrite": 91709, "cost": 0.05292535500000001, "calls": 36 }, "toolCalls": 66, "wallSeconds": 609.102, "reviewLetter": "B" }, { "case": "c07-symlink-utf8", "model": "haiku", "acceptanceTests": { "hardPass": 6, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [], "ownTestSuite": { "passed": 218, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 3, "diff": { "files": 3, "added": 98, "deleted": 0 }, "usage": { "input": 98, "output": 59123, "cacheRead": 2154990, "cacheWrite": 143071, "cost": 0.069005075, "calls": 44 }, "toolCalls": 64, "wallSeconds": 451.942, "reviewLetter": "C" }, { "case": "c08-root-ward-type", "model": "mercury", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Unit test path assertion compares against non-canonical root.join(\".treeward\")", "verified": false, "verifiedHow": "Reasoning: ward_directory canonicalizes root before building ward_path; Linux /tmp is not a symlink so it passes here" } ], "ownTestSuite": { "passed": 230, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 4, "diff": { "files": 5, "added": 276, "deleted": 0 }, "usage": { "input": 90066, "output": 36271, "cacheRead": 2979127, "cacheWrite": 0, "cost": 0.051492720000000006, "calls": 54 }, "toolCalls": 72, "wallSeconds": 162.857, "reviewLetter": "A" }, { "case": "c08-root-ward-type", "model": "luna", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "CLI test expects the non-canonical root path in stderr", "verified": false, "verifiedHow": "Reasoning: ward_directory canonicalizes root; assertion uses root.join(\".treeward\").display()" } ], "ownTestSuite": { "passed": 228, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 2, "diff": { "files": 4, "added": 138, "deleted": 12 }, "usage": { "input": 57, "output": 23230, "cacheRead": 888222, "cacheWrite": 71667, "cost": 0.029461294999999995, "calls": 19 }, "toolCalls": 49, "wallSeconds": 256.829, "reviewLetter": "B" }, { "case": "c08-root-ward-type", "model": "haiku", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "'valid-target' unit fixture links to an empty file, not a valid ward, and the CLI test omits the symlink-to-valid-ward case", "verified": true, "verifiedHow": "Read the test: fs::write(root.join(\"real.treeward\"), \"\") then symlink; tests/init.rs cases are loop, dangling, directory only" } ], "ownTestSuite": { "passed": 229, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 3, "diff": { "files": 3, "added": 194, "deleted": 2 }, "usage": { "input": 84, "output": 52850, "cacheRead": 2130223, "cacheWrite": 130808, "cost": 0.06408662999999999, "calls": 40 }, "toolCalls": 63, "wallSeconds": 363.817, "reviewLetter": "C" }, { "case": "c09-sha256-validate", "model": "haiku", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "ward_directory doc comment still says an existing root is always an AlreadyInitialized error", "verified": true, "verifiedHow": "grep of src/update.rs in a copy of the tree; behavior confirmed by running init on a corrupt root (ward-file error, not AlreadyInitialized)" } ], "ownTestSuite": { "passed": 204, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 6, "diff": { "files": 10, "added": 305, "deleted": 46 }, "usage": { "input": 126, "output": 76830, "cacheRead": 3575212, "cacheWrite": 241872, "cost": 0.10441371999999997, "calls": 60 }, "toolCalls": 95, "wallSeconds": 499.305, "reviewLetter": "A" }, { "case": "c09-sha256-validate", "model": "luna", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [], "ownTestSuite": { "passed": 202, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": true, "claudeMdEdited": false, "testFunctionsAdded": 4, "diff": { "files": 11, "added": 246, "deleted": 49 }, "usage": { "input": 141, "output": 37161, "cacheRead": 3900377, "cacheWrite": 127539, "cost": 0.07354074499999998, "calls": 47 }, "toolCalls": 100, "wallSeconds": 996.249, "reviewLetter": "B" }, { "case": "c09-sha256-validate", "model": "mercury", "acceptanceTests": { "hardPass": 5, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [], "ownTestSuite": { "passed": 205, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": true, "specMdUpdated": true, "changelogEdited": true, "claudeMdEdited": false, "testFunctionsAdded": 7, "diff": { "files": 14, "added": 266, "deleted": 46 }, "usage": { "input": 282243, "output": 46839, "cacheRead": 7088621, "cacheWrite": 0, "cost": 0.11550416000000001, "calls": 79 }, "toolCalls": 125, "wallSeconds": 211.553, "reviewLetter": "C" }, { "case": "c10-status-diff", "model": "haiku", "acceptanceTests": { "hardPass": 2, "hardFail": 12, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "major", "summary": "Status code is right-aligned instead of left-aligned, contradicting the explicit `A new.txt` / `. file.txt` examples", "verified": true, "verifiedHow": "Hidden log shows ` A new.txt`, ` R olddir`; changing `{:>2}` to `{:<2}` in a copy makes all 14 hidden tests pass (only the old substring tests in tests/status.rs then fail, as expected)" }, { "severity": "minor", "summary": "Help-text example under DIFF DETAILS does not match real output (unpadded codes, 6-space indent vs 4)", "verified": true, "verifiedHow": "Compared cli.rs help block with repro output; candidate also admits it" }, { "severity": "minor", "summary": "Size rounding can print `1000.0 KB`", "verified": true, "verifiedHow": "Repro in scratch tree; candidate acknowledges it as cosmetic" } ], "ownTestSuite": { "passed": 140, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 7, "diff": { "files": 7, "added": 568, "deleted": 21 }, "usage": { "input": 150, "output": 311448, "cacheRead": 3647756, "cacheWrite": 388636, "cost": 0.24079606, "calls": 73 }, "toolCalls": 115, "wallSeconds": 1624.24, "reviewLetter": "A" }, { "case": "c10-status-diff", "model": "mercury", "acceptanceTests": { "hardPass": 14, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Checksum shortening slices by byte index and panics on non-ASCII sha256 from a corrupt ward file", "verified": true, "verifiedHow": "Edited .treeward in a scratch tree and ran the copied binary; same latent bug as the reference, so low weight" }, { "severity": "minor", "summary": "`collect_new_state = purpose != Display` also fires for WardUpdate, so `update` now clones prev_entry for every compared entry and the ward entry for every removed one", "verified": false, "verifiedHow": "Code reading of check_modification and compare_entries" }, { "severity": "minor", "summary": "Silent-corruption integration test does not test silent corruption", "verified": true, "verifiedHow": "Counted 'original content' (16) vs 'originaL conte' (14)" } ], "ownTestSuite": { "passed": 142, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 9, "diff": { "files": 8, "added": 855, "deleted": 38 }, "usage": { "input": 144230, "output": 52685, "cacheRead": 5834050, "cacheWrite": 0, "cost": 0.09120325000000001, "calls": 73 }, "toolCalls": 98, "wallSeconds": 248.497, "reviewLetter": "B" }, { "case": "c10-status-diff", "model": "luna", "acceptanceTests": { "hardPass": 14, "hardFail": 0, "softPass": 0, "softFail": 0 }, "defects": [ { "severity": "minor", "summary": "Size rounding can print `1024 KB` instead of `1.0 MB`", "verified": true, "verifiedHow": "Repro in scratch tree" } ], "ownTestSuite": { "passed": 137, "failed": 0 }, "fmtOk": true, "clippyNewDiagnostics": 0, "dprintUnformattedRepoFiles": [], "specMdExists": false, "specMdUpdated": false, "changelogEdited": false, "claudeMdEdited": false, "testFunctionsAdded": 4, "diff": { "files": 8, "added": 779, "deleted": 114 }, "usage": { "input": 153, "output": 50348, "cacheRead": 4937200, "cacheWrite": 148762, "cost": 0.09315655, "calls": 51 }, "toolCalls": 107, "wallSeconds": 806.193, "reviewLetter": "C" } ], "totalCostUsd": 2.9535671650000004 }