Core: cancel moved runs, retry bookkeeping, progress cap, safer classification

- A run whose item leaves In Progress is killed, gets a [worker] cancel comment,
  frees its terminal slot and serialize group and keeps its thread id; if the
  kill fails the slot stays taken and the cancel is retried next tick.
- Transient retries are the same logical round: they do not count toward
  max_rounds_per_thread and do not trigger a max-rounds rebuild.
- New project setting max_progress_rounds (default 12): after that many
  consecutive [progress] rounds the item goes to Waiting (answerer) with a note.
- retry/rebuild markers are consumed only after the run launched, so a failed
  start no longer leaves an In Progress item that is never picked again; a
  rebuild is no longer reported as a retry.
- State is saved after every finished round and every launched run.
- Failure classification: transient patterns only look at the tail, and an
  exit-0 failure is logic (never retried) unless the tail shows a transient error.
- First-line tag: tolerate BOM and a leading code fence / rule line.
- Start comment notes when a handoff summary was attached.
This commit is contained in:
jj.wang
2026-10-21 07:13:18 +00:00
parent 00d42267e5
commit efbc5f6e27
7 changed files with 258 additions and 18 deletions
+7
View File
@@ -87,8 +87,12 @@ class FakeExecutor:
def __init__(self, resume=True):
self.resume = resume
self.started, self.killed, self.done = [], [], {}
self.fail_starts, self.fail_kills = 0, 0
def start(self, run):
if self.fail_starts:
self.fail_starts -= 1
raise RuntimeError("relay down")
self.started.append(run)
return {"run_dir": "/runs/" + run.run_id, "run_id": run.run_id, "thread_id": run.thread_id}
@@ -99,6 +103,9 @@ class FakeExecutor:
return self.done[handle["run_id"]]
def kill(self, handle):
if self.fail_kills:
self.fail_kills -= 1
raise RuntimeError("relay down")
self.killed.append(handle["run_id"])
def finish(self, message, exit_code=0, thread="th-1", tail=""):
+136 -1
View File
@@ -123,7 +123,7 @@ class CoreTest(unittest.TestCase):
self.assertIn("## 交接摘要", run.prompt)
self.assertIn("续线程失败", run.prompt)
self.assertIn("[progress] half way", run.prompt)
self.assertIn("新线程(重建:续线程失败(crash))", self.p.texts(iid)[-1])
self.assertIn("新线程(重建:续线程失败(crash),附交接摘要)", self.p.texts(iid)[-1])
def test_thread_rebuilt_after_max_rounds(self):
iid = self.p.add_issue(1)
@@ -198,6 +198,141 @@ class CoreTest(unittest.TestCase):
self.assertEqual(hb["inflight"][0]["run"], "GW-1-1")
self.assertEqual(hb["errors"], [])
# ---------------------------------------------------------------- reviewer decisions / fixed defects
def test_item_moved_out_of_in_progress_cancels_run(self):
a = self.p.add_issue(1, labels=["needs-game"])
b = self.p.add_issue(2, labels=["needs-game"])
self.h.tick()
self.codex().finish("[progress] step 1")
self.h.tick() # round 2 resumes th-1
self.assertEqual(self.codex().started[-1].run_id, "GW-1-2")
self.p.issues[a]["state"] = "s-wait" # a person moves it while round 2 runs
self.h.tick()
self.assertEqual(self.codex().killed, ["GW-1-2"])
self.assertIn("[worker] 运行已取消:工作项状态被改为 Waiting(run=GW-1-2)", self.p.texts(a)[-1])
rec = self.h.rec(a)
self.assertNotIn("inflight", rec)
self.assertEqual((rec["thread_id"], rec["thread_on"]), ("th-1", ["win10", "codex"]))
self.assertEqual(self.p.state_of(a), "Waiting") # agentd does not touch the state
# terminal slot (concurrency 1) and serialize group were released in the same tick
self.assertEqual(self.codex().started[-1].run_id, "GW-2-1")
self.assertEqual(self.p.state_of(b), "In Progress")
# back to Todo later: resumes the kept thread
self.codex().finish("[review] b done")
self.p.issues[a]["state"] = "s-todo"
self.h.tick()
self.assertEqual((self.codex().started[-1].run_id, self.codex().started[-1].thread_id), ("GW-1-3", "th-1"))
def test_cancel_keeps_slot_when_kill_fails(self):
a = self.p.add_issue(1)
self.p.add_issue(2)
self.h.tick()
self.codex().fail_kills = 1
self.p.issues[a]["state"] = "s-bl"
self.h.tick()
self.assertIn("inflight", self.h.rec(a))
self.assertEqual(len(self.codex().started), 1) # GW-2 must not share win10 with a live run
self.h.tick()
self.assertNotIn("inflight", self.h.rec(a))
self.assertIn("被改为 Backlog", self.p.texts(a)[-1])
self.assertEqual(self.codex().started[-1].run_id, "GW-2-1")
def test_transient_retries_do_not_use_up_thread_rounds(self):
iid = self.p.add_issue(1)
self.h.tick()
self.codex().finish("[progress] 1")
self.h.tick()
self.codex().finish("[progress] 2")
self.h.tick() # round 3: thread_rounds == 3 == max
self.assertEqual(self.h.rec(iid)["thread_rounds"], 3)
self.codex().finish("", exit_code=1, tail="stream disconnected before completion")
self.h.tick()
self.h.clock[0] += 121
self.h.tick()
retry = self.codex().started[-1]
self.assertEqual((retry.run_id, retry.thread_id), ("GW-1-4", "th-1")) # same thread, new run id
self.assertEqual(self.h.rec(iid)["thread_rounds"], 3)
self.codex().finish("[progress] 3")
self.h.tick()
self.assertIsNone(self.codex().started[-1].thread_id) # next real round rebuilds
self.assertIn("线程已续 3 轮(上限 3)", self.p.texts(iid)[-1])
def test_failed_start_keeps_retry_marker(self):
iid = self.p.add_issue(1)
self.h.tick()
self.codex().finish("", exit_code=1, tail="Connection reset by peer")
self.h.tick()
self.h.clock[0] += 121
self.codex().fail_starts = 1
self.h.tick() # relay down while relaunching
self.assertIn("retry", self.h.rec(iid))
self.assertEqual(self.p.state_of(iid), "In Progress")
self.h.tick()
self.assertEqual(self.codex().started[-1].thread_id, "th-1")
self.assertIn("inflight", self.h.rec(iid))
def test_progress_cap_moves_to_waiting(self):
data = config_data()
data["projects"]["GWENT"]["max_progress_rounds"] = 2
h = Harness(data)
iid = h.plane.add_issue(1)
h.tick()
ex = h.execs[("win10", "codex")]
ex.finish("[progress] 1")
h.tick()
self.assertEqual(len(ex.started), 2)
ex.finish("[progress] 2")
h.tick()
self.assertEqual(len(ex.started), 2)
self.assertEqual((h.plane.state_of(iid), h.plane.assignee_of(iid)), ("Waiting", "kappiee"))
self.assertIn("已连续 2 轮 [progress]", h.plane.texts(iid)[-1])
h.plane.issues[iid].update(state="s-todo", assignees=["u-w10"]) # a person lets it continue: fresh allowance
h.tick()
ex.finish("[progress] 3")
h.tick()
self.assertEqual(len(ex.started), 4)
h.close()
def test_tagged_message_counts_as_success_even_with_nonzero_exit(self):
iid = self.p.add_issue(1)
self.h.tick()
self.codex().finish("[review] done, but a hook failed", exit_code=1)
self.h.tick()
self.assertEqual(self.p.state_of(iid), "Review")
self.assertIn("第 1 轮结束(exit=1,", self.p.texts(iid)[-1])
def test_untagged_message_with_nonzero_exit_is_a_failure(self):
iid = self.p.add_issue(1)
self.h.tick()
self.codex().finish("I gave up, tests timed out", exit_code=1, tail="")
self.h.tick()
self.assertEqual(self.p.state_of(iid), "Waiting") # agent prose never makes it transient
self.assertIn("类型=crash,run=GW-1-1", self.p.texts(iid)[-1])
def test_rebuild_after_resume_failure_happens_once(self):
iid = self.p.add_issue(1)
self.h.tick()
self.codex().finish("[progress] half way")
self.h.tick()
self.codex().finish("", exit_code=1, thread=None, tail="thread not found")
self.h.tick() # immediate rebuild (round 3, fresh thread)
self.assertIsNone(self.codex().started[2].thread_id)
self.assertNotIn("重试", self.p.texts(iid)[-1])
self.codex().finish("", exit_code=1, thread=None, tail="thread not found")
self.h.tick()
self.assertEqual(len(self.codex().started), 3) # no second rebuild
self.assertEqual((self.p.state_of(iid), self.p.assignee_of(iid)), ("Waiting", "kappiee"))
def test_state_saved_right_after_each_start(self):
self.p.add_issue(1, assignee="oregon")
self.p.add_issue(2, assignee="oregon")
saves = []
orig = self.h.d.store.save
self.h.d.store.save = lambda data: (saves.append(sorted(
r["inflight"]["run_id"] for r in data["projects"]["GWENT"]["issues"].values() if r.get("inflight"))), orig(data))
self.h.tick()
# one save per launched run (a crash mid-tick cannot orphan it), then the end-of-tick save
self.assertEqual(saves, [["GW-1-1"], ["GW-1-1", "GW-2-1"], ["GW-1-1", "GW-2-1"]])
class EnvFailureTest(unittest.TestCase):
def test_env_failure_pauses_group_until_item_leaves_waiting(self):
+7
View File
@@ -37,6 +37,13 @@ class PromptTest(unittest.TestCase):
self.assertIn("RETRY", p)
self.assertNotIn("没有新评论", p)
def test_new_comment_filter_is_by_id_not_timestamp(self):
late_but_old = [{"id": "c9", "created_at": "t0", "comment_html": "<p>clock-skewed reply</p>"},
{"id": "c1", "created_at": "t9", "comment_html": "<p>edited old one</p>"}]
p = build_prompt("GW-1", ISSUE, late_but_old, {"c1"}, True)
self.assertIn("clock-skewed reply", p)
self.assertNotIn("edited old one", p)
def test_handoff(self):
rec = {"runs": 3, "tags": ["progress", "progress", "question"], "last_message": "[question] A or B?"}
h = handoff_summary(rec, "线程已续 3 轮")
+19
View File
@@ -13,6 +13,18 @@ class FirstLineTest(unittest.TestCase):
self.assertEqual(classify_first_line("**[blocked]** no device"), "blocked")
self.assertEqual(classify_first_line("[progress]"), "progress")
def test_edge_cases(self):
self.assertEqual(classify_first_line("\ufeff[review] ok"), "review")
self.assertEqual(classify_first_line("```\n[progress] still going\n```"), "progress")
self.assertEqual(classify_first_line("```text\n[blocked] no device\n```"), "blocked")
self.assertEqual(classify_first_line("---\n[question] which?"), "question")
self.assertEqual(classify_first_line("[progress]: step 2 of 5, next: tests"), "progress")
self.assertEqual(classify_first_line("> `[review]` merged"), "review")
self.assertIsNone(classify_first_line("note first\n[review] later")) # tag must lead
msg = html_to_text(text_to_html("[codex] 第 1 轮结束", pre="\n [review] ok\n"))
self.assertEqual(msg, "[codex] 第 1 轮结束\n\n [review] ok") # <pre>-wrapped body keeps its lines
self.assertEqual(classify_first_line(html_to_text("<pre>\n[progress] x</pre>")), "progress")
def test_untagged(self):
self.assertIsNone(classify_first_line("done, see [review] below"))
self.assertIsNone(classify_first_line("[done] finished"))
@@ -56,6 +68,13 @@ class FailureTest(unittest.TestCase):
self.assertEqual(self.kind("whatever", timed_out=True), ("timeout", False))
self.assertEqual(self.kind("window lost focus", cfg={"env_patterns": ["lost focus"]}), ("env", False))
def test_logic_is_never_transient_and_prose_is_ignored(self):
self.assertEqual(self.kind("HTTP 429 Too Many Requests", exit_code=0), ("crash", True))
r = ExecResult("tests timed out", 1, None, [], "")
self.assertEqual(classify_failure(r, False, {}), ("crash", False))
r = ExecResult("foreground window lost", 1, None, [], "")
self.assertEqual(classify_failure(r, False, {"env_patterns": ["window lost"]}), ("env", False))
def test_custom_transient_patterns_replace_defaults(self):
self.assertEqual(self.kind("timed out", cfg={"transient_patterns": ["flaky"]}), ("crash", False))
self.assertEqual(self.kind("flaky relay", cfg={"transient_patterns": ["flaky"]}), ("crash", True))