Core: cancel moved runs, retry bookkeeping, progress cap, safer classification
- A run whose item leaves In Progress is killed, gets a [worker] cancel comment, frees its terminal slot and serialize group and keeps its thread id; if the kill fails the slot stays taken and the cancel is retried next tick. - Transient retries are the same logical round: they do not count toward max_rounds_per_thread and do not trigger a max-rounds rebuild. - New project setting max_progress_rounds (default 12): after that many consecutive [progress] rounds the item goes to Waiting (answerer) with a note. - retry/rebuild markers are consumed only after the run launched, so a failed start no longer leaves an In Progress item that is never picked again; a rebuild is no longer reported as a retry. - State is saved after every finished round and every launched run. - Failure classification: transient patterns only look at the tail, and an exit-0 failure is logic (never retried) unless the tail shows a transient error. - First-line tag: tolerate BOM and a leading code fence / rule line. - Start comment notes when a handoff summary was attached.
This commit is contained in:
@@ -87,8 +87,12 @@ class FakeExecutor:
|
||||
def __init__(self, resume=True):
|
||||
self.resume = resume
|
||||
self.started, self.killed, self.done = [], [], {}
|
||||
self.fail_starts, self.fail_kills = 0, 0
|
||||
|
||||
def start(self, run):
|
||||
if self.fail_starts:
|
||||
self.fail_starts -= 1
|
||||
raise RuntimeError("relay down")
|
||||
self.started.append(run)
|
||||
return {"run_dir": "/runs/" + run.run_id, "run_id": run.run_id, "thread_id": run.thread_id}
|
||||
|
||||
@@ -99,6 +103,9 @@ class FakeExecutor:
|
||||
return self.done[handle["run_id"]]
|
||||
|
||||
def kill(self, handle):
|
||||
if self.fail_kills:
|
||||
self.fail_kills -= 1
|
||||
raise RuntimeError("relay down")
|
||||
self.killed.append(handle["run_id"])
|
||||
|
||||
def finish(self, message, exit_code=0, thread="th-1", tail=""):
|
||||
|
||||
+136
-1
@@ -123,7 +123,7 @@ class CoreTest(unittest.TestCase):
|
||||
self.assertIn("## 交接摘要", run.prompt)
|
||||
self.assertIn("续线程失败", run.prompt)
|
||||
self.assertIn("[progress] half way", run.prompt)
|
||||
self.assertIn("新线程(重建:续线程失败(crash))", self.p.texts(iid)[-1])
|
||||
self.assertIn("新线程(重建:续线程失败(crash),附交接摘要)", self.p.texts(iid)[-1])
|
||||
|
||||
def test_thread_rebuilt_after_max_rounds(self):
|
||||
iid = self.p.add_issue(1)
|
||||
@@ -198,6 +198,141 @@ class CoreTest(unittest.TestCase):
|
||||
self.assertEqual(hb["inflight"][0]["run"], "GW-1-1")
|
||||
self.assertEqual(hb["errors"], [])
|
||||
|
||||
# ---------------------------------------------------------------- reviewer decisions / fixed defects
|
||||
def test_item_moved_out_of_in_progress_cancels_run(self):
|
||||
a = self.p.add_issue(1, labels=["needs-game"])
|
||||
b = self.p.add_issue(2, labels=["needs-game"])
|
||||
self.h.tick()
|
||||
self.codex().finish("[progress] step 1")
|
||||
self.h.tick() # round 2 resumes th-1
|
||||
self.assertEqual(self.codex().started[-1].run_id, "GW-1-2")
|
||||
self.p.issues[a]["state"] = "s-wait" # a person moves it while round 2 runs
|
||||
self.h.tick()
|
||||
self.assertEqual(self.codex().killed, ["GW-1-2"])
|
||||
self.assertIn("[worker] 运行已取消:工作项状态被改为 Waiting(run=GW-1-2)", self.p.texts(a)[-1])
|
||||
rec = self.h.rec(a)
|
||||
self.assertNotIn("inflight", rec)
|
||||
self.assertEqual((rec["thread_id"], rec["thread_on"]), ("th-1", ["win10", "codex"]))
|
||||
self.assertEqual(self.p.state_of(a), "Waiting") # agentd does not touch the state
|
||||
# terminal slot (concurrency 1) and serialize group were released in the same tick
|
||||
self.assertEqual(self.codex().started[-1].run_id, "GW-2-1")
|
||||
self.assertEqual(self.p.state_of(b), "In Progress")
|
||||
# back to Todo later: resumes the kept thread
|
||||
self.codex().finish("[review] b done")
|
||||
self.p.issues[a]["state"] = "s-todo"
|
||||
self.h.tick()
|
||||
self.assertEqual((self.codex().started[-1].run_id, self.codex().started[-1].thread_id), ("GW-1-3", "th-1"))
|
||||
|
||||
def test_cancel_keeps_slot_when_kill_fails(self):
|
||||
a = self.p.add_issue(1)
|
||||
self.p.add_issue(2)
|
||||
self.h.tick()
|
||||
self.codex().fail_kills = 1
|
||||
self.p.issues[a]["state"] = "s-bl"
|
||||
self.h.tick()
|
||||
self.assertIn("inflight", self.h.rec(a))
|
||||
self.assertEqual(len(self.codex().started), 1) # GW-2 must not share win10 with a live run
|
||||
self.h.tick()
|
||||
self.assertNotIn("inflight", self.h.rec(a))
|
||||
self.assertIn("被改为 Backlog", self.p.texts(a)[-1])
|
||||
self.assertEqual(self.codex().started[-1].run_id, "GW-2-1")
|
||||
|
||||
def test_transient_retries_do_not_use_up_thread_rounds(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("[progress] 1")
|
||||
self.h.tick()
|
||||
self.codex().finish("[progress] 2")
|
||||
self.h.tick() # round 3: thread_rounds == 3 == max
|
||||
self.assertEqual(self.h.rec(iid)["thread_rounds"], 3)
|
||||
self.codex().finish("", exit_code=1, tail="stream disconnected before completion")
|
||||
self.h.tick()
|
||||
self.h.clock[0] += 121
|
||||
self.h.tick()
|
||||
retry = self.codex().started[-1]
|
||||
self.assertEqual((retry.run_id, retry.thread_id), ("GW-1-4", "th-1")) # same thread, new run id
|
||||
self.assertEqual(self.h.rec(iid)["thread_rounds"], 3)
|
||||
self.codex().finish("[progress] 3")
|
||||
self.h.tick()
|
||||
self.assertIsNone(self.codex().started[-1].thread_id) # next real round rebuilds
|
||||
self.assertIn("线程已续 3 轮(上限 3)", self.p.texts(iid)[-1])
|
||||
|
||||
def test_failed_start_keeps_retry_marker(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("", exit_code=1, tail="Connection reset by peer")
|
||||
self.h.tick()
|
||||
self.h.clock[0] += 121
|
||||
self.codex().fail_starts = 1
|
||||
self.h.tick() # relay down while relaunching
|
||||
self.assertIn("retry", self.h.rec(iid))
|
||||
self.assertEqual(self.p.state_of(iid), "In Progress")
|
||||
self.h.tick()
|
||||
self.assertEqual(self.codex().started[-1].thread_id, "th-1")
|
||||
self.assertIn("inflight", self.h.rec(iid))
|
||||
|
||||
def test_progress_cap_moves_to_waiting(self):
|
||||
data = config_data()
|
||||
data["projects"]["GWENT"]["max_progress_rounds"] = 2
|
||||
h = Harness(data)
|
||||
iid = h.plane.add_issue(1)
|
||||
h.tick()
|
||||
ex = h.execs[("win10", "codex")]
|
||||
ex.finish("[progress] 1")
|
||||
h.tick()
|
||||
self.assertEqual(len(ex.started), 2)
|
||||
ex.finish("[progress] 2")
|
||||
h.tick()
|
||||
self.assertEqual(len(ex.started), 2)
|
||||
self.assertEqual((h.plane.state_of(iid), h.plane.assignee_of(iid)), ("Waiting", "kappiee"))
|
||||
self.assertIn("已连续 2 轮 [progress]", h.plane.texts(iid)[-1])
|
||||
h.plane.issues[iid].update(state="s-todo", assignees=["u-w10"]) # a person lets it continue: fresh allowance
|
||||
h.tick()
|
||||
ex.finish("[progress] 3")
|
||||
h.tick()
|
||||
self.assertEqual(len(ex.started), 4)
|
||||
h.close()
|
||||
|
||||
def test_tagged_message_counts_as_success_even_with_nonzero_exit(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("[review] done, but a hook failed", exit_code=1)
|
||||
self.h.tick()
|
||||
self.assertEqual(self.p.state_of(iid), "Review")
|
||||
self.assertIn("第 1 轮结束(exit=1,", self.p.texts(iid)[-1])
|
||||
|
||||
def test_untagged_message_with_nonzero_exit_is_a_failure(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("I gave up, tests timed out", exit_code=1, tail="")
|
||||
self.h.tick()
|
||||
self.assertEqual(self.p.state_of(iid), "Waiting") # agent prose never makes it transient
|
||||
self.assertIn("类型=crash,run=GW-1-1", self.p.texts(iid)[-1])
|
||||
|
||||
def test_rebuild_after_resume_failure_happens_once(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("[progress] half way")
|
||||
self.h.tick()
|
||||
self.codex().finish("", exit_code=1, thread=None, tail="thread not found")
|
||||
self.h.tick() # immediate rebuild (round 3, fresh thread)
|
||||
self.assertIsNone(self.codex().started[2].thread_id)
|
||||
self.assertNotIn("重试", self.p.texts(iid)[-1])
|
||||
self.codex().finish("", exit_code=1, thread=None, tail="thread not found")
|
||||
self.h.tick()
|
||||
self.assertEqual(len(self.codex().started), 3) # no second rebuild
|
||||
self.assertEqual((self.p.state_of(iid), self.p.assignee_of(iid)), ("Waiting", "kappiee"))
|
||||
|
||||
def test_state_saved_right_after_each_start(self):
|
||||
self.p.add_issue(1, assignee="oregon")
|
||||
self.p.add_issue(2, assignee="oregon")
|
||||
saves = []
|
||||
orig = self.h.d.store.save
|
||||
self.h.d.store.save = lambda data: (saves.append(sorted(
|
||||
r["inflight"]["run_id"] for r in data["projects"]["GWENT"]["issues"].values() if r.get("inflight"))), orig(data))
|
||||
self.h.tick()
|
||||
# one save per launched run (a crash mid-tick cannot orphan it), then the end-of-tick save
|
||||
self.assertEqual(saves, [["GW-1-1"], ["GW-1-1", "GW-2-1"], ["GW-1-1", "GW-2-1"]])
|
||||
|
||||
class EnvFailureTest(unittest.TestCase):
|
||||
def test_env_failure_pauses_group_until_item_leaves_waiting(self):
|
||||
|
||||
@@ -37,6 +37,13 @@ class PromptTest(unittest.TestCase):
|
||||
self.assertIn("RETRY", p)
|
||||
self.assertNotIn("没有新评论", p)
|
||||
|
||||
def test_new_comment_filter_is_by_id_not_timestamp(self):
|
||||
late_but_old = [{"id": "c9", "created_at": "t0", "comment_html": "<p>clock-skewed reply</p>"},
|
||||
{"id": "c1", "created_at": "t9", "comment_html": "<p>edited old one</p>"}]
|
||||
p = build_prompt("GW-1", ISSUE, late_but_old, {"c1"}, True)
|
||||
self.assertIn("clock-skewed reply", p)
|
||||
self.assertNotIn("edited old one", p)
|
||||
|
||||
def test_handoff(self):
|
||||
rec = {"runs": 3, "tags": ["progress", "progress", "question"], "last_message": "[question] A or B?"}
|
||||
h = handoff_summary(rec, "线程已续 3 轮")
|
||||
|
||||
@@ -13,6 +13,18 @@ class FirstLineTest(unittest.TestCase):
|
||||
self.assertEqual(classify_first_line("**[blocked]** no device"), "blocked")
|
||||
self.assertEqual(classify_first_line("[progress]"), "progress")
|
||||
|
||||
def test_edge_cases(self):
|
||||
self.assertEqual(classify_first_line("\ufeff[review] ok"), "review")
|
||||
self.assertEqual(classify_first_line("```\n[progress] still going\n```"), "progress")
|
||||
self.assertEqual(classify_first_line("```text\n[blocked] no device\n```"), "blocked")
|
||||
self.assertEqual(classify_first_line("---\n[question] which?"), "question")
|
||||
self.assertEqual(classify_first_line("[progress]: step 2 of 5, next: tests"), "progress")
|
||||
self.assertEqual(classify_first_line("> `[review]` merged"), "review")
|
||||
self.assertIsNone(classify_first_line("note first\n[review] later")) # tag must lead
|
||||
msg = html_to_text(text_to_html("[codex] 第 1 轮结束", pre="\n [review] ok\n"))
|
||||
self.assertEqual(msg, "[codex] 第 1 轮结束\n\n [review] ok") # <pre>-wrapped body keeps its lines
|
||||
self.assertEqual(classify_first_line(html_to_text("<pre>\n[progress] x</pre>")), "progress")
|
||||
|
||||
def test_untagged(self):
|
||||
self.assertIsNone(classify_first_line("done, see [review] below"))
|
||||
self.assertIsNone(classify_first_line("[done] finished"))
|
||||
@@ -56,6 +68,13 @@ class FailureTest(unittest.TestCase):
|
||||
self.assertEqual(self.kind("whatever", timed_out=True), ("timeout", False))
|
||||
self.assertEqual(self.kind("window lost focus", cfg={"env_patterns": ["lost focus"]}), ("env", False))
|
||||
|
||||
def test_logic_is_never_transient_and_prose_is_ignored(self):
|
||||
self.assertEqual(self.kind("HTTP 429 Too Many Requests", exit_code=0), ("crash", True))
|
||||
r = ExecResult("tests timed out", 1, None, [], "")
|
||||
self.assertEqual(classify_failure(r, False, {}), ("crash", False))
|
||||
r = ExecResult("foreground window lost", 1, None, [], "")
|
||||
self.assertEqual(classify_failure(r, False, {"env_patterns": ["window lost"]}), ("env", False))
|
||||
|
||||
def test_custom_transient_patterns_replace_defaults(self):
|
||||
self.assertEqual(self.kind("timed out", cfg={"transient_patterns": ["flaky"]}), ("crash", False))
|
||||
self.assertEqual(self.kind("flaky relay", cfg={"transient_patterns": ["flaky"]}), ("crash", True))
|
||||
|
||||
Reference in New Issue
Block a user