Scheduling core, state store, CLI and unit tests
One tick polls in-flight runs, posts results, classifies the first line and moves state/assignee; failures get a structured note with transient retry and backoff, timeouts are killed and handed over, resume failures and long threads rebuild with a handoff summary. Then the lowest Todo items are dispatched within terminal concurrency and serialize groups, and a heartbeat is written. CLI: tick, new, show, comment, state, assign, smoke, check-config.
This commit is contained in:
jj.wang
@@ -0,0 +1,223 @@
|
||||
import unittest
|
||||
|
||||
from tests.fakes import Harness, config_data
|
||||
|
||||
|
||||
class CoreTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.h = Harness()
|
||||
self.p = self.h.plane
|
||||
|
||||
def tearDown(self):
|
||||
self.h.close()
|
||||
|
||||
def codex(self):
|
||||
return self.h.execs[("win10", "codex")]
|
||||
|
||||
def test_pick_starts_lowest_todo_and_marks_in_progress(self):
|
||||
a, b = self.p.add_issue(7), self.p.add_issue(3)
|
||||
self.p.add_issue(1, state="Backlog")
|
||||
self.p.add_issue(2, assignee="kappiee") # not routed
|
||||
self.h.tick()
|
||||
runs = self.codex().started
|
||||
self.assertEqual([r.run_id for r in runs], ["GW-3-1"]) # concurrency 1 on win10
|
||||
self.assertEqual(self.p.state_of(b), "In Progress")
|
||||
self.assertEqual(self.p.state_of(a), "Todo")
|
||||
self.assertTrue(self.p.texts(b)[-1].startswith("[worker] 第 1 轮开始：codex / sol / medium @ win10，新线程"))
|
||||
self.assertIsNone(runs[0].thread_id)
|
||||
self.assertIn("do it", runs[0].prompt)
|
||||
|
||||
def test_review_question_progress_transitions(self):
|
||||
for tag, state, who in (("review", "Review", "oregon"), ("question", "Waiting", "kappiee"),
|
||||
("blocked", "Waiting", "kappiee"), ("progress", "Todo", "window10")):
|
||||
with self.subTest(tag=tag):
|
||||
h = Harness()
|
||||
iid = h.plane.add_issue(1)
|
||||
h.tick()
|
||||
ex = h.execs[("win10", "codex")]
|
||||
ex.finish("[%s] text" % tag)
|
||||
h.tick()
|
||||
if tag == "progress": # picked again in the same tick, resuming the thread
|
||||
self.assertEqual(len(ex.started), 2)
|
||||
self.assertEqual(ex.started[1].thread_id, "th-1")
|
||||
self.assertEqual(h.plane.state_of(iid), "In Progress")
|
||||
self.assertIn(("i1", "s-todo", ["u-w10"]), h.plane.patches)
|
||||
else:
|
||||
self.assertEqual((h.plane.state_of(iid), h.plane.assignee_of(iid)), (state, who))
|
||||
texts = h.plane.texts(iid)
|
||||
result = [t for t in texts if t.startswith("[codex]")][0]
|
||||
self.assertIn("第 1 轮结束（exit=0，run=GW-1-1，终端 win10", result)
|
||||
self.assertIn("[%s] text" % tag, result)
|
||||
self.assertEqual(h.rec(iid)["tags"], [tag])
|
||||
h.close()
|
||||
|
||||
def test_untagged_message_becomes_review(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("all done")
|
||||
self.h.tick()
|
||||
self.assertEqual(self.p.state_of(iid), "Review")
|
||||
self.assertIn("未按协议标注首行", self.p.texts(iid)[-1])
|
||||
|
||||
def test_resume_sends_only_new_comments(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.p.add_comment(iid, "<p>old context</p>")
|
||||
self.h.tick()
|
||||
self.codex().finish("[question] A or B?")
|
||||
self.h.tick()
|
||||
self.p.add_comment(iid, "<p>B please</p>")
|
||||
self.p.issues[iid].update(state="s-todo", assignees=["u-w10"])
|
||||
self.h.tick()
|
||||
run = self.codex().started[-1]
|
||||
self.assertEqual(run.thread_id, "th-1")
|
||||
self.assertIn("B please", run.prompt)
|
||||
self.assertNotIn("old context", run.prompt)
|
||||
self.assertNotIn("A or B?", run.prompt)
|
||||
self.assertIn("续线程 th-1", self.p.texts(iid)[-1])
|
||||
|
||||
def test_transient_failure_retries_with_backoff_then_waits(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("", exit_code=1, tail="Connection reset by peer")
|
||||
self.h.tick()
|
||||
note = self.p.texts(iid)[-1]
|
||||
self.assertIn("类型=crash（临时性）", note)
|
||||
self.assertIn("重试 1/2", note)
|
||||
self.assertEqual(self.p.state_of(iid), "In Progress")
|
||||
self.assertEqual(len(self.codex().started), 1) # backoff 2 min not elapsed
|
||||
self.h.clock[0] += 121
|
||||
self.h.tick()
|
||||
self.assertEqual(len(self.codex().started), 2)
|
||||
self.assertEqual(self.codex().started[1].thread_id, "th-1") # resumes the original thread
|
||||
self.assertIn("临时性错误中断", self.codex().started[1].prompt)
|
||||
self.codex().finish("", exit_code=1, tail="HTTP 503 Service Unavailable")
|
||||
self.h.tick()
|
||||
self.h.clock[0] += 301
|
||||
self.h.tick()
|
||||
self.assertEqual(len(self.codex().started), 3)
|
||||
self.codex().finish("", exit_code=1, tail="banner exchange failed")
|
||||
self.h.tick()
|
||||
self.assertEqual((self.p.state_of(iid), self.p.assignee_of(iid)), ("Waiting", "kappiee"))
|
||||
self.assertIn("不再自动重试", self.p.texts(iid)[-1])
|
||||
self.assertEqual(self.h.rec(iid)["retries"], 0)
|
||||
|
||||
def test_logic_failure_on_fresh_thread_goes_waiting(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("", exit_code=2, thread=None, tail="panic: bad config")
|
||||
self.h.tick()
|
||||
self.assertEqual(self.p.state_of(iid), "Waiting")
|
||||
self.assertIn("类型=crash，run=GW-1-1，exit=2，重试 0/2", self.p.texts(iid)[-1])
|
||||
self.assertIn("panic: bad config", self.p.texts(iid)[-1])
|
||||
|
||||
def test_resume_failure_rebuilds_thread_with_handoff(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("[progress] half way")
|
||||
self.h.tick() # finishes round 1, starts round 2 resuming th-1
|
||||
self.assertEqual(self.codex().started[1].thread_id, "th-1")
|
||||
self.codex().finish("", exit_code=1, thread=None, tail="thread not found")
|
||||
self.h.tick() # round 2 fails -> rebuild immediately
|
||||
run = self.codex().started[2]
|
||||
self.assertIsNone(run.thread_id)
|
||||
self.assertIn("## 交接摘要", run.prompt)
|
||||
self.assertIn("续线程失败", run.prompt)
|
||||
self.assertIn("[progress] half way", run.prompt)
|
||||
self.assertIn("新线程（重建：续线程失败（crash））", self.p.texts(iid)[-1])
|
||||
|
||||
def test_thread_rebuilt_after_max_rounds(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
for _ in range(3):
|
||||
self.codex().finish("[progress] more")
|
||||
self.h.tick()
|
||||
runs = self.codex().started
|
||||
self.assertEqual([r.thread_id for r in runs], [None, "th-1", "th-1", None])
|
||||
self.assertIn("线程已续 3 轮（上限 3）", runs[3].prompt)
|
||||
self.assertIn("progress, progress, progress", runs[3].prompt)
|
||||
self.assertEqual(self.h.rec(iid)["thread_rounds"], 1)
|
||||
|
||||
def test_timeout_kills_and_waits(self):
|
||||
iid = self.p.add_issue(1, labels=["budget:30m"])
|
||||
self.h.tick()
|
||||
self.h.clock[0] += 29 * 60
|
||||
self.h.tick()
|
||||
self.assertEqual(self.codex().killed, [])
|
||||
self.h.clock[0] += 2 * 60
|
||||
self.h.tick()
|
||||
self.assertEqual(self.codex().killed, ["GW-1-1"])
|
||||
self.assertEqual(self.p.state_of(iid), "Waiting")
|
||||
self.assertIn("类型=timeout", self.p.texts(iid)[-1])
|
||||
self.assertIn("0.5 h", self.p.texts(iid)[-1])
|
||||
|
||||
def test_serialize_group_and_concurrency(self):
|
||||
a = self.p.add_issue(1, assignee="oregon", labels=["needs-game"])
|
||||
b = self.p.add_issue(2, assignee="oregon", labels=["needs-game"])
|
||||
c = self.p.add_issue(3, assignee="oregon")
|
||||
d = self.p.add_issue(4, assignee="oregon")
|
||||
self.h.tick()
|
||||
started = [r.run_id for r in self.h.execs[("oregon", "claude-code")].started]
|
||||
self.assertEqual(started, ["GW-1-1", "GW-3-1"]) # 2 slots; GW-2 waits for the game group
|
||||
self.assertEqual([self.p.state_of(x) for x in (a, b, c, d)], ["In Progress", "Todo", "In Progress", "Todo"])
|
||||
|
||||
def test_label_routing(self):
|
||||
self.p.add_issue(1, labels=["model:astra"])
|
||||
self.p.add_issue(2, assignee="oregon", labels=["cap:gpu"]) # oregon lacks the capability
|
||||
self.h.tick()
|
||||
run = self.codex().started[0]
|
||||
self.assertEqual((run.model, run.effort), ("astra", "high"))
|
||||
self.assertNotIn(("oregon", "claude-code"), self.h.execs)
|
||||
|
||||
def test_human_moving_item_during_run_is_respected(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.p.issues[iid]["state"] = "s-done"
|
||||
self.codex().finish("[review] ok")
|
||||
self.h.tick()
|
||||
self.assertEqual(self.p.state_of(iid), "Done")
|
||||
self.assertIn("[review] ok", self.p.texts(iid)[-1])
|
||||
|
||||
def test_post_failure_keeps_run_in_flight(self):
|
||||
iid = self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
self.codex().finish("[review] ok")
|
||||
orig = self.p.post_comment
|
||||
self.p.post_comment = lambda *a: (_ for _ in ()).throw(RuntimeError("plane down"))
|
||||
self.h.tick()
|
||||
self.assertIn("inflight", self.h.rec(iid))
|
||||
self.p.post_comment = orig
|
||||
self.h.tick()
|
||||
self.assertNotIn("inflight", self.h.rec(iid))
|
||||
self.assertEqual(self.p.state_of(iid), "Review")
|
||||
|
||||
def test_heartbeat_written(self):
|
||||
self.p.add_issue(1)
|
||||
self.h.tick()
|
||||
import json, pathlib
|
||||
hb = json.loads((pathlib.Path(self.h.tmp.name) / "heartbeat.json").read_text())
|
||||
self.assertEqual(hb["inflight"][0]["run"], "GW-1-1")
|
||||
self.assertEqual(hb["errors"], [])
|
||||
|
||||
|
||||
class EnvFailureTest(unittest.TestCase):
|
||||
def test_env_failure_pauses_group_until_item_leaves_waiting(self):
|
||||
data = config_data()
|
||||
data["projects"]["GWENT"]["retry"]["env_patterns"] = ["foreground window lost"]
|
||||
h = Harness(data)
|
||||
a = h.plane.add_issue(1, labels=["needs-game"])
|
||||
b = h.plane.add_issue(2, labels=["needs-game"])
|
||||
h.tick()
|
||||
ex = h.execs[("win10", "codex")]
|
||||
ex.finish("", exit_code=1, tail="foreground window lost")
|
||||
h.tick()
|
||||
self.assertEqual(h.plane.state_of(a), "Waiting")
|
||||
self.assertEqual(h.plane.state_of(b), "Todo") # group paused
|
||||
self.assertIn("暂停派发", h.plane.texts(a)[-1])
|
||||
h.plane.issues[a]["state"] = "s-done"
|
||||
h.tick()
|
||||
self.assertEqual(h.plane.state_of(b), "In Progress")
|
||||
h.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user