kappiee/plane-agentd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick

Browse Source 
Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching
marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run
directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same
run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot.
Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped
for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
jj.wang
2026-10-21 11:54:40 +00:00
parent dbe305103c
commit 4f45d368ad
8 changed files with 357 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_core.py
+117 -3
View File
@@ -1,3 +1,4 @@
import json
import unittest
from tests.fakes import Harness, config_data
@@ -337,10 +338,13 @@ class CoreTest(unittest.TestCase):
saves = []
orig = self.h.d.store.save
self.h.d.store.save = lambda data: (saves.append(sorted(
r["inflight"]["run_id"] for r in data["projects"]["GWENT"]["issues"].values() if r.get("inflight"))), orig(data))
("L" if r.get("launching") else "") + (r.get("inflight") or r.get("launching"))["run_id"]
for r in data["projects"]["GWENT"]["issues"].values() if r.get("inflight") or r.get("launching"))), orig(data))
self.h.tick()
# one save per launched run (a crash mid-tick cannot orphan it), then the end-of-tick save
self.assertEqual(saves, [["GW-1-1"], ["GW-1-1", "GW-2-1"], ["GW-1-1", "GW-2-1"]])
# the launching marker is on disk before each launch call, the run right after it (a crash mid-tick cannot
# orphan it or start it twice), then the end-of-tick save
self.assertEqual(saves, [["LGW-1-1"], ["GW-1-1"], ["GW-1-1", "LGW-2-1"], ["GW-1-1", "GW-2-1"],
["GW-1-1", "GW-2-1"]])
class EnvFailureTest(unittest.TestCase):
def test_env_failure_pauses_group_until_item_leaves_waiting(self):
@@ -391,3 +395,113 @@ class TierDefaultTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
class DoubleStartGuardTest(unittest.TestCase):
"""A launch whose outcome is unknown is probed on the next tick: adopted if it exists, relaunched in a fresh
directory if not; never started twice in the same directory."""
def setUp(self):
self.h = Harness()
self.p = self.h.plane
def tearDown(self):
self.h.close()
def codex(self):
return self.h.execs[("win10", "codex")]
def test_lost_launch_is_adopted_not_restarted(self):
iid = self.p.add_issue(1)
self.h.d.executor("win10", "codex").lost_starts = 1
self.h.tick() # run starts, but the relay reports a timeout
rec = self.h.rec(iid)
self.assertNotIn("inflight", rec)
self.assertEqual((rec["launching"]["run_id"], rec["launching"]["handle"]["run_dir"]), ("GW-1-1", "/runs/GW-1-1"))
self.assertEqual(self.p.state_of(iid), "Todo")
self.h.tick() # probe finds launch.pid/pid.txt -> adopt
self.assertEqual(len(self.codex().started), 1)
rec = self.h.rec(iid)
self.assertNotIn("launching", rec)
self.assertEqual((rec["inflight"]["run_id"], rec["runs"]), ("GW-1-1", 1))
self.assertEqual(self.p.state_of(iid), "In Progress")
self.assertIn("启动未确认、已接管", self.p.texts(iid)[-1])
self.codex().finish("[review] ok")
self.h.tick()
self.assertEqual(self.p.state_of(iid), "Review")
def test_failed_launch_relaunches_same_run_id_in_fresh_dir(self):
iid = self.p.add_issue(1)
self.codex_ex = self.h.d.executor("win10", "codex")
self.codex_ex.fail_starts = 1
self.h.tick() # nothing launched
self.assertIn("launching", self.h.rec(iid))
self.h.tick() # probe: nothing -> relaunch as GW-1-1 in GW-1-1-r2
run = self.codex().started[-1]
self.assertEqual((run.run_id, run.dir_name), ("GW-1-1", "GW-1-1-r2"))
rec = self.h.rec(iid)
self.assertEqual(rec["inflight"]["handle"]["run_dir"], "/runs/GW-1-1-r2")
self.assertNotIn("relaunch", rec)
self.assertIn("重新启动（目录 GW-1-1-r2）", self.p.texts(iid)[-1])
def test_unprobed_marker_holds_the_terminal_slot(self):
a = self.p.add_issue(1)
b = self.p.add_issue(2)
ex = self.h.d.executor("win10", "codex")
ex.lost_starts = 1
self.h.tick()
self.assertEqual([r.run_id for r in ex.started], ["GW-1-1"]) # GW-2 not started next to a maybe-live run
ex.fail_probes = 1
self.h.tick() # relay down: marker kept, win10 skipped, GW-2 still waits
self.assertIn("launching", self.h.rec(a))
self.assertEqual(len(ex.started), 1)
self.assertEqual(self.h.d.store.load()["projects"]["GWENT"]["issues"].get(b), None)
self.h.tick() # relay back: adopt GW-1-1; concurrency 1 keeps GW-2 waiting
self.assertEqual(self.h.rec(a)["inflight"]["run_id"], "GW-1-1")
self.assertEqual(len(ex.started), 1)
def test_marker_dropped_when_item_moved_away(self):
iid = self.p.add_issue(1)
self.h.d.executor("win10", "codex").fail_starts = 1
self.h.tick()
self.p.issues[iid]["state"] = "s-bl" # a person parks it
self.h.tick()
rec = self.h.rec(iid)
self.assertNotIn("launching", rec)
self.assertNotIn("relaunch", rec)
self.assertEqual(self.codex().started, [])
def test_never_restarts_into_a_dir_with_a_pid_file(self):
iid = self.p.add_issue(1)
ex = self.h.d.executor("win10", "codex")
ex.dirs["/runs/GW-1-1"] = {"pid.txt"} # leftover of a launch the state does not know about
self.h.tick() # HASPID -> marker
self.assertIn("launching", self.h.rec(iid))
self.h.tick() # probe finds pid.txt -> adopt instead of wiping
self.assertEqual(self.h.rec(iid)["inflight"]["handle"]["run_dir"], "/runs/GW-1-1")
self.assertEqual(ex.started, [])
class RelayDownTest(unittest.TestCase):
def test_down_terminal_is_skipped_and_others_continue(self):
data = config_data()
data["terminals"]["win10"]["concurrency"] = 2
h = Harness(data)
w1, w2 = h.plane.add_issue(1), h.plane.add_issue(2)
o = h.plane.add_issue(3, assignee="oregon")
h.tick()
win, ore = h.execs[("win10", "codex")], h.execs[("oregon", "claude-code")]
self.assertEqual(len(win.started), 2)
ore.finish("[review] done")
w3 = h.plane.add_issue(4)
win.fail_polls, win.polls[:] = 1, []
h.tick()
self.assertEqual(len(win.polls), 1) # second win10 run not polled once the relay failed
self.assertEqual(h.plane.state_of(o), "Review") # oregon carried on
self.assertEqual(h.plane.state_of(w3), "Todo") # nothing new started on win10
hb = json.loads((h.d.store.dir / "heartbeat.json").read_text())
self.assertEqual((hb["down"], hb["errors"]), (["win10"], []))
self.assertTrue(all("inflight" in h.rec(i) for i in (w1, w2)))
win.finish("[review] w")
h.tick() # relay back: polled again
self.assertIn("win10", {r["terminal"] for r in json.loads((h.d.store.dir / "heartbeat.json").read_text())["inflight"]})
h.close()