Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick

Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching
marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run
directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same
run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot.
Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped
for the rest of the tick (heartbeat.down) while other terminals carry on.

tests/fakes.py

@@ -6,6 +6,7 @@ from agentd.config import Config
 from agentd.core import Agentd
 from agentd.executors import ExecResult
 from agentd.store import Store
+from agentd.transports import TransportError
 
 STATES = {"Backlog": "s-bl", "Todo": "s-todo", "In Progress": "s-run", "Waiting": "s-wait", "Review": "s-rev", "Done": "s-done"}
 MEMBERS = {"window10": "u-w10", "oregon": "u-or", "kappiee": "u-k"}
@@ -89,15 +90,39 @@ class FakeExecutor:
         self.resume = resume
         self.started, self.killed, self.done = [], [], {}
         self.fail_starts, self.fail_kills = 0, 0
+        self.lost_starts = 0           # the run launches but the relay reports a failure (e.g. timeout)
+        self.fail_probes = self.fail_polls = 0
+        self.dirs = {}                 # run_dir -> files on the "terminal"
+        self.polls = []
+
+    def handle(self, run):
+        return {"run_dir": "/runs/" + (run.dir_name or run.run_id), "run_id": run.run_id, "thread_id": run.thread_id}
 
     def start(self, run):
         if self.fail_starts:
             self.fail_starts -= 1
-            raise RuntimeError("relay down")
+            raise TransportError("relay down")
+        h = self.handle(run)
+        if "pid.txt" in self.dirs.get(h["run_dir"], ()):
+            raise TransportError("HASPID")
         self.started.append(run)
-        return {"run_dir": "/runs/" + run.run_id, "run_id": run.run_id, "thread_id": run.thread_id}
+        self.dirs[h["run_dir"]] = {"launch.pid", "pid.txt"}
+        if self.lost_starts:
+            self.lost_starts -= 1
+            raise TransportError("relay timeout after 150s")
+        return h
+
+    def probe(self, handle):
+        if self.fail_probes:
+            self.fail_probes -= 1
+            raise TransportError("relay down")
+        return sorted(self.dirs.get(handle["run_dir"], ()))
 
     def poll(self, handle):
+        self.polls.append(handle["run_id"])
+        if self.fail_polls:
+            self.fail_polls -= 1
+            raise TransportError("relay down")
         return "finished" if handle["run_id"] in self.done else "running"
 
     def collect(self, handle):

tests/test_core.py

@@ -1,3 +1,4 @@
+import json
 import unittest
 
 from tests.fakes import Harness, config_data
@@ -337,10 +338,13 @@ class CoreTest(unittest.TestCase):
         saves = []
         orig = self.h.d.store.save
         self.h.d.store.save = lambda data: (saves.append(sorted(
-            r["inflight"]["run_id"] for r in data["projects"]["GWENT"]["issues"].values() if r.get("inflight"))), orig(data))
+            ("L" if r.get("launching") else "") + (r.get("inflight") or r.get("launching"))["run_id"]
+            for r in data["projects"]["GWENT"]["issues"].values() if r.get("inflight") or r.get("launching"))), orig(data))
         self.h.tick()
-        # one save per launched run (a crash mid-tick cannot orphan it), then the end-of-tick save
-        self.assertEqual(saves, [["GW-1-1"], ["GW-1-1", "GW-2-1"], ["GW-1-1", "GW-2-1"]])
+        # the launching marker is on disk before each launch call, the run right after it (a crash mid-tick cannot
+        # orphan it or start it twice), then the end-of-tick save
+        self.assertEqual(saves, [["LGW-1-1"], ["GW-1-1"], ["GW-1-1", "LGW-2-1"], ["GW-1-1", "GW-2-1"],
+                                 ["GW-1-1", "GW-2-1"]])
 
 class EnvFailureTest(unittest.TestCase):
     def test_env_failure_pauses_group_until_item_leaves_waiting(self):
@@ -391,3 +395,113 @@ class TierDefaultTest(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class DoubleStartGuardTest(unittest.TestCase):
+    """A launch whose outcome is unknown is probed on the next tick: adopted if it exists, relaunched in a fresh
+    directory if not; never started twice in the same directory."""
+    def setUp(self):
+        self.h = Harness()
+        self.p = self.h.plane
+
+    def tearDown(self):
+        self.h.close()
+
+    def codex(self):
+        return self.h.execs[("win10", "codex")]
+
+    def test_lost_launch_is_adopted_not_restarted(self):
+        iid = self.p.add_issue(1)
+        self.h.d.executor("win10", "codex").lost_starts = 1
+        self.h.tick()                                   # run starts, but the relay reports a timeout
+        rec = self.h.rec(iid)
+        self.assertNotIn("inflight", rec)
+        self.assertEqual((rec["launching"]["run_id"], rec["launching"]["handle"]["run_dir"]), ("GW-1-1", "/runs/GW-1-1"))
+        self.assertEqual(self.p.state_of(iid), "Todo")
+        self.h.tick()                                   # probe finds launch.pid/pid.txt -> adopt
+        self.assertEqual(len(self.codex().started), 1)
+        rec = self.h.rec(iid)
+        self.assertNotIn("launching", rec)
+        self.assertEqual((rec["inflight"]["run_id"], rec["runs"]), ("GW-1-1", 1))
+        self.assertEqual(self.p.state_of(iid), "In Progress")
+        self.assertIn("启动未确认、已接管", self.p.texts(iid)[-1])
+        self.codex().finish("[review] ok")
+        self.h.tick()
+        self.assertEqual(self.p.state_of(iid), "Review")
+
+    def test_failed_launch_relaunches_same_run_id_in_fresh_dir(self):
+        iid = self.p.add_issue(1)
+        self.codex_ex = self.h.d.executor("win10", "codex")
+        self.codex_ex.fail_starts = 1
+        self.h.tick()                                   # nothing launched
+        self.assertIn("launching", self.h.rec(iid))
+        self.h.tick()                                   # probe: nothing -> relaunch as GW-1-1 in GW-1-1-r2
+        run = self.codex().started[-1]
+        self.assertEqual((run.run_id, run.dir_name), ("GW-1-1", "GW-1-1-r2"))
+        rec = self.h.rec(iid)
+        self.assertEqual(rec["inflight"]["handle"]["run_dir"], "/runs/GW-1-1-r2")
+        self.assertNotIn("relaunch", rec)
+        self.assertIn("重新启动（目录 GW-1-1-r2）", self.p.texts(iid)[-1])
+
+    def test_unprobed_marker_holds_the_terminal_slot(self):
+        a = self.p.add_issue(1)
+        b = self.p.add_issue(2)
+        ex = self.h.d.executor("win10", "codex")
+        ex.lost_starts = 1
+        self.h.tick()
+        self.assertEqual([r.run_id for r in ex.started], ["GW-1-1"])   # GW-2 not started next to a maybe-live run
+        ex.fail_probes = 1
+        self.h.tick()                                   # relay down: marker kept, win10 skipped, GW-2 still waits
+        self.assertIn("launching", self.h.rec(a))
+        self.assertEqual(len(ex.started), 1)
+        self.assertEqual(self.h.d.store.load()["projects"]["GWENT"]["issues"].get(b), None)
+        self.h.tick()                                   # relay back: adopt GW-1-1; concurrency 1 keeps GW-2 waiting
+        self.assertEqual(self.h.rec(a)["inflight"]["run_id"], "GW-1-1")
+        self.assertEqual(len(ex.started), 1)
+
+    def test_marker_dropped_when_item_moved_away(self):
+        iid = self.p.add_issue(1)
+        self.h.d.executor("win10", "codex").fail_starts = 1
+        self.h.tick()
+        self.p.issues[iid]["state"] = "s-bl"            # a person parks it
+        self.h.tick()
+        rec = self.h.rec(iid)
+        self.assertNotIn("launching", rec)
+        self.assertNotIn("relaunch", rec)
+        self.assertEqual(self.codex().started, [])
+
+    def test_never_restarts_into_a_dir_with_a_pid_file(self):
+        iid = self.p.add_issue(1)
+        ex = self.h.d.executor("win10", "codex")
+        ex.dirs["/runs/GW-1-1"] = {"pid.txt"}         # leftover of a launch the state does not know about
+        self.h.tick()                                   # HASPID -> marker
+        self.assertIn("launching", self.h.rec(iid))
+        self.h.tick()                                   # probe finds pid.txt -> adopt instead of wiping
+        self.assertEqual(self.h.rec(iid)["inflight"]["handle"]["run_dir"], "/runs/GW-1-1")
+        self.assertEqual(ex.started, [])
+
+
+class RelayDownTest(unittest.TestCase):
+    def test_down_terminal_is_skipped_and_others_continue(self):
+        data = config_data()
+        data["terminals"]["win10"]["concurrency"] = 2
+        h = Harness(data)
+        w1, w2 = h.plane.add_issue(1), h.plane.add_issue(2)
+        o = h.plane.add_issue(3, assignee="oregon")
+        h.tick()
+        win, ore = h.execs[("win10", "codex")], h.execs[("oregon", "claude-code")]
+        self.assertEqual(len(win.started), 2)
+        ore.finish("[review] done")
+        w3 = h.plane.add_issue(4)
+        win.fail_polls, win.polls[:] = 1, []
+        h.tick()
+        self.assertEqual(len(win.polls), 1)             # second win10 run not polled once the relay failed
+        self.assertEqual(h.plane.state_of(o), "Review")  # oregon carried on
+        self.assertEqual(h.plane.state_of(w3), "Todo")  # nothing new started on win10
+        hb = json.loads((h.d.store.dir / "heartbeat.json").read_text())
+        self.assertEqual((hb["down"], hb["errors"]), (["win10"], []))
+        self.assertTrue(all("inflight" in h.rec(i) for i in (w1, w2)))
+        win.finish("[review] w")
+        h.tick()                                        # relay back: polled again
+        self.assertIn("win10", {r["terminal"] for r in json.loads((h.d.store.dir / "heartbeat.json").read_text())["inflight"]})
+        h.close()

tests/test_posix_remote.py

@@ -203,3 +203,17 @@ class PosixRemoteTest(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+    def test_probe_dir_name_and_pid_guard(self):
+        ex = make_executor(self.cfg, "mac", "shell", self.t)
+        run = Run("GW-5-1", self.cfg.terminals["mac"]["repo"], "", script="sleep 0.2", dir_name="GW-5-1-r2")
+        self.assertEqual(ex.probe(ex.handle(run)), [])
+        h = ex.start(run)
+        self.assertEqual((os.path.basename(h["run_dir"]), h["task"]), ("GW-5-1-r2", "agentd-GW-5-1-r2"))
+        self.assertTrue(wait(ex, h))
+        self.assertEqual(ex.probe(h), ["exit.txt", "launch.pid", "pid.txt"])
+        n = len(self.t.calls)
+        with self.assertRaises(AssertionError):        # LocalBash: marker LAUNCHED= missing (HASPID)
+            ex.start(run)
+        self.assertIn("HASPID", self.t.calls[n])
+        self.assertTrue(os.path.exists(os.path.join(h["run_dir"], "exit.txt")))   # nothing wiped

tests/test_transports.py

@@ -120,3 +120,30 @@ class StoreTest(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class GuardTest(unittest.TestCase):
+    def test_local_clean_refuses_dir_with_pid_file(self):
+        from agentd.transports import TransportError
+        with tempfile.TemporaryDirectory() as tmp:
+            t = LocalTransport(Config(config_data()), "oregon")
+            (Path(tmp) / "prompt.md").write_text("x")
+            t.mkdir(tmp, clean=True)
+            self.assertEqual(t.probe(tmp, ("exit.txt", "pid.txt")), [])
+            (Path(tmp) / "pid.txt").write_text("1")
+            with self.assertRaises(TransportError):
+                t.mkdir(tmp, clean=True)
+            self.assertEqual(t.probe(tmp, ("exit.txt", "pid.txt")), ["pid.txt"])
+
+    def test_windows_probe_and_clean_guard(self):
+        from agentd.transports import TransportError
+        cfg = config.load(ROOT / "config" / "agentd.example.yaml")
+        t = WindowsWarpgateTransport(cfg, "win10")
+        scripts = []
+        t.ps = lambda s, timeout=150: (scripts.append(s), "PROBE=launch.pid,pid.txt\n")[1]
+        self.assertEqual(t.probe("C:\\r\\GW-1-1", ("exit.txt", "launch.pid", "pid.txt")), ["launch.pid", "pid.txt"])
+        self.assertIn("Join-Path $d $_", scripts[-1])
+        t.ps = lambda s, timeout=150: (scripts.append(s), "HASPID\n")[1]
+        with self.assertRaises(TransportError):
+            t.mkdir("C:\\r\\GW-1-1", clean=True)
+        self.assertIn("'pid.txt'", scripts[-1])