Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick

Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching
marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run
directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same
run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot.
Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped
for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
jj.wang
2026-10-21 11:54:40 +00:00
parent dbe305103c
commit 4f45d368ad
8 changed files with 357 additions and 50 deletions
+27 -2
View File
@@ -6,6 +6,7 @@ from agentd.config import Config
from agentd.core import Agentd
from agentd.executors import ExecResult
from agentd.store import Store
from agentd.transports import TransportError
STATES = {"Backlog": "s-bl", "Todo": "s-todo", "In Progress": "s-run", "Waiting": "s-wait", "Review": "s-rev", "Done": "s-done"}
MEMBERS = {"window10": "u-w10", "oregon": "u-or", "kappiee": "u-k"}
@@ -89,15 +90,39 @@ class FakeExecutor:
self.resume = resume
self.started, self.killed, self.done = [], [], {}
self.fail_starts, self.fail_kills = 0, 0
self.lost_starts = 0 # the run launches but the relay reports a failure (e.g. timeout)
self.fail_probes = self.fail_polls = 0
self.dirs = {} # run_dir -> files on the "terminal"
self.polls = []
def handle(self, run):
return {"run_dir": "/runs/" + (run.dir_name or run.run_id), "run_id": run.run_id, "thread_id": run.thread_id}
def start(self, run):
if self.fail_starts:
self.fail_starts -= 1
raise RuntimeError("relay down")
raise TransportError("relay down")
h = self.handle(run)
if "pid.txt" in self.dirs.get(h["run_dir"], ()):
raise TransportError("HASPID")
self.started.append(run)
return {"run_dir": "/runs/" + run.run_id, "run_id": run.run_id, "thread_id": run.thread_id}
self.dirs[h["run_dir"]] = {"launch.pid", "pid.txt"}
if self.lost_starts:
self.lost_starts -= 1
raise TransportError("relay timeout after 150s")
return h
def probe(self, handle):
if self.fail_probes:
self.fail_probes -= 1
raise TransportError("relay down")
return sorted(self.dirs.get(handle["run_dir"], ()))
def poll(self, handle):
self.polls.append(handle["run_id"])
if self.fail_polls:
self.fail_polls -= 1
raise TransportError("relay down")
return "finished" if handle["run_id"] in self.done else "running"
def collect(self, handle):