kappiee/plane-agentd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick

Browse Source 
Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching
marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run
directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same
run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot.
Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped
for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
jj.wang
2026-10-21 11:54:40 +00:00
parent dbe305103c
commit 4f45d368ad
8 changed files with 357 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agentd/core.py
+120 -39
View File
@@ -3,7 +3,13 @@
on failure post a structured note and retry (transient) or hand over (Waiting); kill on timeout;
(b) start the lowest-sequence Todo items whose terminal has a free slot and whose serialize group is idle;
(c) write the heartbeat.
Everything is deterministic; models only run inside executors."""
Everything is deterministic; models only run inside executors.
Double-start guard: before the launch call a `launching` marker (the planned run and its handle) is saved. If the
launch call fails or the process dies before the run is recorded, the next tick probes the run directory: launch
evidence (exit.txt / launch.pid / pid.txt) -> adopt the run; nothing -> relaunch with the same run id in a fresh
directory (`<run_id>-r2`, -r3 ...); probe unreachable -> keep the marker (it holds the terminal slot).
A terminal whose relay fails is skipped for the rest of the tick; other terminals carry on."""
import os
import re
import time
@@ -15,7 +21,7 @@ from .executors import ExecResult, Run, make_executor
from .executors.base import fill
from .plane import Plane, html_to_text, text_to_html
from .store import Store
from .transports import make_transport
from .transports import TransportError, make_transport
TAGS = ("review", "question", "blocked", "progress")
WORKER = "[worker]"
@@ -143,7 +149,7 @@ class Ctx:
class Agentd:
def __init__(self, cfg, store=None, plane_factory=None, executor_factory=None, transport_factory=None, now=time.time):
self.cfg = cfg
self.store = store or Store(cfg.state_dir)
self.store = store or Store(cfg.state_dir, log_dir=cfg.log_dir)
self.now = now
self.log = self.store.log
self.plane_factory = plane_factory or (lambda term: Plane(
@@ -154,6 +160,12 @@ class Agentd:
def _reset_caches(self):
self._clients, self._transports, self._executors, self._ready = {}, {}, {}, {}
self._down = set() # terminals whose relay failed during this tick
def mark_down(self, term, exc):
if term not in self._down:
self.log("%s unreachable, skipping it for the rest of this tick: %s" % (term, exc))
self._down.add(term)
# ---------------------------------------------------------------- caches
def client(self, terminal):
@@ -195,9 +207,11 @@ class Agentd:
self.log("%s tick error: %r" % (name, exc))
finally:
self.store.save(data)
self.store.heartbeat({"errors": errors, "inflight": [
self.store.heartbeat({"errors": errors, "down": sorted(self._down), "inflight": [
{"project": p, "run": r["inflight"]["run_id"], "terminal": r["inflight"]["terminal"]}
for p, ps in data["projects"].items() for r in ps["issues"].values() if r.get("inflight")]})
for p, ps in data["projects"].items() for r in ps["issues"].values() if r.get("inflight")],
"launching": [{"project": p, "run": r["launching"]["run_id"], "terminal": r["launching"]["terminal"]}
for p, ps in data["projects"].items() for r in ps["issues"].values() if r.get("launching")]})
return data
def loop(self, interval):
@@ -216,14 +230,43 @@ class Agentd:
pstate = data["projects"].setdefault(name, {"issues": {}})
ctx = Ctx(name, pcfg, self.reader(pcfg))
for iid, rec in list(pstate["issues"].items()):
if rec.get("inflight"):
try:
run = rec.get("inflight") or rec.get("launching")
if not run or run["terminal"] in self._down:
continue
try:
if rec.get("inflight"):
self.poll(ctx, data, iid, rec)
except Exception as exc:
self.log("%s poll error: %r" % (ctx.key(rec.get("seq")), exc))
self.store.save(data) # persist each round's outcome (thread id, retry) before touching the next
else:
self.resolve_launch(ctx, iid, rec)
except TransportError as exc:
self.mark_down(run["terminal"], exc)
except Exception as exc:
self.log("%s poll error: %r" % (ctx.key(rec.get("seq")), exc))
self.store.save(data) # persist each round's outcome (thread id, retry) before touching the next
self.pick(ctx, data, pstate, only_issue)
def resolve_launch(self, ctx, iid, rec):
"""A launch that was never confirmed (the launch call failed or agentd died mid-start): probe the run
directory and adopt the run if it exists; otherwise relaunch in a fresh directory, never wiping one."""
m = rec["launching"]
key = ctx.key(rec["seq"])
present = self.executor(m["terminal"], m["executor"]).probe(m["handle"]) # TransportError -> marker kept
if present:
self.log("%s run %s: unconfirmed launch found on %s (%s); adopting it" % (
key, m["run_id"], m["terminal"], ",".join(present)))
return self.launched(ctx, iid, rec, m, m["handle"], adopted=True)
rec.pop("launching")
issue = ctx.issues.get(iid)
state = issue and issue.get("state")
if state == ctx.state_id("todo") or (state == ctx.state_id("running") and rec.get("retry")):
rec["relaunch"] = {"run_id": m["run_id"], "attempt": m.get("attempt", 1) + 1}
self.log("%s run %s: nothing at %s; relaunching in %s-r%d" % (
key, m["run_id"], m["handle"].get("run_dir"), m["run_id"], rec["relaunch"]["attempt"]))
else:
rec.pop("relaunch", None)
self.log("%s run %s: nothing at %s and the item is no longer dispatchable; marker dropped" % (
key, m["run_id"], m["handle"].get("run_dir")))
# ---------------------------------------------------------------- (a) poll / finish
def poll(self, ctx, data, iid, rec):
fl = rec["inflight"]
@@ -404,7 +447,7 @@ class Agentd:
load, groups = Counter(), set()
for p in data["projects"].values():
for r in p["issues"].values():
fl = r.get("inflight")
fl = r.get("inflight") or r.get("launching") # an unconfirmed launch holds its slot
if fl:
load[fl["terminal"]] += 1
if fl.get("group"):
@@ -422,7 +465,7 @@ class Agentd:
if only_issue and iid != only_issue:
continue
rec = pstate["issues"].get(iid, {})
if rec.get("inflight"):
if rec.get("inflight") or rec.get("launching"):
continue
if issue.get("state") == todo:
cands.append(issue)
@@ -438,15 +481,21 @@ class Agentd:
self.log("%s not dispatched: %s" % (ctx.key(issue.get("sequence_id")), route))
continue
term, group = route["terminal"], route["group"]
if term in self._down:
continue
if load[term] >= self.cfg.terminals[term].get("concurrency", 1) or (group and group in groups):
continue
if not self.terminal_ready(term, load[term]):
continue
try:
self.start(ctx, pstate, issue, route)
self.start(ctx, data, pstate, issue, route)
except Exception as exc:
self.log("%s start failed on %s: %r" % (ctx.key(issue.get("sequence_id")), term, exc))
continue
if isinstance(exc, TransportError):
self.mark_down(term, exc)
if not pstate["issues"].get(issue["id"], {}).get("launching"):
continue
# the launch call failed but may have started the run: its marker keeps the slot until probed
self.store.save(data) # a launched run must be on disk before anything else can fail
load[term] += 1
if group:
@@ -454,14 +503,17 @@ class Agentd:
def terminal_ready(self, term, our_load):
"""Optional guard against processes we did not start (e.g. a manual codex session)."""
if term in self._down:
return False
if term not in self._ready:
names = self.cfg.terminals[term].get("busy_processes", [])
ready = True
for name in names:
count = self.transport(term).process_count(name)
if count is None:
self.log("%s unreachable; skipping" % term)
self.mark_down(term, "busy-process probe failed")
ready = False
break
elif count > 0 and our_load == 0:
self.log("%s busy (%d %s running that agentd did not start)" % (term, count, name))
ready = False
@@ -522,7 +574,7 @@ class Agentd:
return ""
return p.read_text(encoding="utf-8")
def start(self, ctx, pstate, issue, route):
def start(self, ctx, data, pstate, issue, route):
iid, seq = issue["id"], issue["sequence_id"]
key = ctx.key(seq)
rec = pstate["issues"].setdefault(iid, new_rec(seq))
@@ -552,25 +604,11 @@ class Agentd:
intro = fill(ctx.cfg.get("intro", DEFAULT_INTRO),
dict(terminal=term, executor=exname, repo=tcfg["repo"], prefix=ctx.prefix, key=key))
prompt = build_prompt(key, issue, comments, seen, bool(thread), self.protocol(pcfg), intro, handoff, retry_note)
run = Run(run_id, tcfg["repo"], prompt, route["model"], route["effort"], route["tier"], thread, route["script"])
handle = ex.start(run)
rec.pop("retry", None)
rec.pop("rebuild", None)
if not is_retry:
rec["retries"] = 0
now = self.now()
# transient retries continue the same logical round: they do not use up max_rounds_per_thread
rounds = (rec.get("thread_rounds", 0) + (0 if is_retry else 1)) if thread else 1
rec.update(runs=k, thread_rounds=rounds, inflight={
"run_id": run_id, "terminal": term, "executor": exname, "handle": handle, "round": k,
"started": now, "deadline": now + route["timeout_hours"] * 3600, "resumed": bool(thread),
"group": route["group"], "model": route["model"], "effort": route["effort"]})
if not thread:
rec.pop("thread_id", None)
rec.pop("thread_on", None)
rec["seen_comments"] = sorted(seen | {c["id"] for c in comments if c.get("id")})
rec["last_comment_at"] = max([c.get("created_at", "") for c in comments] + [rec.get("last_comment_at", "")])
client = self.client(term)
relaunch = rec.get("relaunch") or {}
attempt = relaunch.get("attempt", 1) if relaunch.get("run_id") == run_id else 1
dir_name = run_id if attempt == 1 else "%s-r%d" % (run_id, attempt)
run = Run(run_id, tcfg["repo"], prompt, route["model"], route["effort"], route["tier"], thread, route["script"],
dir_name=dir_name)
if thread:
mode = "续线程 %s" % thread[:12]
elif reason:
@@ -579,10 +617,53 @@ class Agentd:
mode = "新线程（附交接摘要）" if handoff else "新线程"
if is_retry:
mode += "，重试 %d" % rec.get("retries", 0)
detail = " / ".join(x for x in (exname, route["model"], route["effort"], route["tier"]) if x)
if attempt > 1:
mode += "，重新启动（目录 %s）" % dir_name
# Everything the bookkeeping needs once the run is known to exist; saved before the launch call so that a
# launch whose outcome is unknown (relay timeout, crash) is probed and adopted instead of started twice.
plan = {"run_id": run_id, "terminal": term, "executor": exname, "attempt": attempt, "at": self.now(),
"handle": ex.handle(run), "round": k, "budget_s": route["timeout_hours"] * 3600,
"resumed": bool(thread), "group": route["group"], "model": route["model"], "effort": route["effort"],
"is_retry": is_retry,
# transient retries continue the same logical round: they do not use up max_rounds_per_thread
"thread_rounds": (rec.get("thread_rounds", 0) + (0 if is_retry else 1)) if thread else 1,
"seen": sorted(seen | {c["id"] for c in comments if c.get("id")}),
"last_comment_at": max([c.get("created_at", "") for c in comments] + [rec.get("last_comment_at", "")]),
"mode": mode,
"detail": " / ".join(x for x in (exname, route["model"], route["effort"], route["tier"]) if x)}
rec["launching"] = plan
self.store.save(data)
handle = ex.start(run) # raises -> the marker stays and the next tick probes the run directory
self.launched(ctx, iid, rec, plan, handle)
def launched(self, ctx, iid, rec, plan, handle, adopted=False):
"""The run exists on the terminal: record it as in flight, then move the item and post the start note."""
key, term = ctx.key(rec["seq"]), plan["terminal"]
rec.pop("launching", None)
rec.pop("relaunch", None)
rec.pop("retry", None)
rec.pop("rebuild", None)
if not plan["is_retry"]:
rec["retries"] = 0
rec.update(runs=plan["round"], thread_rounds=plan["thread_rounds"], inflight={
"run_id": plan["run_id"], "terminal": term, "executor": plan["executor"], "handle": handle,
"round": plan["round"], "started": plan["at"], "deadline": plan["at"] + plan["budget_s"],
"resumed": plan["resumed"], "group": plan["group"], "model": plan["model"], "effort": plan["effort"]})
if not plan["resumed"]:
rec.pop("thread_id", None)
rec.pop("thread_on", None)
rec["seen_comments"] = plan["seen"]
rec["last_comment_at"] = plan["last_comment_at"]
mode = plan["mode"] + ("，启动未确认、已接管" if adopted else "")
issue = ctx.issues.get(iid)
client = self.client(term)
try:
self.move(ctx, client, iid, "running", tcfg["plane_user"])
self.post(ctx, client, iid, rec, "%s 第 %d 轮开始：%s @ %s，%s，run=%s。" % (WORKER, k, detail, term, mode, run_id))
# an adopted run whose item a person moved meanwhile is left alone: poll cancels it next tick
if issue is not None and issue.get("state") in (ctx.state_id("todo"), ctx.state_id("running")):
self.move(ctx, client, iid, "running", self.cfg.terminals[term]["plane_user"])
self.post(ctx, client, iid, rec, "%s 第 %d 轮开始：%s @ %s，%s，run=%s。" % (
WORKER, plan["round"], plan["detail"], term, mode, plan["run_id"]))
except Exception as exc:
self.log("%s bookkeeping after start failed: %r" % (key, exc))
self.log("%s run %s started on %s (%s, %s)" % (key, run_id, term, detail, mode))
self.log("%s run %s %s on %s (%s, %s)" % (key, plan["run_id"], "adopted" if adopted else "started", term,
plan["detail"], mode))