kappiee/plane-agentd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scheduling core, state store, CLI and unit tests

Browse Source 
One tick polls in-flight runs, posts results, classifies the first line and
moves state/assignee; failures get a structured note with transient retry and
backoff, timeouts are killed and handed over, resume failures and long threads
rebuild with a handoff summary. Then the lowest Todo items are dispatched within
terminal concurrency and serialize groups, and a heartbeat is written.
CLI: tick, new, show, comment, state, assign, smoke, check-config.
This commit is contained in:
jj.wang
2026-10-21 07:04:32 +00:00
parent 9535122c99
commit 31850c4a81
11 changed files with 1426 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agentd/core.py
+512
View File
@@ -0,0 +1,512 @@
"""Scheduling core. One tick per project:
(a) poll in-flight runs; on finish post the result, classify its first line and move state/assignee;
on failure post a structured note and retry (transient) or hand over (Waiting); kill on timeout;
(b) start the lowest-sequence Todo items whose terminal has a free slot and whose serialize group is idle;
(c) write the heartbeat.
Everything is deterministic; models only run inside executors."""
import os
import re
import time
from collections import Counter
from pathlib import Path
from . import ROOT
from .executors import ExecResult, Run, make_executor
from .executors.base import fill
from .plane import Plane, html_to_text, text_to_html
from .store import Store
from .transports import make_transport
TAGS = ("review", "question", "blocked", "progress")
WORKER = "[worker]"
DEFAULT_TRANSIENT = [
r"time[sd]?[ -]?out",
r"connection (refused|reset)",
r"banner exchange",
r"\b429\b|too many requests|rate.?limit",
r"\b50[0-4]\b|internal server error|bad gateway|service unavailable",
r"stream (disconnected|error)",
]
DEFAULT_INTRO = ("你是 {terminal} 上的 {executor}，仓库 {repo}。先读仓库根目录 AGENTS.md。下面是看板协作协议"
"（看板是 Plane，工作项编号 {prefix}-k），然后是分派给你的工作项。"
"你的最后一条消息会被原样贴回工作项评论，首行必须是 [review]/[question]/[blocked]/[progress] 之一。")
OUTPUT_RULE = "输出要求：最后一条消息首行为 [review]/[question]/[blocked]/[progress] 之一，内容按协议；不要等待人工回复。"
# ---------------------------------------------------------------- pure helpers (unit-tested)
def classify_first_line(text):
"""Tag of the first non-empty line (`[review]` etc.), tolerating markdown emphasis; None if untagged."""
first = next((l.strip() for l in (text or "").splitlines() if l.strip()), "")
m = re.match(r"\[(\w+)\]", first.lstrip("*_#> `").lower())
return m.group(1) if m and m.group(1) in TAGS else None
def matches_any(text, patterns):
return any(re.search(p, text or "", re.I) for p in patterns)
def classify_failure(result, timed_out, retry_cfg):
"""-> (kind, transient). kind in timeout | env | crash | logic (design §4)."""
if timed_out:
return "timeout", False
text = "%s\n%s" % (result.tail or "", result.last_message or "")
if matches_any(text, retry_cfg.get("env_patterns", [])):
return "env", False
kind = "logic" if result.exit_code == 0 else "crash"
return kind, matches_any(text, retry_cfg.get("transient_patterns", DEFAULT_TRANSIENT))
def parse_budget(label):
"""`budget:2h` / `budget:90m` / `budget:1.5h` -> hours."""
m = re.fullmatch(r"budget:(\d+(?:\.\d+)?)([hm]?)", label)
if not m:
return None
value = float(m.group(1))
return value / 60 if m.group(2) == "m" else value
def handoff_summary(rec, reason):
s = "本工作项此前已进行 %d 轮，执行器线程已重建（原因：%s）。各轮首行标签：%s。" % (
rec.get("runs", 0), reason or "上一线程不可续", ", ".join(rec.get("tags", [])) or "无")
if rec.get("last_message"):
s += "\n\n上一轮最后消息：\n" + rec["last_message"]
return s
def build_prompt(key, issue, comments, seen, resumed, protocol="", intro="", handoff=None, retry_note=None):
"""Fresh thread: intro + protocol + work item + handoff + all comments.
Resumed thread: only comments not seen before. `[worker]` comments are always skipped."""
parts = []
if resumed:
parts.append("工作项 %s 有新消息，请在同一线程里继续：" % key)
new = [c for c in comments if c.get("id") not in seen]
else:
if intro:
parts.append(intro)
if protocol:
parts.append("## 协议\n" + protocol.strip())
parts.append("## 工作项 %s: %s\n\n%s" % (key, issue.get("name", ""), html_to_text(issue.get("description_html"))))
if handoff:
parts.append("## 交接摘要\n" + handoff)
new = comments
shown = 0
for c in new:
body = html_to_text(c.get("comment_html"))
if not body or body.startswith(WORKER):
continue
parts.append("### 评论（%s）\n%s" % (c.get("created_at", ""), body))
shown += 1
if retry_note:
parts.append(retry_note)
elif resumed and not shown:
parts.append("（没有新评论；请从上次停下的地方继续推进。）")
parts.append(OUTPUT_RULE)
return "\n\n".join(parts)
def new_rec(seq):
return {"seq": seq, "runs": 0, "retries": 0, "tags": [], "seen_comments": [], "thread_rounds": 0}
# ---------------------------------------------------------------- per-project view for one tick
class Ctx:
def __init__(self, name, pcfg, reader):
self.name, self.cfg, self.reader = name, pcfg, reader
self.pid = pcfg["id"]
self.prefix = pcfg.get("key_prefix", name)
self.states = reader.states(self.pid)
self.labels = reader.labels(self.pid)
self.members = reader.members()
self.label_names = {v: k for k, v in self.labels.items()}
self.member_names = {v: k for k, v in self.members.items()}
self.issues = {i["id"]: i for i in reader.list_issues(self.pid)}
def state_id(self, key):
return self.states[self.cfg["states"][key]]
def key(self, seq):
return "%s-%s" % (self.prefix, seq)
class Agentd:
def __init__(self, cfg, store=None, plane_factory=None, executor_factory=None, transport_factory=None, now=time.time):
self.cfg = cfg
self.store = store or Store(cfg.state_dir)
self.now = now
self.log = self.store.log
self.plane_factory = plane_factory or (lambda term: Plane(
cfg.plane["base_url"], cfg.plane["workspace"], cfg.token(term), cfg.plane.get("timeout", 60)))
self.transport_factory = transport_factory or (lambda term: make_transport(cfg, term))
self.executor_factory = executor_factory or (lambda term, name: make_executor(cfg, term, name, self.transport(term)))
self._reset_caches()
def _reset_caches(self):
self._clients, self._transports, self._executors, self._ready = {}, {}, {}, {}
# ---------------------------------------------------------------- caches
def client(self, terminal):
if terminal not in self._clients:
self._clients[terminal] = self.plane_factory(terminal)
return self._clients[terminal]
def transport(self, terminal):
if terminal not in self._transports:
self._transports[terminal] = self.transport_factory(terminal)
return self._transports[terminal]
def executor(self, terminal, name):
if (terminal, name) not in self._executors:
self._executors[(terminal, name)] = self.executor_factory(terminal, name)
return self._executors[(terminal, name)]
def reader(self, pcfg):
"""Any identity with a token can read; prefer the project's own terminals."""
order = list(dict.fromkeys(list(pcfg["routing"]["by_assignee"].values()) + list(self.cfg.terminals)))
for term in order:
try:
return self.client(term)
except Exception:
continue
raise RuntimeError("no Plane token available for any terminal")
# ---------------------------------------------------------------- tick
def tick(self, project=None, only_issue=None):
with self.store.lock():
self._reset_caches()
data = self.store.load()
errors = []
for name in ([project] if project else list(self.cfg.projects)):
try:
self.tick_project(data, name, only_issue)
except Exception as exc:
errors.append("%s: %r" % (name, exc))
self.log("%s tick error: %r" % (name, exc))
finally:
self.store.save(data)
self.store.heartbeat({"errors": errors, "inflight": [
{"project": p, "run": r["inflight"]["run_id"], "terminal": r["inflight"]["terminal"]}
for p, ps in data["projects"].items() for r in ps["issues"].values() if r.get("inflight")]})
return data
def loop(self, interval):
from .store import Busy
while True:
try:
self.tick()
except Busy as exc:
self.log("skip: %s" % exc)
except Exception as exc:
self.log("tick error: %r" % (exc,))
time.sleep(interval)
def tick_project(self, data, name, only_issue=None):
pcfg = self.cfg.project(name)
pstate = data["projects"].setdefault(name, {"issues": {}})
ctx = Ctx(name, pcfg, self.reader(pcfg))
for iid, rec in list(pstate["issues"].items()):
if rec.get("inflight"):
try:
self.poll(ctx, data, iid, rec)
except Exception as exc:
self.log("%s poll error: %r" % (ctx.key(rec.get("seq")), exc))
self.pick(ctx, data, pstate, only_issue)
# ---------------------------------------------------------------- (a) poll / finish
def poll(self, ctx, data, iid, rec):
fl = rec["inflight"]
ex = self.executor(fl["terminal"], fl["executor"])
if ex.poll(fl["handle"]) == "running":
if self.now() < fl["deadline"]:
return
self.log("%s run %s exceeded its budget; killing" % (ctx.key(rec["seq"]), fl["run_id"]))
try:
ex.kill(fl["handle"])
except Exception as exc:
self.log("kill %s failed: %r" % (fl["run_id"], exc))
hours = (fl["deadline"] - fl["started"]) / 3600
result = ExecResult("", None, fl["handle"].get("thread_id"), [fl["handle"].get("run_dir", "")],
"超过时限 %.1f h 未结束，已结束任务进程。" % hours)
return self.finish(ctx, data, iid, rec, result, timed_out=True)
result = ex.collect(fl["handle"])
if hasattr(ex, "forget"):
ex.forget(fl["handle"])
self.finish(ctx, data, iid, rec, result)
def finish(self, ctx, data, iid, rec, result, timed_out=False):
"""Post first, then forget the in-flight run: if Plane fails, the next tick collects again."""
fl = rec["inflight"]
term, exname = fl["terminal"], fl["executor"]
client = self.client(term)
key = ctx.key(rec["seq"])
issue = ctx.issues.get(iid)
still_ours = issue is None or issue.get("state") == ctx.state_id("running")
message = (result.last_message or "").strip()
tag = classify_first_line(message)
ok = not timed_out and message and (tag or result.exit_code == 0)
if not ok:
return self.fail(ctx, data, iid, rec, client, result, timed_out, still_ours)
if tag is None:
message, tag = "[review]\n（未按协议标注首行，按 review 处理）\n\n" + message, "review"
body = message
limit = self.cfg.agentd.get("comment_max_chars", 6000)
if len(body) > limit:
where = result.log_paths[-1] if result.log_paths else fl["handle"].get("run_dir", "")
body = body[:limit] + "\n…（已截断，全文见 %s 上 %s）" % (term, where)
header = "[%s] 第 %d 轮结束（exit=%s，run=%s，终端 %s，日志 %s）" % (
exname, fl["round"], result.exit_code, fl["run_id"], term, fl["handle"].get("run_dir", ""))
self.post(ctx, client, iid, rec, header, body)
self.close_run(rec, result)
rec["tags"].append(tag)
rec.update(last_message=message[:4000], last_tag=tag, retries=0)
if still_ours:
self.guarded(key, self.transition, ctx, client, iid, tag, self.cfg.terminals[term]["plane_user"])
else:
self.log("%s was moved during the run; not changing its state" % key)
self.log("%s run %s finished exit=%s tag=%s" % (key, fl["run_id"], result.exit_code, tag))
def close_run(self, rec, result):
fl = rec.pop("inflight")
rec["last_exit"] = result.exit_code
if result.thread_id and self.executor(fl["terminal"], fl["executor"]).resume:
rec["thread_id"], rec["thread_on"] = result.thread_id, [fl["terminal"], fl["executor"]]
return fl
def guarded(self, key, fn, *args):
try:
fn(*args)
except Exception as exc:
self.log("%s %s failed: %r" % (key, fn.__name__, exc))
def fail(self, ctx, data, iid, rec, client, result, timed_out, still_ours):
fl, pcfg = rec["inflight"], ctx.cfg
key = ctx.key(rec["seq"])
retry_cfg = pcfg.get("retry", {})
max_n = retry_cfg.get("transient_max", 3)
backoff = retry_cfg.get("backoff_minutes", [2, 5, 10])
kind, transient = classify_failure(result, timed_out, retry_cfg)
n = rec.get("retries", 0)
who = pcfg.get("answerer") or pcfg["reviewer"]
if not still_ours:
plan, action = "none", "工作项已被移出 %s，不重试。" % pcfg["states"]["running"]
elif fl.get("resumed") and kind in ("crash", "logic") and not transient:
plan, action = "rebuild", "续线程失败，下一轮在新线程重跑（附交接摘要）。"
elif transient and n < max_n:
delay = backoff[min(n, len(backoff) - 1)]
n += 1
plan, action = "retry", "临时性错误，%g 分钟后自动重试（续原线程）。" % delay
else:
plan, action = "waiting", "不再自动重试，转 %s 并指派给 %s。" % (pcfg["states"]["waiting"], who)
if kind == "env" and fl.get("group"):
action += "串行组 %s 暂停派发，直到本项离开 %s。" % (fl["group"], pcfg["states"]["waiting"])
note = "%s 第 %d 轮失败：类型=%s%s，run=%s，exit=%s，重试 %d/%d。%s\n日志：%s" % (
WORKER, fl["round"], kind, "（临时性）" if transient else "", fl["run_id"], result.exit_code, n, max_n,
action, ", ".join(result.log_paths) or fl["handle"].get("run_dir", ""))
frame = result.tail or "（无输出）"
if result.last_message:
frame += "\n--- 最后消息\n" + result.last_message[:1500]
self.post(ctx, client, iid, rec, note, frame)
self.close_run(rec, result)
rec["last_tag"] = "failed:" + kind
if plan == "rebuild":
rec["rebuild"] = "续线程失败（%s）" % kind
rec.pop("thread_id", None)
rec.pop("thread_on", None)
rec["retry"] = {"at": self.now(), "reason": "rebuild"}
elif plan == "retry":
rec["retries"] = n
rec["retry"] = {"at": self.now() + delay * 60, "reason": kind}
elif plan == "waiting":
rec["retries"] = 0
if kind == "env" and fl.get("group"):
data["paused_groups"][fl["group"]] = {"project": ctx.name, "issue": iid, "key": key}
self.guarded(key, self.move, ctx, client, iid, "waiting", who)
self.log("%s run %s failed kind=%s transient=%s -> %s" % (key, fl["run_id"], kind, transient, plan))
def transition(self, ctx, client, iid, tag, terminal_user):
pcfg = ctx.cfg
if tag == "progress":
self.move(ctx, client, iid, "todo", terminal_user)
elif tag in ("question", "blocked"):
self.move(ctx, client, iid, "waiting", pcfg.get("answerer") or pcfg["reviewer"])
else:
self.move(ctx, client, iid, "review", pcfg["reviewer"])
def move(self, ctx, client, iid, state_key, user):
state, users = ctx.state_id(state_key), [ctx.members[user]]
client.set_state(ctx.pid, iid, state, users)
if iid in ctx.issues: # keep this tick's view consistent
ctx.issues[iid].update(state=state, assignees=users)
def post(self, ctx, client, iid, rec, text, pre=None):
c = client.post_comment(ctx.pid, iid, text_to_html(text, pre))
if isinstance(c, dict) and c.get("id"):
rec.setdefault("seen_comments", []).append(c["id"])
# ---------------------------------------------------------------- (b) pick / start
def pick(self, ctx, data, pstate, only_issue=None):
load, groups = Counter(), set()
for p in data["projects"].values():
for r in p["issues"].values():
fl = r.get("inflight")
if fl:
load[fl["terminal"]] += 1
if fl.get("group"):
groups.add(fl["group"])
for g, info in list(data["paused_groups"].items()):
if info["project"] == ctx.name:
issue = ctx.issues.get(info["issue"])
if issue is None or issue.get("state") != ctx.state_id("waiting"):
del data["paused_groups"][g]
self.log("serialize group %s resumed (%s left Waiting)" % (g, info.get("key")))
groups |= set(data["paused_groups"])
todo, running, now = ctx.state_id("todo"), ctx.state_id("running"), self.now()
cands = []
for iid, issue in ctx.issues.items():
if only_issue and iid != only_issue:
continue
rec = pstate["issues"].get(iid, {})
if rec.get("inflight"):
continue
if issue.get("state") == todo:
cands.append(issue)
elif rec.get("retry"):
if issue.get("state") != running:
rec.pop("retry")
elif rec["retry"]["at"] <= now:
cands.append(issue)
for issue in sorted(cands, key=lambda i: i.get("sequence_id", 0)):
route = self.route(ctx, issue)
if isinstance(route, str):
if route:
self.log("%s not dispatched: %s" % (ctx.key(issue.get("sequence_id")), route))
continue
term, group = route["terminal"], route["group"]
if load[term] >= self.cfg.terminals[term].get("concurrency", 1) or (group and group in groups):
continue
if not self.terminal_ready(term, load[term]):
continue
try:
self.start(ctx, pstate, issue, route)
except Exception as exc:
self.log("%s start failed on %s: %r" % (ctx.key(issue.get("sequence_id")), term, exc))
continue
load[term] += 1
if group:
groups.add(group)
def terminal_ready(self, term, our_load):
"""Optional guard against processes we did not start (e.g. a manual codex session)."""
if term not in self._ready:
names = self.cfg.terminals[term].get("busy_processes", [])
ready = True
for name in names:
count = self.transport(term).process_count(name)
if count is None:
self.log("%s unreachable; skipping" % term)
ready = False
elif count > 0 and our_load == 0:
self.log("%s busy (%d %s running that agentd did not start)" % (term, count, name))
ready = False
self._ready[term] = ready
return self._ready[term]
def route(self, ctx, issue):
"""Work item -> {terminal, executor, model, effort, tier, group, timeout_hours, script}; a str = not routable."""
pcfg, routing = ctx.cfg, ctx.cfg["routing"]
users = [ctx.member_names.get(a, a) for a in issue.get("assignees") or []]
term = next((routing["by_assignee"][u] for u in users if u in routing["by_assignee"]), None)
if term is None:
return "" # not assigned to a terminal: silently ignored
tcfg = self.cfg.terminals[term]
r = {"terminal": term, "executor": routing.get("default_executor", {}).get(term) or tcfg["executors"][0],
"model": None, "effort": None, "tier": None, "group": None, "script": None,
"timeout_hours": pcfg.get("timeout_hours", 4)}
caps = []
for label in [ctx.label_names.get(l, l) for l in issue.get("labels") or []]:
spec = routing.get("by_label", {}).get(label)
if spec:
for k, v in spec.items():
r["group" if k == "serialize_group" else k] = v
elif label.startswith("exec:") and label[5:] in self.cfg.executors:
r["executor"] = label[5:]
elif label.startswith("model:"):
r["model"] = label[6:]
elif label.startswith("effort:"):
r["effort"] = label[7:]
elif label.startswith("cap:"):
caps.append(label[4:])
elif parse_budget(label) is not None:
r["timeout_hours"] = parse_budget(label)
missing = [c for c in caps if c not in tcfg.get("capabilities", [])]
if missing:
return "terminal %s lacks capabilities %s" % (term, missing)
if r["executor"] not in tcfg["executors"]:
return "executor %s not available on %s" % (r["executor"], term)
ecfg = self.cfg.executors[r["executor"]]
r["model"] = r["model"] or (ecfg.get("models") or [None])[0]
r["effort"] = r["effort"] or ecfg.get("default_effort")
return r
def protocol(self, pcfg):
path = pcfg.get("protocol_file")
if not path:
return ""
p = Path(os.path.expanduser(path))
p = p if p.is_absolute() else ROOT / p
if not p.exists():
self.log("protocol file %s missing" % p)
return ""
return p.read_text(encoding="utf-8")
def start(self, ctx, pstate, issue, route):
iid, seq = issue["id"], issue["sequence_id"]
key = ctx.key(seq)
rec = pstate["issues"].setdefault(iid, new_rec(seq))
term, exname = route["terminal"], route["executor"]
tcfg, pcfg = self.cfg.terminals[term], ctx.cfg
ex = self.executor(term, exname)
is_retry = rec.pop("retry", None) is not None and issue.get("state") == ctx.state_id("running")
if not is_retry:
rec["retries"] = 0
k = rec["runs"] + 1
run_id = "%s-%d-%d" % (ctx.prefix, seq, k)
thread, reason = rec.get("thread_id"), rec.pop("rebuild", None)
max_rounds = pcfg.get("max_rounds_per_thread") or self.cfg.agentd.get("max_rounds_per_thread", 8)
if thread and not ex.resume:
thread = None
elif thread and rec.get("thread_on") != [term, exname]:
thread, reason = None, "换了终端/执行器"
elif thread and rec.get("thread_rounds", 0) >= max_rounds:
thread, reason = None, "线程已续 %d 轮（上限 %d）" % (rec["thread_rounds"], max_rounds)
handoff = handoff_summary(rec, reason) if not thread and rec["runs"] > 0 else None
retry_note = None
if is_retry and thread:
retry_note = "（上一轮因临时性错误中断，请在同一线程从中断处继续。）"
comments = ctx.reader.list_comments(ctx.pid, iid)
seen = set(rec.get("seen_comments", []))
intro = fill(ctx.cfg.get("intro", DEFAULT_INTRO),
dict(terminal=term, executor=exname, repo=tcfg["repo"], prefix=ctx.prefix, key=key))
prompt = build_prompt(key, issue, comments, seen, bool(thread), self.protocol(pcfg), intro, handoff, retry_note)
run = Run(run_id, tcfg["repo"], prompt, route["model"], route["effort"], route["tier"], thread, route["script"])
handle = ex.start(run)
now = self.now()
rec.update(runs=k, thread_rounds=rec.get("thread_rounds", 0) + 1 if thread else 1, inflight={
"run_id": run_id, "terminal": term, "executor": exname, "handle": handle, "round": k,
"started": now, "deadline": now + route["timeout_hours"] * 3600, "resumed": bool(thread),
"group": route["group"], "model": route["model"], "effort": route["effort"]})
if not thread:
rec.pop("thread_id", None)
rec.pop("thread_on", None)
rec["seen_comments"] = sorted(seen | {c["id"] for c in comments if c.get("id")})
rec["last_comment_at"] = max([c.get("created_at", "") for c in comments] + [rec.get("last_comment_at", "")])
client = self.client(term)
mode = ("续线程 %s" % thread[:12]) if thread else ("新线程（重建：%s）" % reason if reason else "新线程")
if is_retry:
mode += "，重试 %d" % rec.get("retries", 0)
detail = " / ".join(x for x in (exname, route["model"], route["effort"], route["tier"]) if x)
try:
self.move(ctx, client, iid, "running", tcfg["plane_user"])
self.post(ctx, client, iid, rec, "%s 第 %d 轮开始：%s @ %s，%s，run=%s。" % (WORKER, k, detail, term, mode, run_id))
except Exception as exc:
self.log("%s bookkeeping after start failed: %r" % (key, exc))
self.log("%s run %s started on %s (%s, %s)" % (key, run_id, term, detail, mode))