Compare commits

..

9 Commits

Author SHA1 Message Date
jj.wang 94a09e3bf0 Runbook: PAD-12 rollout record (keys, ping, smoke latencies, rollback) 2026-10-22 03:12:11 +00:00
jj.wang 5192e04088 PAD-12: per-terminal transport (direct Tailscale SSH or Warpgate, pre-session fallback, pinned host keys, agent key from Vault), one-call win10 launch and one-call collect, remote call timings, agentd ping 2026-10-22 02:59:37 +00:00
jj.wang 83bc0f243c Docs: usage guide for dispatching work through Plane (routing, labels, replies, cancel, phone app notes) 2026-10-22 02:01:33 +00:00
jj.wang ccbc68369c Runbook: exact API recovery time for the 2026-10-22 Plane restart 2026-10-22 01:31:43 +00:00
jj.wang 709d65ef83 Docs: Plane-side webhook enabled (allowlist applied on VM 506, v2 deliveries verified, latency record) 2026-10-22 01:29:47 +00:00
jj.wang 57887a90f2 Docs: static-port NAT path for the LXC, dispatcher back on LXC, Plane webhook allowlist notes 2026-10-21 15:45:54 +00:00
jj.wang 1b4d965baf Runbook: rollback record, dispatcher back on oregon (LXC to NAS path via DERP) 2026-10-21 14:22:29 +00:00
jj.wang 4ba6d0ca5f Docs: webhook receiver (runbook, design §10, deploy-lxc §8), PAD-7 rollout record
Runbook: service, endpoint, signature, logs, secret rotation, disabling,
heartbeat skipped reasons, new failure rows. Design: webhook trigger design,
notify hook, Plane SSRF guard (WEBHOOK_ALLOWED_IPS). Deploy: install steps.
Record: synthetic-delivery acceptance and measured latencies.
2026-10-21 13:52:19 +00:00
jj.wang f6c4221f0e Board protocol: no git writes on the win10 checkout 2026-10-21 13:33:06 +00:00
24 changed files with 2062 additions and 139 deletions
+2 -1
View File
@@ -2,11 +2,12 @@
Plane 看板驱动的通用 agent 调度服务:在 Plane 建工作项并指派给某个终端,agentd 在该终端起对应的 CLI agent(Codex / Claude Code / 脚本)执行,把结果贴回工作项并流转状态。调度是确定性代码;模型只出现在执行器和总指挥里。 Plane 看板驱动的通用 agent 调度服务:在 Plane 建工作项并指派给某个终端,agentd 在该终端起对应的 CLI agent(Codex / Claude Code / 脚本)执行,把结果贴回工作项并流转状态。调度是确定性代码;模型只出现在执行器和总指挥里。
- 使用说明(在 Plane 上怎么派活、回答、取消,标签和默认值):`docs/usage.md`
- 设计:`docs/design.md`(架构、契约、失败策略、配置、阶段) - 设计:`docs/design.md`(架构、契约、失败策略、配置、阶段)
- 首个租户:GWENT 项目(看板 https://plane.kappiee.com/kappiee/projects/79cb5a60-aa4f-4138-9864-1dd70426e665/issues/ ) - 首个租户:GWENT 项目(看板 https://plane.kappiee.com/kappiee/projects/79cb5a60-aa4f-4138-9864-1dd70426e665/issues/ )
- 凭据全部经 Agent Vault 注入内存,仓库和配置里只有键名。 - 凭据全部经 Agent Vault 注入内存,仓库和配置里只有键名。
## 使用 ## 使用
- 契约与扩展:`docs/executor-contract.md`;安装、cron、日志、故障:`docs/runbook.md`;LXC 部署(调度器 2026-10-21 起跑在 `agentd-01`):`docs/deploy-lxc.md` - 契约与扩展:`docs/executor-contract.md`;安装、cron、日志、故障:`docs/runbook.md`;LXC 部署(调度器 2026-10-21 起跑在 `agentd-01`):`docs/deploy-lxc.md`
- 入口:`cli/agentd tick|new|show|comment|state|assign|smoke|check-config`(在 `agent-access.py exec --secret-env …` 下运行) - 入口:`cli/agentd tick|new|show|comment|state|assign|smoke|ping|check-config`(在 `agent-access.py exec --secret-env …` 下运行;`ping` 验证到终端的 direct-ssh / warpgate-ssh 路径)
- 测试:`python3 -m unittest`(离线,不访问 Plane/win10) - 测试:`python3 -m unittest`(离线,不访问 Plane/win10)
+40
View File
@@ -9,6 +9,7 @@ from . import config as config_mod
from .core import Agentd from .core import Agentd
from .plane import Plane, brief_to_html, html_to_text, text_to_html from .plane import Plane, brief_to_html, html_to_text, text_to_html
from .store import Busy from .store import Busy
from .transports import CallLog, TransportError, direct_ssh, make_transport
def identity(cfg, wanted=None): def identity(cfg, wanted=None):
@@ -159,6 +160,31 @@ def cmd_smoke(cfg, a):
sys.exit(0 if ok else 1) sys.exit(0 if ok else 1)
def cmd_ping(cfg, a):
"""Trivial command on a terminal, timed per call (verifies direct-ssh / warpgate-ssh after a deployment). The
direct-ssh key comes from the environment like the Plane tokens (agent-access.py exec --secret-env)."""
if a.terminal not in cfg.terminals:
sys.exit("unknown terminal %s" % a.terminal)
t = cfg.terminals[a.terminal]
if (a.transport or t["transport"]) == "local":
sys.exit("%s is a local terminal: nothing to ping" % a.terminal)
if a.transport == "direct-ssh" and "ssh" not in t:
sys.exit("terminals.%s has no ssh section" % a.terminal)
log = CallLog(event=lambda m: print("event: " + m, flush=True), echo=True)
tr = make_transport(cfg, a.terminal, log=log, force=a.transport)
ok = True
for i in range(a.count):
t0 = time.monotonic()
try:
pong = tr.ping()
except TransportError as exc:
ok = False
print("ping %d: FAILED after %d ms: %s" % (i + 1, (time.monotonic() - t0) * 1000, exc), flush=True)
continue
print("ping %d: %d ms %s" % (i + 1, (time.monotonic() - t0) * 1000, pong), flush=True)
sys.exit(0 if ok else 1)
def cmd_check_config(cfg, a): def cmd_check_config(cfg, a):
print("config %s is valid" % cfg.path) print("config %s is valid" % cfg.path)
print("projects: %s" % ", ".join(cfg.projects)) print("projects: %s" % ", ".join(cfg.projects))
@@ -166,6 +192,13 @@ def cmd_check_config(cfg, a):
env = cfg.token_env(name) env = cfg.token_env(name)
print("terminal %-10s %-13s %-8s executors=%s token %s: %s" % ( print("terminal %-10s %-13s %-8s executors=%s token %s: %s" % (
name, t["transport"], t["os"], ",".join(t["executors"]), env, "set" if os.environ.get(env) else "MISSING")) name, t["transport"], t["os"], ",".join(t["executors"]), env, "set" if os.environ.get(env) else "MISSING"))
if t["transport"] == "direct-ssh":
s = t["ssh"]
print(" direct-ssh %s@%s:%s, fallback %s" % (s["user"], s["host"], s.get("port", 22), t.get("fallback", "none")))
if cfg.uses_direct_ssh():
print("direct-ssh key %s: %s; ssh_dir %s" % (
cfg.ssh_key_env, "set" if direct_ssh.key_present(cfg.ssh_key_env) else "MISSING (direct terminals use their fallback)",
cfg.ssh_dir))
print("cron/exec secret args: %s" % " ".join(cfg.secret_env_args())) print("cron/exec secret args: %s" % " ".join(cfg.secret_env_args()))
if cfg.webhook: if cfg.webhook:
k = cfg.webhook["secret_key"] k = cfg.webhook["secret_key"]
@@ -221,6 +254,12 @@ def parser():
s.add_argument("--port", type=int, help="port (default: webhook.port)") s.add_argument("--port", type=int, help="port (default: webhook.port)")
s.set_defaults(fn=cmd_webhook_serve) s.set_defaults(fn=cmd_webhook_serve)
sub.add_parser("check-config", help="validate the config and list token env vars").set_defaults(fn=cmd_check_config) sub.add_parser("check-config", help="validate the config and list token env vars").set_defaults(fn=cmd_check_config)
s = sub.add_parser("ping", help="time a trivial command on a terminal (direct-ssh needs the key env var, see check-config)")
s.add_argument("terminal")
s.add_argument("--transport", choices=["direct-ssh", "warpgate-ssh"],
help="force one channel, no fallback (default: the terminal's transport with its fallback)")
s.add_argument("-n", "--count", type=int, default=3, help="calls (default 3: the 2nd+ reuse the SSH connection)")
s.set_defaults(fn=cmd_ping)
return ap return ap
@@ -230,4 +269,5 @@ def main(argv=None):
cfg = config_mod.load(a.config) cfg = config_mod.load(a.config)
except config_mod.ConfigError as exc: except config_mod.ConfigError as exc:
sys.exit(str(exc)) sys.exit(str(exc))
direct_ssh.take_key(cfg.ssh_key_env) # out of os.environ before any child process starts
a.fn(cfg, a) a.fn(cfg, a)
+93 -17
View File
@@ -2,8 +2,10 @@
The config only ever holds Vault *key names* (`token_key`). The token itself is read from the environment The config only ever holds Vault *key names* (`token_key`). The token itself is read from the environment
variable of the same name, which `agent-access.py exec --secret-env KEY=KEY` injects into the process.""" variable of the same name, which `agent-access.py exec --secret-env KEY=KEY` injects into the process."""
import base64
import json import json
import os import os
import struct
from pathlib import Path from pathlib import Path
import yaml import yaml
@@ -11,6 +13,7 @@ import yaml
from . import ROOT from . import ROOT
SCHEMA_PATH = ROOT / "config" / "schema.json" SCHEMA_PATH = ROOT / "config" / "schema.json"
DEFAULT_SSH_KEY = "AGENTD_SSH_KEY" # env var (== Vault key name) carrying the direct-ssh private key
DEFAULT_PATHS = (ROOT / "config" / "agentd.local.yaml", ROOT / "config" / "agentd.yaml") DEFAULT_PATHS = (ROOT / "config" / "agentd.local.yaml", ROOT / "config" / "agentd.yaml")
@@ -83,6 +86,48 @@ def schema_errors(data, schema):
for e in validator.iter_errors(data)] for e in validator.iter_errors(data)]
def host_key_error(value):
"""None if `value` is a usable pinned host key ('ssh-ed25519 <base64>' whose blob names the same type)."""
parts = str(value).split()
if len(parts) < 2 or parts[0] != "ssh-ed25519":
return "must be 'ssh-ed25519 <base64>' (the terminal's /etc/ssh/ssh_host_ed25519_key.pub, verified out of band)"
try:
blob = base64.b64decode(parts[1], validate=True)
n = struct.unpack(">I", blob[:4])[0]
ok = blob[4:4 + n] == b"ssh-ed25519" and len(blob) == 4 + n + 4 + 32
except (ValueError, struct.error):
ok = False
return None if ok else "the base64 part is not an ssh-ed25519 public key (copy/paste damage?)"
def ssh_dir_of(data):
d = (data.get("agentd") or {}).get("ssh_dir")
if d:
p = Path(os.path.expanduser(d))
return p if p.is_absolute() else ROOT / p
s = Path(os.path.expanduser((data.get("agentd") or {}).get("state_dir", "state")))
return (s if s.is_absolute() else ROOT / s) / "ssh"
def remote_errors(data, tname, t, kind):
"""Rules every remote transport (warpgate-ssh, direct-ssh, and the fallback channel) shares."""
errs = []
if t.get("os") == "windows":
if t.get("launch") != "schtasks":
errs.append("terminals.%s: %s on windows needs launch=schtasks" % (tname, kind))
if kind == "warpgate-ssh" and t.get("target", tname) != "win10":
errs.append("terminals.%s: the Windows helper scripts only reach target 'win10' for now" % tname)
else:
if t.get("launch") != "nohup":
errs.append("terminals.%s: %s on %s needs launch=nohup" % (tname, kind, t.get("os")))
for key in ("repo", "run_root"):
if key in t and not str(t[key]).startswith("/"):
errs.append("terminals.%s.%s: must be an absolute path on the remote (no ~)" % (tname, key))
if kind == "warpgate-ssh" and not data.get("vault", {}).get("config"):
errs.append("terminals.%s: remote POSIX terminals need vault.config (relay identity)" % tname)
return errs
def semantic_errors(data): def semantic_errors(data):
"""Cross-references the schema cannot express.""" """Cross-references the schema cannot express."""
errs = [] errs = []
@@ -91,21 +136,36 @@ def semantic_errors(data):
for ex in t.get("executors", []): for ex in t.get("executors", []):
if ex not in execs: if ex not in execs:
errs.append("terminals.%s: unknown executor '%s'" % (tname, ex)) errs.append("terminals.%s: unknown executor '%s'" % (tname, ex))
if t.get("transport") == "warpgate-ssh" and t.get("os") == "windows": kind = t.get("transport")
if t.get("launch") != "schtasks": if kind == "warpgate-ssh":
errs.append("terminals.%s: warpgate-ssh on windows needs launch=schtasks" % tname) errs += remote_errors(data, tname, t, kind)
if t.get("target", tname) != "win10": elif kind == "direct-ssh":
errs.append("terminals.%s: the Windows helper scripts only reach target 'win10' for now" % tname) errs += remote_errors(data, tname, t, kind)
elif t.get("transport") == "warpgate-ssh": s = t.get("ssh")
if t.get("launch") != "nohup": if not s:
errs.append("terminals.%s: warpgate-ssh on %s needs launch=nohup" % (tname, t.get("os"))) errs.append("terminals.%s: direct-ssh needs ssh: {host, user, port, host_key}" % tname)
for key in ("repo", "run_root"): else:
if key in t and not str(t[key]).startswith("/"): if not s.get("host") or str(s["host"]).startswith("-") or not s.get("user"):
errs.append("terminals.%s.%s: must be an absolute path on the remote (no ~)" % (tname, key)) errs.append("terminals.%s.ssh: host and user must be non-empty (host not starting with '-')" % tname)
if not data.get("vault", {}).get("config"): if s.get("port", 22) > 65535:
errs.append("terminals.%s: remote POSIX terminals need vault.config (relay identity)" % tname) errs.append("terminals.%s.ssh.port: %r > 65535" % (tname, s["port"]))
if t.get("transport") == "local" and t.get("os") != "linux": why = host_key_error(s.get("host_key", ""))
if why:
errs.append("terminals.%s.ssh.host_key: %s" % (tname, why))
if t.get("fallback") == "warpgate-ssh":
if "target" not in t:
errs.append("terminals.%s: fallback warpgate-ssh needs the Warpgate 'target'" % tname)
elif t.get("os") == "windows" and t["target"] != "win10":
errs.append("terminals.%s: the Windows helper scripts (fallback) only reach target 'win10' for now" % tname)
if t.get("os") != "windows" and not data.get("vault", {}).get("config"):
errs.append("terminals.%s: fallback warpgate-ssh needs vault.config (relay identity)" % tname)
elif t.get("fallback"):
errs.append("terminals.%s: 'fallback' only applies to transport direct-ssh" % tname)
if kind == "local" and t.get("os") != "linux":
errs.append("terminals.%s: local transport supports only os=linux" % tname) errs.append("terminals.%s: local transport supports only os=linux" % tname)
ctl = ssh_dir_of(data) # "/<pin8>-<40-char %C>", plus ".<16 chars>" while ssh sets the socket up
if any(t.get("transport") == "direct-ssh" for t in terms.values()) and len(str(ctl)) + 1 + 9 + 40 + 17 > 104:
errs.append("agentd.ssh_dir: %s is too long for the ControlMaster socket path (UNIX limit 104)" % ctl)
for ename, e in execs.items(): for ename, e in execs.items():
adapter = e.get("adapter", ename) adapter = e.get("adapter", ename)
if adapter not in ("codex", "claude-code", "shell"): if adapter not in ("codex", "claude-code", "shell"):
@@ -156,6 +216,19 @@ class Config:
p = Path(os.path.expanduser(self.agentd.get("state_dir", "state"))) p = Path(os.path.expanduser(self.agentd.get("state_dir", "state")))
return p if p.is_absolute() else ROOT / p return p if p.is_absolute() else ROOT / p
@property
def ssh_dir(self):
"""direct-ssh ControlMaster sockets + generated known_hosts (default <state_dir>/ssh)."""
return ssh_dir_of(self.data)
@property
def ssh_key_env(self):
"""Env var (== Vault key name) holding the direct-ssh private key."""
return self.agentd.get("ssh_key") or DEFAULT_SSH_KEY
def uses_direct_ssh(self):
return any(t["transport"] == "direct-ssh" for t in self.terminals.values())
def helpers_dir(self, terminal): def helpers_dir(self, terminal):
d = self.terminals[terminal].get("helpers_dir") or self.agentd.get("helpers_dir") or "tools/win10" d = self.terminals[terminal].get("helpers_dir") or self.agentd.get("helpers_dir") or "tools/win10"
p = Path(os.path.expanduser(d)) p = Path(os.path.expanduser(d))
@@ -185,13 +258,16 @@ class Config:
return keys return keys
def scrubbed_env(self): def scrubbed_env(self):
"""os.environ without the secrets above, for every child process agentd starts.""" """os.environ without the secrets above, for every child process agentd starts. The direct-ssh key variable
drop = self.secret_keys() is always dropped, even with no direct-ssh terminal (a rollback may leave the cron script injecting it)."""
drop = self.secret_keys() | {self.ssh_key_env}
return {k: v for k, v in os.environ.items() if k not in drop} return {k: v for k, v in os.environ.items() if k not in drop}
def secret_env_args(self): def secret_env_args(self):
"""`--secret-env` arguments for agent-access.py exec, one per distinct token key.""" """`--secret-env` arguments for agent-access.py exec: one per distinct token key, plus the direct-ssh key."""
keys = sorted({t["token_key"] for t in self.terminals.values()}) keys = sorted({t["token_key"] for t in self.terminals.values()})
if self.uses_direct_ssh():
keys.append(self.ssh_key_env)
return ["--secret-env %s=%s" % (k, k) for k in keys] return ["--secret-env %s=%s" % (k, k) for k in keys]
def project(self, name): def project(self, name):
+4 -2
View File
@@ -21,7 +21,7 @@ from .executors import ExecResult, Run, make_executor
from .executors.base import fill from .executors.base import fill
from .plane import Plane, html_to_text, text_to_html from .plane import Plane, html_to_text, text_to_html
from .store import Store from .store import Store
from .transports import TransportError, make_transport from .transports import CallLog, TransportError, make_transport
TAGS = ("review", "question", "blocked", "progress") TAGS = ("review", "question", "blocked", "progress")
WORKER = "[worker]" WORKER = "[worker]"
@@ -152,9 +152,11 @@ class Agentd:
self.store = store or Store(cfg.state_dir, log_dir=cfg.log_dir) self.store = store or Store(cfg.state_dir, log_dir=cfg.log_dir)
self.now = now self.now = now
self.log = self.store.log self.log = self.store.log
# one line per remote call (terminal, channel, operation, ms, rc) -> remote.log; fallback/host-key -> agentd.log
self.calllog = CallLog(self.store.log_dir / "remote.log", self.log)
self.plane_factory = plane_factory or (lambda term: Plane( self.plane_factory = plane_factory or (lambda term: Plane(
cfg.plane["base_url"], cfg.plane["workspace"], cfg.token(term), cfg.plane.get("timeout", 60))) cfg.plane["base_url"], cfg.plane["workspace"], cfg.token(term), cfg.plane.get("timeout", 60)))
self.transport_factory = transport_factory or (lambda term: make_transport(cfg, term)) self.transport_factory = transport_factory or (lambda term: make_transport(cfg, term, log=self.calllog))
self.executor_factory = executor_factory or (lambda term, name: make_executor(cfg, term, name, self.transport(term))) self.executor_factory = executor_factory or (lambda term, name: make_executor(cfg, term, name, self.transport(term)))
self._reset_caches() self._reset_caches()
+16 -4
View File
@@ -6,6 +6,8 @@ import re
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import List, Optional from typing import List, Optional
from ..transports import fetch_each
@dataclass @dataclass
class Run: class Run:
@@ -55,6 +57,11 @@ class Executor:
raise NotImplementedError raise NotImplementedError
def parse_exit(text):
m = re.search(r"exit=(-?\d+)", text or "")
return int(m.group(1)) if m else None
def fill(template, params): def fill(template, params):
"""Replace {name} placeholders that exist in params; leave anything else untouched.""" """Replace {name} placeholders that exist in params; leave anything else untouched."""
return re.sub(r"\{(\w+)\}", lambda m: str(params[m.group(1)]) if m.group(1) in params else m.group(0), template) return re.sub(r"\{(\w+)\}", lambda m: str(params[m.group(1)]) if m.group(1) in params else m.group(0), template)
@@ -101,7 +108,7 @@ class RunnerExecutor(Executor):
run_dir, task = h["run_dir"], h["task"] run_dir, task = h["run_dir"], h["task"]
script_name, script = t.runner(run_dir, run.repo, self.command(run, run_dir), self.stdout_name) script_name, script = t.runner(run_dir, run.repo, self.command(run, run_dir), self.stdout_name)
files = {"prompt.md": run.prompt, script_name: script} files = {"prompt.md": run.prompt, script_name: script}
if hasattr(t, "start_run"): # remote POSIX: stage + launch in one relay round trip if hasattr(t, "start_run"): # remote: stage + launch in one call (direct-ssh; POSIX also on the relay)
t.start_run(task, run_dir, files, script_name) t.start_run(task, run_dir, files, script_name)
else: else:
t.mkdir(run_dir, clean=True) t.mkdir(run_dir, clean=True)
@@ -121,9 +128,14 @@ class RunnerExecutor(Executor):
return "finished" if text and "exit=" in text else "running" return "finished" if text and "exit=" in text else "running"
def exit_code(self, handle): def exit_code(self, handle):
text = self.transport.read_text(self.transport.join(handle["run_dir"], "exit.txt")) or "" return parse_exit(self.transport.read_text(self.transport.join(handle["run_dir"], "exit.txt")))
m = re.search(r"exit=(-?\d+)", text)
return int(m.group(1)) if m else None def fetch(self, handle, ops):
"""Reads of the run directory for `collect`: one call on remote transports (see transports.fetch)."""
t = self.transport
if hasattr(t, "fetch"):
return t.fetch(handle["run_dir"], ops)
return fetch_each(t, handle["run_dir"], ops)
def kill(self, handle): def kill(self, handle):
self.transport.kill(handle["task"], handle["run_dir"]) self.transport.kill(handle["task"], handle["run_dir"])
+6 -5
View File
@@ -2,7 +2,7 @@
resume with `--resume <session_id>`. The JSON result carries `result` (last message) and `session_id`.""" resume with `--resume <session_id>`. The JSON result carries `result` (last message) and `session_id`."""
import json import json
from .base import ExecResult, RunnerExecutor from .base import ExecResult, RunnerExecutor, parse_exit
def parse_output(text): def parse_output(text):
@@ -26,13 +26,14 @@ def parse_output(text):
class ClaudeCodeExecutor(RunnerExecutor): class ClaudeCodeExecutor(RunnerExecutor):
stdout_name = "out.json" stdout_name = "out.json"
COLLECT = [("read", "exit.txt"), ("read", "out.json"), ("tail", "stderr.txt", 5)]
def collect(self, handle): def collect(self, handle):
t, d = self.transport, handle["run_dir"] exit_text, out, tail = self.fetch(handle, self.COLLECT) # one remote call
obj = parse_output(t.read_text(t.join(d, "out.json"))) obj = parse_output(out)
err = bool(obj.get("is_error")) or obj.get("subtype", "success") != "success" err = bool(obj.get("is_error")) or obj.get("subtype", "success") != "success"
last = "" if err else (obj.get("result") or "") last = "" if err else (obj.get("result") or "")
tail = self.stderr_tail(handle)
if err: if err:
tail = ("%s %s" % (obj.get("subtype", ""), str(obj.get("result", ""))[:400]) + "\n" + tail).strip() tail = ("%s %s" % (obj.get("subtype", ""), str(obj.get("result", ""))[:400]) + "\n" + tail).strip()
return ExecResult(last.strip(), self.exit_code(handle), obj.get("session_id") or handle.get("thread_id"), return ExecResult(last.strip(), parse_exit(exit_text), obj.get("session_id") or handle.get("thread_id"),
self.logs(handle, "out.json", "stderr.txt"), tail) self.logs(handle, "out.json", "stderr.txt"), tail)
+6 -6
View File
@@ -2,22 +2,22 @@
Thread id comes from the `thread.started` event in events.jsonl; the last message from last.md.""" Thread id comes from the `thread.started` event in events.jsonl; the last message from last.md."""
import json import json
from .base import ExecResult, RunnerExecutor from .base import ExecResult, RunnerExecutor, parse_exit
class CodexExecutor(RunnerExecutor): class CodexExecutor(RunnerExecutor):
stdout_name = "events.jsonl" stdout_name = "events.jsonl"
COLLECT = [("read", "exit.txt"), ("read", "last.md"), ("grep", "events.jsonl", "thread.started"),
("tail", "stderr.txt", 5), ("tail", "events.jsonl", 1)]
def collect(self, handle): def collect(self, handle):
t, d = self.transport, handle["run_dir"] exit_text, last, line, err_tail, last_event = self.fetch(handle, self.COLLECT) # one remote call
last = t.read_text(t.join(d, "last.md")) or ""
thread = handle.get("thread_id") thread = handle.get("thread_id")
line = t.grep_first(t.join(d, "events.jsonl"), "thread.started")
if line: if line:
try: try:
thread = json.loads(line).get("thread_id") or thread thread = json.loads(line).get("thread_id") or thread
except ValueError: except ValueError:
pass pass
tail = (self.stderr_tail(handle) + "\n" + t.tail(t.join(d, "events.jsonl"), 1)[-600:]).strip() tail = (err_tail + "\n" + last_event[-600:]).strip()
return ExecResult(last.strip(), self.exit_code(handle), thread, return ExecResult((last or "").strip(), parse_exit(exit_text), thread,
self.logs(handle, "events.jsonl", "stderr.txt", "last.md"), tail) self.logs(handle, "events.jsonl", "stderr.txt", "last.md"), tail)
+7 -6
View File
@@ -1,6 +1,6 @@
"""Shell/script executor: runs `{script}` with the prompt on stdin. The last message is `last.md` in the run """Shell/script executor: runs `{script}` with the prompt on stdin. The last message is `last.md` in the run
directory if the script wrote it (its path is passed as {out}), else the tail of stdout. No threads.""" directory if the script wrote it (its path is passed as {out}), else the tail of stdout. No threads."""
from .base import ExecResult, RunnerExecutor from .base import ExecResult, RunnerExecutor, parse_exit
class ShellExecutor(RunnerExecutor): class ShellExecutor(RunnerExecutor):
@@ -10,10 +10,11 @@ class ShellExecutor(RunnerExecutor):
def resume(self): def resume(self):
return False return False
COLLECT = [("read", "exit.txt"), ("read", "last.md"), ("read", "stdout.txt"), ("tail", "stderr.txt", 5)]
def collect(self, handle): def collect(self, handle):
t, d = self.transport, handle["run_dir"] exit_text, last, stdout, tail = self.fetch(handle, self.COLLECT) # one remote call
last = t.read_text(t.join(d, "last.md"))
if last is None: if last is None:
last = (t.read_text(t.join(d, "stdout.txt")) or "")[-6000:] last = (stdout or "")[-6000:]
return ExecResult(last.strip(), self.exit_code(handle), None, return ExecResult(last.strip(), parse_exit(exit_text), None,
self.logs(handle, "stdout.txt", "stderr.txt"), self.stderr_tail(handle)) self.logs(handle, "stdout.txt", "stderr.txt"), tail)
+146 -5
View File
@@ -4,21 +4,162 @@ Interface (duck-typed, see docs/executor-contract.md):
os, join(*parts), quote(s), default_run_root() os, join(*parts), quote(s), default_run_root()
mkdir(path, clean=False), write_text(path, text), read_text(path) -> str|None, mkdir(path, clean=False), write_text(path, text), read_text(path) -> str|None,
grep_first(path, needle) -> str|None, tail(path, n) -> str grep_first(path, needle) -> str|None, tail(path, n) -> str
fetch(run_dir, ops) -> [result per op] (ops: ("read", name) | ("grep", name, needle) | ("tail", name, n))
runner(run_dir, cwd, command, stdout_name) -> (script_name, script_text) runner(run_dir, cwd, command, stdout_name) -> (script_name, script_text)
launch(task, script_path, run_dir), kill(task, run_dir), forget(task) start_run(task, run_dir, files, script_name) (remote only), launch(task, script_path, run_dir), kill(task, run_dir),
process_count(name) -> int|None (None = unreachable)""" forget(task), process_count(name) -> int|None (None = unreachable)
Remote terminals are reached through a channel (design §9): `warpgate-ssh` (agent-access.py run through the
Warpgate gateway) or `direct-ssh` (OpenSSH over Tailscale with a dedicated key, optional warpgate fallback)."""
import base64
import functools
import re
import time
class TransportError(Exception): class TransportError(Exception):
pass pass
def make_transport(cfg, terminal): class Unreached(TransportError):
kind = cfg.terminals[terminal]["transport"] """The remote command certainly did not run (connect / auth / agent failure, direct path unavailable): the
only kind of failure a direct-ssh terminal may repeat on its fallback channel."""
class HostKeyError(TransportError):
"""The terminal presented a host key other than the pinned one: a security error, never retried elsewhere."""
class CallLog:
"""One line per remote call -> `path` (remote.log); notable events (fallback, host key) also -> `event`."""
def __init__(self, path=None, event=None, echo=False):
self.path, self.event_fn, self.echo = path, event, echo
self._once = set()
def call(self, terminal, via, op, ms, rc, note=""):
line = "term=%s via=%s op=%s ms=%d rc=%s%s" % (terminal, via, op or "-", ms, rc, " " + note if note else "")
if self.echo:
print(line, flush=True)
if self.path:
try:
self.path.parent.mkdir(parents=True, exist_ok=True)
with open(self.path, "a", encoding="utf-8") as fh:
fh.write(time.strftime("%Y-%m-%dT%H:%M:%S") + " " + line + "\n")
except OSError:
pass
def event(self, msg, once=None):
"""`once` = a key: the message is logged only the first time per process."""
if once is not None:
if once in self._once:
return
self._once.add(once)
if self.event_fn:
self.event_fn(msg)
NULL_LOG = CallLog()
def operation(name):
"""Method decorator: names the remote operation for the call log (innermost wins, restored on return)."""
def deco(fn):
@functools.wraps(fn)
def wrapper(self, *args, **kwargs):
prev, self._op = getattr(self, "_op", None), name
try:
return fn(self, *args, **kwargs)
finally:
self._op = prev
return wrapper
return deco
def fetch_each(t, run_dir, ops):
"""`fetch` built from the single-file primitives (one call per op): local transport and fallbacks."""
out = []
for o in ops:
path = t.join(run_dir, o[1])
if o[0] == "read":
out.append(t.read_text(path))
elif o[0] == "grep":
out.append(t.grep_first(path, o[2]))
elif o[0] == "tail":
out.append(t.tail(path, o[2]))
else:
raise ValueError("unknown fetch op %r" % (o,))
return out
def parse_fetch(out, ops, what=""):
"""Decode the `F<i>=<base64>` / `M<i>` (missing) lines of a remote fetch script into results per op:
read -> text|None, grep -> line|None, tail -> text ("" when missing)."""
res = []
for i, o in enumerate(ops):
m = re.search(r"^F%d=([A-Za-z0-9+/=]*)$" % i, out, re.M)
if m:
text = base64.b64decode(m.group(1)).decode("utf-8", "replace") if m.group(1) else ""
elif re.search(r"^M%d$" % i, out, re.M):
text = None
else:
raise TransportError("fetch %s: unexpected output: %s" % (what, out.strip()[-200:]))
if o[0] == "grep":
text = (text or "").rstrip("\n") or None
elif o[0] == "tail":
text = (text or "").rstrip("\n")
res.append(text)
return res
class RoutedTransport:
"""direct-ssh terminal: every operation runs on the primary (direct) transport; when it fails before the remote
command ran (`Unreached`) the operation is repeated on the fallback (warpgate-ssh) transport, which then serves
the rest of this transport's life (one tick). Each direct operation is a single remote call, so a repeat never
doubles a side effect. Host-key failures and failures after the command may have run propagate unchanged."""
PURE = ("join", "quote", "default_run_root", "runner")
def __init__(self, terminal, primary, fallback=None, log=None):
self.terminal, self.primary, self.fallback = terminal, primary, fallback
self.log = log or NULL_LOG
self.on_fallback = False
def __getattr__(self, name):
attr = getattr(self.primary, name)
if name in self.PURE or not callable(attr):
return attr
def routed(*args, **kwargs):
if self.on_fallback:
return getattr(self.fallback, name)(*args, **kwargs)
try:
return attr(*args, **kwargs)
except Unreached as exc:
if self.fallback is None:
raise
self.on_fallback = True
self.log.event("%s: direct-ssh failed before the command ran (%s); using the warpgate-ssh fallback "
"for the rest of this tick" % (self.terminal, exc))
return getattr(self.fallback, name)(*args, **kwargs)
return routed
def make_transport(cfg, terminal, log=None, force=None):
"""`force` = direct-ssh | warpgate-ssh: that channel only, no fallback (agentd ping)."""
t = cfg.terminals[terminal]
kind = force or t["transport"]
if kind == "local": if kind == "local":
from .local import LocalTransport from .local import LocalTransport
return LocalTransport(cfg, terminal) return LocalTransport(cfg, terminal)
if kind == "warpgate-ssh": if kind == "warpgate-ssh":
from .warpgate_ssh import WarpgateSshTransport from .warpgate_ssh import WarpgateSshTransport
return WarpgateSshTransport(cfg, terminal) return WarpgateSshTransport(cfg, terminal, log=log)
if kind == "direct-ssh":
from .direct_ssh import DirectChannel
from .warpgate_ssh import WarpgateSshTransport
primary = WarpgateSshTransport(cfg, terminal, channel=DirectChannel(cfg, terminal, log))
fallback = None
if not force and t.get("fallback") == "warpgate-ssh":
fallback = WarpgateSshTransport(cfg, terminal, log=log, label="fallback")
return RoutedTransport(terminal, primary, fallback, log)
raise TransportError("unknown transport %s" % kind) raise TransportError("unknown transport %s" % kind)
+313
View File
@@ -0,0 +1,313 @@
"""direct-ssh channel (PAD-12): OpenSSH from the dispatcher straight to the terminal over Tailscale, no Warpgate.
- Auth: one dedicated agentd key. Its private text arrives in the env var named by `agentd.ssh_key` (default
AGENTD_SSH_KEY, injected by `agent-access.py exec --secret-env`), is popped from os.environ (children never see
it) and loaded with `ssh-add -` into a per-process ssh-agent on a socket in a private 0700 temp dir. The key is
never written to disk, argv, logs or exceptions. No key / agent failure -> the channel is unavailable
(`Unreached`), so a terminal with `fallback: warpgate-ssh` uses Warpgate instead.
- Host key: pinned per terminal (`ssh.host_key`); a known_hosts file is generated from it (public data) and
checked strictly. A mismatch raises HostKeyError: never a fallback.
- Reuse: ControlMaster=auto / ControlPersist=60 under `agentd.ssh_dir` (root-only 0700), so the second call of a
tick (and the next tick within a minute) skips the handshake. The socket name carries a fingerprint of the pinned
host key: a changed pin never rides an open master that was verified against the old one.
- Payload on stdin: Windows runs a fixed PowerShell bootstrap (-EncodedCommand) that reads the base64 of the real
script from stdin; POSIX runs a fixed bash loader that reads the script from stdin. One call per operation, no
argv size limit."""
import atexit
import base64
import hashlib
import os
import re
import shutil
import signal
import subprocess
import tempfile
import time
from . import NULL_LOG, HostKeyError, TransportError, Unreached
from .local import sh_quote
CONNECT_TIMEOUT = 8
CONTROL_PERSIST = 60
SOCKET_MAX = 104 # sun_path; ssh adds ".<16 random chars>" while it sets up the master socket
CTL_NAME = 1 + 9 + 40 + 17 # "/<pin8>-<%C: 40 hex>" + that suffix
# Fixed PowerShell bootstrap: the operation's script arrives on stdin as base64 of its UTF-8 text (ASCII on the
# wire, so the console input code page does not matter).
BOOTSTRAP = ("$ProgressPreference='SilentlyContinue'\n"
"$ErrorActionPreference='Continue'\n"
"[Console]::OutputEncoding=[Text.Encoding]::UTF8\n"
"$s=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String([Console]::In.ReadToEnd().Trim()))\n"
"& ([ScriptBlock]::Create($s))\n")
def encoded_command(script):
"""PowerShell -EncodedCommand value: base64 of the UTF-16LE text."""
return base64.b64encode(script.encode("utf-16-le")).decode("ascii")
PS_COMMAND = "powershell.exe -NoProfile -NonInteractive -EncodedCommand " + encoded_command(BOOTSTRAP)
# POSIX loader: the script is read from stdin into a variable (no MAX_ARG_STRLEN), then runs in a subshell with
# stdin = /dev/null. Its exit 255 is reported as 254: 255 stays ssh's own "no session" code. (The PowerShell
# bootstrap exits 0/1 by $?; none of our PowerShell scripts calls `exit`.)
SH_COMMAND = "bash -c " + sh_quote('exec 3<&0 </dev/null; s=$(cat <&3); exec 3<&-; (eval "$s"); r=$?; '
'[ $r = 255 ] && r=254; exit $r')
def ps_payload(script):
return (base64.b64encode(script.encode("utf-8")).decode("ascii") + "\n").encode("ascii")
# ---------------------------------------------------------------- failure classification
# A host offering no ssh-ed25519 key at all counts as a host-key failure too (a downgrade is a security signal).
HOSTKEY = re.compile(r"Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED|"
r"No \S+ host key is known for|host key for \S+ has changed|no matching host key type", re.I)
# ssh exit 255 with one of these = no remote session ran the command. Only ssh's own messages, anchored at the start
# of a line (stderr also carries the remote command's stderr).
UNREACHED = re.compile(r"^(?:ssh: connect to host |ssh: Could not resolve hostname |banner exchange: |"
r"Connection timed out during banner exchange|kex_exchange_identification: |"
r"Connection (?:closed|reset) by \S+ port \d+\s*$|"
r"\S+: Permission denied \((?:publickey|password|keyboard-interactive|hostbased|gssapi)|"
r"Received disconnect from .*Too many authentication failures|"
r"sign_and_send_pubkey: |error fetching identities|Error connecting to agent|"
r"Unable to negotiate with |Session open refused by peer)", re.I | re.M)
def classify(rc, err, out=""):
"""-> 'ok' (the remote command ran, whatever its exit code), 'hostkey', 'unreached' (certainly not run) or
'unknown' (ssh exit 255 without a recognisable pre-session error: the command may have run). Any stdout proves
that a session ran the command, so it is never 'unreached'."""
if rc != 255:
return "ok"
if (out or "").strip():
return "unknown"
if HOSTKEY.search(err or ""):
return "hostkey"
if UNREACHED.search(err or ""):
return "unreached"
return "unknown"
# ---------------------------------------------------------------- per-process key agent
def _die_with_parent():
"""preexec: the agent gets SIGTERM when agentd dies, even by SIGKILL (Linux PR_SET_PDEATHSIG)."""
try:
import ctypes
ctypes.CDLL("libc.so.6", use_errno=True).prctl(1, signal.SIGTERM)
except Exception:
pass
def normalise_key(text):
"""Vault may hand the key with literal `\\n` escapes or without the final newline."""
if "\n" not in text.strip() and "\\n" in text:
text = text.replace("\\n", "\n")
return text.strip() + "\n"
_KEYS = {} # env var name -> key text, taken out of os.environ at CLI start until the agent loads it
def take_key(env_name):
"""Move the key out of os.environ right away (called by the CLI before anything else runs), so no child -- not
even one started before the first direct call -- can inherit it."""
if env_name not in _KEYS:
_KEYS[env_name] = os.environ.pop(env_name, None)
else:
os.environ.pop(env_name, None)
def key_present(env_name):
return bool((_KEYS.get(env_name) or os.environ.get(env_name) or "").strip())
class KeyAgent:
"""ssh-agent of this process holding the agentd key. `error` set = direct-ssh unavailable here."""
def __init__(self, env_name):
self.env_name = env_name
self.dir = self.sock = self.pub = self.proc = None
self.error = None
def start(self):
key = os.environ.pop(self.env_name, None) # popped even if loading fails: children never inherit it
key = _KEYS.pop(self.env_name, None) or key
if not key or not key.strip():
self.error = "%s is not set (agent-access.py exec --secret-env %s=<vault key>)" % (self.env_name, self.env_name)
return self
agent_bin, add_bin = shutil.which("ssh-agent"), shutil.which("ssh-add")
if not (agent_bin and add_bin and shutil.which("ssh")):
self.error = "ssh / ssh-agent / ssh-add not installed"
return self
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LANG": "C"}
try:
self.dir = tempfile.mkdtemp(prefix="agentd-agent-") # 0700
self.sock = os.path.join(self.dir, "agent.sock")
atexit.register(self.close)
self.proc = subprocess.Popen([agent_bin, "-D", "-a", self.sock], stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env,
close_fds=True, preexec_fn=_die_with_parent)
end = time.monotonic() + 5
while not os.path.exists(self.sock):
if self.proc.poll() is not None or time.monotonic() > end:
self.error = "ssh-agent did not start"
return self
time.sleep(0.02)
env["SSH_AUTH_SOCK"] = self.sock
add = subprocess.run([add_bin, "-q", "-"], input=normalise_key(key).encode("utf-8"), env=env,
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=15)
key = None
if add.returncode != 0:
why = add.stderr.decode("utf-8", "replace").strip().splitlines()[:1]
why = why[0][:120] if why and "-----" not in why[0] else ""
self.error = "ssh-add failed (rc %d) %s" % (add.returncode, why)
return self
listed = subprocess.run([add_bin, "-L"], env=env, capture_output=True, timeout=15)
pub = next((l for l in listed.stdout.decode("ascii", "replace").splitlines() if l.startswith("ssh-")), None)
if not pub:
self.error = "ssh-agent holds no key after ssh-add"
return self
self.pub = os.path.join(self.dir, "agentd.pub") # public half: selects the agent key (IdentitiesOnly)
with open(self.pub, "w", encoding="ascii") as fh:
fh.write(pub + "\n")
except (OSError, subprocess.SubprocessError) as exc:
self.error = "ssh-agent setup failed: %s" % type(exc).__name__
finally:
key = None
if self.error:
self.close()
return self
def close(self):
if self.proc and self.proc.poll() is None:
self.proc.terminate()
try:
self.proc.wait(3)
except subprocess.TimeoutExpired:
self.proc.kill()
if self.dir:
shutil.rmtree(self.dir, ignore_errors=True)
_AGENTS = {}
def key_agent(cfg, log=NULL_LOG):
"""The process-wide agent for cfg's key env var, started on first use; raises Unreached when unavailable."""
name = cfg.ssh_key_env
agent = _AGENTS.get(name)
if agent is None:
agent = _AGENTS[name] = KeyAgent(name).start()
if agent.error:
log.event("direct-ssh unavailable in this process: %s" % agent.error, once="agent:" + name)
raise Unreached("direct-ssh unavailable: %s" % agent.error)
return agent
def control_dir(cfg):
"""agentd.ssh_dir: ControlMaster sockets + generated known_hosts; created 0700, must be ours."""
d = cfg.ssh_dir
try:
d.mkdir(parents=True, exist_ok=True)
os.chmod(d, 0o700)
if os.stat(d).st_uid != os.geteuid():
raise OSError("not owned by this user")
except OSError as exc:
raise Unreached("ssh_dir %s unusable: %s" % (d, exc)) from None
return d
# ---------------------------------------------------------------- channel
class DirectChannel:
"""One terminal's direct OpenSSH path. ps() / sh() mirror the warpgate channel; payloads go on stdin."""
stdin = True
def __init__(self, cfg, terminal, log=None, label="direct"):
self.cfg, self.terminal, self.log, self.via = cfg, terminal, log or NULL_LOG, label
s = cfg.terminals[terminal]["ssh"]
self.host, self.user, self.port = s["host"], s["user"], int(s.get("port", 22))
self.host_key = " ".join(s["host_key"].split()[:2])
self.pin = hashlib.sha256(self.host_key.encode("ascii")).hexdigest()[:8]
self.alias = "agentd-" + terminal
self.hostkey_failed = None
self.name = "ssh %s@%s" % (self.user, self.host)
def known_hosts(self, d):
path = d / ("known_hosts." + self.terminal)
line = "%s %s\n" % (self.alias, self.host_key)
try:
if not path.exists() or path.read_text(encoding="ascii") != line:
tmp = path.with_name("%s.%d.tmp" % (path.name, os.getpid())) # per terminal and process
tmp.write_text(line, encoding="ascii")
os.replace(tmp, path)
except OSError as exc:
raise Unreached("known_hosts %s: %s" % (path, exc)) from None
return path
def argv(self, agent, remote_command):
d = control_dir(self.cfg)
ctl = str(d / (self.pin + "-%C")) # pin fingerprint: a new pin never reuses a master verified on the old one
if len(str(d)) + CTL_NAME > SOCKET_MAX:
self.log.event("agentd.ssh_dir %s is too long for a control socket; connection reuse off" % d, once="ctl")
mux = ["-o", "ControlMaster=no", "-o", "ControlPath=none"]
else:
mux = ["-o", "ControlMaster=auto", "-o", "ControlPersist=%d" % CONTROL_PERSIST, "-o", "ControlPath=" + ctl]
return (["ssh", "-F", "/dev/null", "-T", "-p", str(self.port), "-l", self.user,
"-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
"-o", "IdentityAgent=" + agent.sock, "-o", "IdentityFile=" + agent.pub,
"-o", "PreferredAuthentications=publickey", "-o", "PasswordAuthentication=no",
"-o", "KbdInteractiveAuthentication=no",
"-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + str(self.known_hosts(d)),
"-o", "GlobalKnownHostsFile=/dev/null", "-o", "HostKeyAlias=" + self.alias,
"-o", "HostKeyAlgorithms=" + self.host_key.split()[0], "-o", "UpdateHostKeys=no",
"-o", "ConnectTimeout=%d" % CONNECT_TIMEOUT, "-o", "ServerAliveInterval=15",
"-o", "ServerAliveCountMax=3", "-o", "ForwardAgent=no", "-o", "ForwardX11=no",
"-o", "ClearAllForwardings=yes", "-o", "LogLevel=ERROR", "-o", "EscapeChar=none"]
+ mux + ["--", self.host, remote_command])
def run(self, remote_command, payload, timeout, op):
"""-> (rc, stdout, stderr) of a call whose command ran; raises HostKeyError / Unreached / TransportError."""
if self.hostkey_failed:
raise HostKeyError(self.hostkey_failed)
agent = key_agent(self.cfg, self.log)
argv = self.argv(agent, remote_command)
env = self.cfg.scrubbed_env()
env.pop("SSH_AUTH_SOCK", None)
t0 = time.monotonic()
ms = lambda: (time.monotonic() - t0) * 1000 # noqa: E731
# output goes to files, not pipes: a ControlPersist master forked by this call must never hold our pipes open
with tempfile.TemporaryFile() as fo, tempfile.TemporaryFile() as fe:
try:
p = subprocess.run(argv, input=payload, stdout=fo, stderr=fe, env=env, timeout=timeout)
except subprocess.TimeoutExpired:
self.log.call(self.terminal, self.via, op, ms(), "timeout")
raise TransportError("%s timeout after %ss" % (self.name, timeout)) from None # may have run
except OSError as exc:
self.log.call(self.terminal, self.via, op, ms(), "launch-failed")
raise Unreached("%s: ssh launch failed: %s" % (self.name, type(exc).__name__)) from None
fo.seek(0)
fe.seek(0)
dec = lambda b: b.decode("utf-8", "replace").replace("\r", "") # noqa: E731
out, err = dec(fo.read()), dec(fe.read())
kind = classify(p.returncode, err, out)
self.log.call(self.terminal, self.via, op, ms(), p.returncode, "" if kind == "ok" else kind)
if kind == "hostkey":
self.hostkey_failed = ("%s:%d host key verification FAILED (pinned %s...; other key or no such key type "
"offered): not falling back; check terminals.%s.ssh.host_key out of band"
% (self.host, self.port, self.host_key[:40], self.terminal))
self.log.event("SECURITY %s: %s" % (self.terminal, self.hostkey_failed), once="hostkey:" + self.terminal)
raise HostKeyError(self.hostkey_failed)
if kind == "unreached":
raise Unreached("%s exit 255: %s" % (self.name, err.strip()[-300:]))
if kind == "unknown":
raise TransportError("%s exit 255: %s" % (self.name, err.strip()[-300:] or "(no message)"))
return p.returncode, out, err
def ps(self, script, timeout, op=None):
rc, out, err = self.run(PS_COMMAND, ps_payload(script), timeout, op)
if rc != 0 and not out.strip():
raise TransportError("%s exit %s: %s" % (self.name, rc, err.strip()[-300:]))
return out
def sh(self, script, timeout, op=None):
return self.run(SH_COMMAND, script.encode("utf-8"), timeout, op)
+1 -1
View File
@@ -91,5 +91,5 @@ class LocalTransport:
pass pass
def process_count(self, name): def process_count(self, name):
out = subprocess.run(["pgrep", "-c", "-x", name], capture_output=True, text=True) out = subprocess.run(["pgrep", "-c", "-x", name], capture_output=True, text=True, env=self.cfg.scrubbed_env())
return int(out.stdout.strip() or 0) return int(out.stdout.strip() or 0)
+223 -45
View File
@@ -1,17 +1,25 @@
"""Terminals behind Warpgate, reached through the server-access relay client: """Remote terminals (Windows and Linux/macOS) and the Warpgate channel.
The transport classes build the scripts; a *channel* carries them (design §9):
- WarpgateChannel (`transport: warpgate-ssh`, also the direct-ssh fallback): the server-access relay client
agent-access.py --config <identity> run --target <target> --timeout N -- <argv...> agent-access.py --config <identity> run --target <target> --timeout N -- <argv...>
The client shlex-joins argv and the target account's login shell runs it (cmd.exe on Windows); stdin is The client shlex-joins argv and the target account's login shell runs it (cmd.exe on Windows); stdin is
/dev/null, so everything travels in argv. `relay()` is the single place that builds that call. /dev/null (agent-access.py run_remote uses stdin=DEVNULL), so everything travels in argv. `relay()` is the single
place that builds that call.
- DirectChannel (`transport: direct-ssh`, agentd/transports/direct_ssh.py): OpenSSH over Tailscale; the script goes
on stdin, so every operation (including a launch with its prompt) is one call.
- os linux / darwin -> PosixWarpgateTransport: every operation is one `bash -c '<script>'` relay call. Files go up - os linux / darwin -> PosixWarpgateTransport: every operation is one `bash -c '<script>'` call. On the relay files go
as base64 chunks (`printf %s '<b64>' | base64 -d`), the runner starts detached with `setsid nohup bash run.sh` up as base64 chunks (`printf %s '<b64>' | base64 -d`), on the direct channel in the same single call. The runner
(its own session: pid == pgid == sid), `poll` reads exit.txt, `kill` signals the process group and session. starts detached with `setsid nohup bash run.sh` (its own session: pid == pgid == sid), `poll` reads exit.txt,
- os windows -> WindowsWarpgateTransport: still drives the helper scripts win10-ps.sh (PowerShell `fetch` returns everything `collect` needs in one call, `kill` signals the process group and session.
-EncodedCommand) and win10-put.sh (chunked upload, SHA-256 verified), which hard-code `--target win10`; - os windows -> WindowsWarpgateTransport: PowerShell scripts; runs are one-shot scheduled tasks (schtasks) so they
runs are one-shot scheduled tasks (schtasks) so they outlive the SSH session. outlive the SSH session. On the relay it still drives the helper scripts win10-ps.sh (PowerShell -EncodedCommand,
TODO: move Windows onto relay() (`powershell -EncodedCommand` as argv) and drop the helper scripts.""" argv-size bound) and win10-put.sh (chunked upload, SHA-256 verified), which hard-code `--target win10`; on the
direct channel mkdir + clean + prompt.md + run.ps1 + schtasks is one call.
The class names are historical: the channel, not the class, decides the path."""
import base64 import base64
import hashlib import hashlib
import os import os
@@ -20,8 +28,9 @@ import re
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time
from . import TransportError from . import NULL_LOG, TransportError, Unreached, fetch_each, operation, parse_fetch
from .local import RUNNER as POSIX_RUNNER, sh_quote from .local import RUNNER as POSIX_RUNNER, sh_quote
DEFAULT_CLIENT = "~/.local/share/server-access-client/agent-access.py" DEFAULT_CLIENT = "~/.local/share/server-access-client/agent-access.py"
@@ -55,6 +64,55 @@ def relay(cfg, target, argv, timeout=150):
return call(cfg, relay_argv(cfg, target, argv, timeout), timeout) return call(cfg, relay_argv(cfg, target, argv, timeout), timeout)
PS_ARGV_MAX = 7600 # win10-ps.sh refuses longer -EncodedCommand values (cmd.exe line limit 8191)
PS_PRELUDE = 120 # characters win10-ps.sh puts in front of every script
class WarpgateChannel:
"""Calls through agent-access.py run (argv only). Each call is logged with its duration."""
stdin = False
def __init__(self, cfg, terminal, log=None, label="warpgate"):
self.cfg, self.terminal, self.log, self.via = cfg, terminal, log or NULL_LOG, label
t = cfg.terminals[terminal]
self.target = t.get("target", terminal)
self.name = "relay %s" % self.target
if t["os"] == "windows":
helpers = cfg.helpers_dir(terminal)
self.ps_script, self.put_script = str(helpers / "win10-ps.sh"), str(helpers / "win10-put.sh")
def timed(self, op, fn, *args):
t0 = time.monotonic()
try:
res = fn(*args)
except TransportError:
self.log.call(self.terminal, self.via, op, (time.monotonic() - t0) * 1000, "error")
raise
self.log.call(self.terminal, self.via, op, (time.monotonic() - t0) * 1000, res[0])
return res
def fits(self, script):
"""Whether win10-ps.sh accepts the script (its -EncodedCommand stays under PS_ARGV_MAX)."""
units = len(script.encode("utf-16-le")) // 2 + PS_PRELUDE
return (2 * units + 2) // 3 * 4 <= PS_ARGV_MAX
def ps(self, script, timeout, op=None):
rc, out, err = self.timed(op, call, self.cfg, [self.ps_script, script], timeout, {"WIN10_TIMEOUT": str(timeout)})
if rc != 0 and not out.strip():
raise TransportError("relay exit %s: %s" % (rc, err.strip()[-300:]))
return out
def put(self, local, remote, op=None):
with open(local, "rb") as fh:
sha = hashlib.sha256(fh.read()).hexdigest()
rc, out, err = self.timed(op, call, self.cfg, [self.put_script, local, remote], 540)
if ("remote sha256 " + sha) not in out:
raise TransportError("upload of %s failed: %s" % (remote, (out + err).strip()[-300:]))
def sh(self, script, timeout, op=None):
return self.timed(op, relay, self.cfg, self.target, ["bash", "-c", script], timeout)
RUNNER = r"""$ErrorActionPreference = 'Continue' RUNNER = r"""$ErrorActionPreference = 'Continue'
[Console]::OutputEncoding = [Text.Encoding]::UTF8 [Console]::OutputEncoding = [Text.Encoding]::UTF8
$OutputEncoding = [Text.Encoding]::UTF8 $OutputEncoding = [Text.Encoding]::UTF8
@@ -68,6 +126,15 @@ Set-Content -Path '{d}\exit.txt' -Value ("exit=" + $LASTEXITCODE + " at " + (Get
""" """
# Windows fetch helpers: F<i>=<base64> per result, M<i> = file missing (parse_fetch). Names must not collide with
# PowerShell's built-in aliases (aliases win over functions: `r` is Invoke-History).
PS_FETCH = r"""function AdRead($i,$n){$p=Join-Path $d $n; if(Test-Path -LiteralPath $p){"F$i="+[Convert]::ToBase64String([IO.File]::ReadAllBytes($p))}else{"M$i"}}
function AdOut($i,$t){"F$i="+[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))}
function AdGrep($i,$n,$x){$p=Join-Path $d $n; $m=$null; if(Test-Path -LiteralPath $p){$m=Select-String -LiteralPath $p -SimpleMatch -Pattern $x|Select-Object -First 1}; if($m){AdOut $i $m.Line}else{"F$i="}}
function AdTail($i,$n,$k){$p=Join-Path $d $n; $t=''; if(Test-Path -LiteralPath $p){$t=(Get-Content -LiteralPath $p -Encoding UTF8 -Tail $k) -join "`n"}; AdOut $i $t}
"""
def _b64_payload(out, marker): def _b64_payload(out, marker):
m = re.search(marker + r"=([A-Za-z0-9+/=]*)", out) m = re.search(marker + r"=([A-Za-z0-9+/=]*)", out)
if not m: if not m:
@@ -78,12 +145,11 @@ def _b64_payload(out, marker):
class WindowsWarpgateTransport: class WindowsWarpgateTransport:
os = "windows" os = "windows"
def __init__(self, cfg, terminal): def __init__(self, cfg, terminal, channel=None, log=None, label="warpgate"):
self.cfg, self.terminal = cfg, terminal self.cfg, self.terminal = cfg, terminal
t = cfg.terminals[terminal] t = cfg.terminals[terminal]
self.target = t.get("target", terminal) self.target = t.get("target", terminal)
helpers = cfg.helpers_dir(terminal) self.channel = channel or WarpgateChannel(cfg, terminal, log, label)
self.ps_script, self.put_script = str(helpers / "win10-ps.sh"), str(helpers / "win10-put.sh")
self.repo = t["repo"] self.repo = t["repo"]
def default_run_root(self): def default_run_root(self):
@@ -95,32 +161,38 @@ class WindowsWarpgateTransport:
def quote(self, s): def quote(self, s):
return "'" + str(s).replace("'", "''") + "'" return "'" + str(s).replace("'", "''") + "'"
# ---------------------------------------------------------------- relay primitives # ---------------------------------------------------------------- channel primitives
def ps(self, script, timeout=150): def ps(self, script, timeout=150):
rc, out, err = call(self.cfg, [self.ps_script, script], timeout, {"WIN10_TIMEOUT": str(timeout)}) return self.channel.ps(script, timeout, getattr(self, "_op", None))
if rc != 0 and not out.strip():
raise TransportError("relay exit %s: %s" % (rc, err.strip()[-300:]))
return out
def put(self, local, remote): def put(self, local, remote):
with open(local, "rb") as fh: self.channel.put(local, remote, getattr(self, "_op", None))
sha = hashlib.sha256(fh.read()).hexdigest()
rc, out, err = call(self.cfg, [self.put_script, local, remote], 540) @operation("ping")
if ("remote sha256 " + sha) not in out: def ping(self):
raise TransportError("upload of %s failed: %s" % (remote, (out + err).strip()[-300:])) out = self.ps("'PONG=' + $PSVersionTable.PSVersion.ToString() + ' ' + $env:COMPUTERNAME", timeout=60)
m = re.search(r"PONG=[^\n]*", out)
if not m:
raise TransportError("ping %s: unexpected output: %s" % (self.terminal, out.strip()[-200:]))
return m.group(0)
# ---------------------------------------------------------------- file ops # ---------------------------------------------------------------- file ops
def mkdir(self, path, clean=False): def mkdir_script(self, path, clean=False):
q = self.quote(path) q = self.quote(path)
script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q
if clean: # never wipe a directory a runner has already started in if clean: # never wipe a directory a runner has already started in
script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) " script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) "
"{ 'HASPID'; return }\n" % (q, q)) "{ 'HASPID'; return }\n" % (q, q))
script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q
out = self.ps(script + "'MKDIR=ok'") return script
@operation("mkdir")
def mkdir(self, path, clean=False):
out = self.ps(self.mkdir_script(path, clean) + "'MKDIR=ok'")
if "MKDIR=ok" not in out: if "MKDIR=ok" not in out:
raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:])) raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:]))
@operation("probe")
def probe(self, run_dir, names): def probe(self, run_dir, names):
out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')" out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')"
% (self.quote(run_dir), ",".join(self.quote(n) for n in names))) % (self.quote(run_dir), ",".join(self.quote(n) for n in names)))
@@ -129,7 +201,20 @@ class WindowsWarpgateTransport:
raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:])) raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:]))
return [n for n in m.group(1).strip().split(",") if n] return [n for n in m.group(1).strip().split(",") if n]
def write_script(self, path, data):
"""Write bytes exactly as given (UTF-8 without BOM for our texts), echo their SHA-256."""
return ("$p = %s\n[IO.File]::WriteAllBytes($p, [Convert]::FromBase64String('%s'))\n"
"'WROTE=' + (Get-FileHash -LiteralPath $p -Algorithm SHA256).Hash.ToLower()\n"
% (self.quote(path), base64.b64encode(data).decode("ascii")))
@operation("write")
def write_text(self, path, text): def write_text(self, path, text):
if self.channel.stdin:
data = text.encode("utf-8")
out = self.ps(self.write_script(path, data))
if re.findall(r"WROTE=([0-9a-f]{64})", out) != [hashlib.sha256(data).hexdigest()]:
raise TransportError("upload of %s failed: %s" % (path, out.strip()[-300:]))
return
fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt") fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt")
try: try:
with os.fdopen(fd, "w", encoding="utf-8", newline="") as fh: with os.fdopen(fd, "w", encoding="utf-8", newline="") as fh:
@@ -138,6 +223,7 @@ class WindowsWarpgateTransport:
finally: finally:
os.unlink(tmp) os.unlink(tmp)
@operation("read")
def read_text(self, path): def read_text(self, path):
q = self.quote(path) q = self.quote(path)
out = self.ps("if (Test-Path -LiteralPath %s) { 'B64=' + [Convert]::ToBase64String([IO.File]::ReadAllBytes(%s)) }" out = self.ps("if (Test-Path -LiteralPath %s) { 'B64=' + [Convert]::ToBase64String([IO.File]::ReadAllBytes(%s)) }"
@@ -147,8 +233,9 @@ class WindowsWarpgateTransport:
text = _b64_payload(out, "B64") text = _b64_payload(out, "B64")
if text is None: if text is None:
raise TransportError("read %s: unexpected relay output" % path) raise TransportError("read %s: unexpected relay output" % path)
return text.lstrip("") return text.lstrip("\ufeff")
@operation("grep")
def grep_first(self, path, needle): def grep_first(self, path, needle):
q = self.quote(path) q = self.quote(path)
out = self.ps("$m = if (Test-Path -LiteralPath %s) { Select-String -LiteralPath %s -SimpleMatch -Pattern %s | Select-Object -First 1 }\n" out = self.ps("$m = if (Test-Path -LiteralPath %s) { Select-String -LiteralPath %s -SimpleMatch -Pattern %s | Select-Object -First 1 }\n"
@@ -156,12 +243,30 @@ class WindowsWarpgateTransport:
% (q, q, self.quote(needle))) % (q, q, self.quote(needle)))
return _b64_payload(out, "B64") or None return _b64_payload(out, "B64") or None
@operation("tail")
def tail(self, path, n): def tail(self, path, n):
q = self.quote(path) q = self.quote(path)
out = self.ps("$t = if (Test-Path -LiteralPath %s) { (Get-Content -LiteralPath %s -Encoding UTF8 -Tail %d) -join \"`n\" } else { '' }\n" out = self.ps("$t = if (Test-Path -LiteralPath %s) { (Get-Content -LiteralPath %s -Encoding UTF8 -Tail %d) -join \"`n\" } else { '' }\n"
"'B64=' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))" % (q, q, n)) "'B64=' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))" % (q, q, n))
return _b64_payload(out, "B64") or "" return _b64_payload(out, "B64") or ""
def fetch_script(self, run_dir, ops):
lines = ["$d = %s" % self.quote(run_dir), PS_FETCH.rstrip("\n")]
for i, o in enumerate(ops):
fn = {"read": "AdRead", "grep": "AdGrep", "tail": "AdTail"}[o[0]]
arg = "" if o[0] == "read" else " " + (self.quote(o[2]) if o[0] == "grep" else str(int(o[2])))
lines.append("%s %d %s%s" % (fn, i, self.quote(o[1]), arg))
return "\n".join(lines) + "\n"
@operation("fetch")
def fetch(self, run_dir, ops):
"""Several reads of one run directory in one call (collect)."""
script = self.fetch_script(run_dir, ops)
if not self.channel.stdin and not self.channel.fits(script):
return fetch_each(self, run_dir, ops)
res = parse_fetch(self.ps(script, timeout=200), ops, run_dir)
return [r.lstrip("\ufeff") if o[0] == "read" and r else r for o, r in zip(ops, res)]
# ---------------------------------------------------------------- processes # ---------------------------------------------------------------- processes
def runner(self, run_dir, cwd, command, stdout_name): def runner(self, run_dir, cwd, command, stdout_name):
return "run.ps1", RUNNER.format(cwd=self.quote(cwd), d=run_dir.replace("'", "''"), command=command, stdout=stdout_name) return "run.ps1", RUNNER.format(cwd=self.quote(cwd), d=run_dir.replace("'", "''"), command=command, stdout=stdout_name)
@@ -176,23 +281,60 @@ class WindowsWarpgateTransport:
"schtasks /Run /TN '%(t)s' | Out-Null\n" "schtasks /Run /TN '%(t)s' | Out-Null\n"
"'LAUNCHED=' + $LASTEXITCODE" % dict(t=task.replace("'", "''"), s=script_path.replace("'", "''"))) "'LAUNCHED=' + $LASTEXITCODE" % dict(t=task.replace("'", "''"), s=script_path.replace("'", "''")))
@operation("launch")
def launch(self, task, script_path, run_dir): def launch(self, task, script_path, run_dir):
out = self.ps(self.launch_script(task, script_path)) out = self.ps(self.launch_script(task, script_path))
if "LAUNCHED=0" not in out: if "LAUNCHED=0" not in out:
raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:])) raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:]))
def start_script(self, task, run_dir, files, script_name):
"""mkdir + clean (HASPID refusal) + every file + schtasks, as one script. -> (script, [sha256 per file])"""
script, shas = "$ErrorActionPreference = 'Stop'\n" + self.mkdir_script(run_dir, clean=True), []
for name, text in files.items():
data = text.encode("utf-8")
script += self.write_script(self.join(run_dir, name), data)
shas.append(hashlib.sha256(data).hexdigest())
script += "$ErrorActionPreference = 'Continue'\n" + self.launch_script(task, self.join(run_dir, script_name))
return script, shas
@operation("start")
def start_run(self, task, run_dir, files, script_name):
"""Stage the run directory and start the task: one call on the direct channel; on the relay (argv only)
mkdir, chunked uploads and the launch go one by one."""
if not self.channel.stdin:
self.mkdir(run_dir, clean=True)
for name, text in files.items():
self.write_text(self.join(run_dir, name), text)
return self.launch(task, self.join(run_dir, script_name), run_dir)
script, shas = self.start_script(task, run_dir, files, script_name)
out = self.ps(script)
if re.search(r"^HASPID$", out, re.M):
raise TransportError("refusing to clean %s: it holds a pid file (HASPID)" % run_dir)
wrote = re.findall(r"WROTE=([0-9a-f]{64})", out)
if wrote != shas:
raise TransportError("staging %s: hash mismatch (%d/%d files): %s" % (run_dir, len(wrote), len(shas),
out.strip()[-300:]))
if "LAUNCHED=0" not in out:
raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:]))
@operation("kill")
def kill(self, task, run_dir): def kill(self, task, run_dir):
t = task.replace("'", "''")
self.ps("$p = Join-Path %s 'pid.txt'\n" self.ps("$p = Join-Path %s 'pid.txt'\n"
"if (Test-Path $p) { taskkill /T /F /PID (Get-Content $p | Select-Object -First 1) 2>&1 | Out-Null }\n" "if (Test-Path $p) { taskkill /T /F /PID (Get-Content $p | Select-Object -First 1) 2>&1 | Out-Null }\n"
"schtasks /End /TN '%s' 2>&1 | Out-Null\n" "schtasks /End /TN '%s' 2>&1 | Out-Null\n"
"schtasks /Delete /F /TN '%s' 2>&1 | Out-Null\n'KILLED=ok'" % (self.quote(run_dir), task, task)) "schtasks /Delete /F /TN '%s' 2>&1 | Out-Null\n'KILLED=ok'" % (self.quote(run_dir), t, t))
@operation("forget")
def forget(self, task): def forget(self, task):
self.ps("schtasks /Delete /F /TN '%s' 2>&1 | Out-Null; 'FORGOT=ok'" % task) self.ps("schtasks /Delete /F /TN '%s' 2>&1 | Out-Null; 'FORGOT=ok'" % task.replace("'", "''"))
@operation("busy")
def process_count(self, name): def process_count(self, name):
try: try:
out = self.ps("'COUNT=' + @(Get-Process -Name %s -ErrorAction SilentlyContinue).Count" % self.quote(name)) out = self.ps("'COUNT=' + @(Get-Process -Name %s -ErrorAction SilentlyContinue).Count" % self.quote(name))
except Unreached:
raise # a direct-ssh terminal retries this on its fallback
except TransportError: except TransportError:
return None return None
m = re.search(r"COUNT=(\d+)", out) m = re.search(r"COUNT=(\d+)", out)
@@ -210,15 +352,17 @@ DETACH = {
class PosixWarpgateTransport: class PosixWarpgateTransport:
"""Linux/macOS terminal through the relay. Each primitive is one `bash -c` round trip (~10 s through """Linux/macOS terminal. Each primitive is one `bash -c` round trip (~10 s through Vault + Warpgate, well under
Vault + Warpgate), so `start_run` stages the run directory and launches it in a single call.""" a second on a reused direct connection), so `start_run` stages the run directory and launches it in a single
CHUNK = 60000 # base64 chars per call: one argv element must stay below 128 KiB (MAX_ARG_STRLEN) on both ends call and `fetch` reads everything `collect` needs in one."""
CHUNK = 60000 # relay only: base64 chars per call; one argv element must stay below 128 KiB (MAX_ARG_STRLEN)
def __init__(self, cfg, terminal): def __init__(self, cfg, terminal, channel=None, log=None, label="warpgate"):
self.cfg, self.terminal = cfg, terminal self.cfg, self.terminal = cfg, terminal
t = cfg.terminals[terminal] t = cfg.terminals[terminal]
self.os = t["os"] self.os = t["os"]
self.target = t.get("target", terminal) self.target = t.get("target", terminal)
self.channel = channel or WarpgateChannel(cfg, terminal, log, label)
self.repo = t["repo"] self.repo = t["repo"]
def default_run_root(self): def default_run_root(self):
@@ -230,14 +374,19 @@ class PosixWarpgateTransport:
def quote(self, s): def quote(self, s):
return sh_quote(s) return sh_quote(s)
# ---------------------------------------------------------------- relay primitive # ---------------------------------------------------------------- channel primitive
def sh(self, script, timeout=150, marker=None): def sh(self, script, timeout=150, marker=None):
"""Run a bash script on the target. With `marker`, its absence in stdout is an error.""" """Run a bash script on the target. With `marker`, its absence in stdout is an error."""
rc, out, err = relay(self.cfg, self.target, ["bash", "-c", script], timeout) rc, out, err = self.channel.sh(script, timeout, getattr(self, "_op", None))
if (marker and marker not in out) or (not marker and rc != 0 and not out.strip()): if (marker and marker not in out) or (not marker and rc != 0 and not out.strip()):
raise TransportError("relay %s exit %s: %s" % (self.target, rc, (out + err).strip()[-300:])) raise TransportError("%s exit %s: %s" % (self.channel.name, rc, (out + err).strip()[-300:]))
return out return out
@operation("ping")
def ping(self):
out = self.sh("echo \"PONG=$(uname -sr) $(uname -n)\"", timeout=60, marker="PONG=")
return re.search(r"PONG=[^\n]*", out).group(0)
# ---------------------------------------------------------------- file ops # ---------------------------------------------------------------- file ops
def mkdir_script(self, path, clean=False): def mkdir_script(self, path, clean=False):
q = self.quote(path) q = self.quote(path)
@@ -248,6 +397,7 @@ class PosixWarpgateTransport:
s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q
return s return s
@operation("mkdir")
def mkdir(self, path, clean=False): def mkdir(self, path, clean=False):
self.sh(self.mkdir_script(path, clean) + "echo MKDIR=ok\n", marker="MKDIR=ok") self.sh(self.mkdir_script(path, clean) + "echo MKDIR=ok\n", marker="MKDIR=ok")
@@ -261,10 +411,11 @@ class PosixWarpgateTransport:
return ("printf %%s '%s' | base64 -d > %s && mv -f %s %s || exit 1\n" return ("printf %%s '%s' | base64 -d > %s && mv -f %s %s || exit 1\n"
"echo \"WROTE=$(wc -c < %s | tr -d ' ')\"\n" % (b64, part, part, q, q)) "echo \"WROTE=$(wc -c < %s | tr -d ' ')\"\n" % (b64, part, part, q, q))
@operation("write")
def write_text(self, path, text): def write_text(self, path, text):
data = self.b64(text) data = self.b64(text)
size = len(text.encode("utf-8")) size = len(text.encode("utf-8"))
if len(data) <= self.CHUNK: if len(data) <= self.CHUNK or self.channel.stdin: # stdin carries the script: no argv limit
out = self.sh(self.put_script(path, data), marker="WROTE=") out = self.sh(self.put_script(path, data), marker="WROTE=")
else: # append chunks to <path>.b64, decode on the last call else: # append chunks to <path>.b64, decode on the last call
q, qb = self.quote(path), self.quote(path + ".b64") q, qb = self.quote(path), self.quote(path + ".b64")
@@ -279,6 +430,7 @@ class PosixWarpgateTransport:
if re.findall(r"WROTE=(\d+)", out) != [str(size)]: if re.findall(r"WROTE=(\d+)", out) != [str(size)]:
raise TransportError("upload of %s: size mismatch (%s)" % (path, out.strip()[-200:])) raise TransportError("upload of %s: size mismatch (%s)" % (path, out.strip()[-200:]))
@operation("read")
def read_text(self, path): def read_text(self, path):
q = self.quote(path) q = self.quote(path)
out = self.sh("if [ -f %s ]; then printf B64=; base64 < %s | tr -d '\\n'; echo; else echo MISSING; fi" % (q, q), out = self.sh("if [ -f %s ]; then printf B64=; base64 < %s | tr -d '\\n'; echo; else echo MISSING; fi" % (q, q),
@@ -289,6 +441,7 @@ class PosixWarpgateTransport:
raise TransportError("read %s: unexpected relay output" % path) raise TransportError("read %s: unexpected relay output" % path)
return _b64_payload(out, "B64") return _b64_payload(out, "B64")
@operation("probe")
def probe(self, run_dir, names): def probe(self, run_dir, names):
d = self.quote(run_dir) d = self.quote(run_dir)
out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo" out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo"
@@ -302,13 +455,33 @@ class PosixWarpgateTransport:
raise TransportError("relay %s: unexpected output" % self.target) raise TransportError("relay %s: unexpected output" % self.target)
return text return text
@operation("grep")
def grep_first(self, path, needle): def grep_first(self, path, needle):
line = self._b64_cmd("grep -F -m1 -e %s %s" % (self.quote(needle), self.quote(path))) line = self._b64_cmd("grep -F -m1 -e %s %s" % (self.quote(needle), self.quote(path)))
return line.rstrip("\n") or None return line.rstrip("\n") or None
@operation("tail")
def tail(self, path, n): def tail(self, path, n):
return self._b64_cmd("tail -n %d %s" % (int(n), self.quote(path))).rstrip("\n") return self._b64_cmd("tail -n %d %s" % (int(n), self.quote(path))).rstrip("\n")
def fetch_script(self, run_dir, ops):
s = "d=%s\n" % self.quote(run_dir)
for i, o in enumerate(ops):
f = "\"$d\"/%s" % self.quote(o[1])
if o[0] == "read":
s += ("if [ -f %s ]; then printf F%d=; base64 < %s | tr -d '\\n'; echo; else echo M%d; fi\n"
% (f, i, f, i))
else:
cmd = ("grep -F -m1 -e %s %s" % (self.quote(o[2]), f) if o[0] == "grep"
else "tail -n %d %s" % (int(o[2]), f))
s += "printf F%d=; { %s; } 2>/dev/null | base64 | tr -d '\\n'; echo\n" % (i, cmd)
return s + "echo FETCH=ok\n"
@operation("fetch")
def fetch(self, run_dir, ops):
"""Several reads of one run directory in one call (collect)."""
return parse_fetch(self.sh(self.fetch_script(run_dir, ops), timeout=200, marker="FETCH=ok"), ops, run_dir)
# ---------------------------------------------------------------- processes # ---------------------------------------------------------------- processes
def runner(self, run_dir, cwd, command, stdout_name): def runner(self, run_dir, cwd, command, stdout_name):
return "run.sh", POSIX_RUNNER.format(cwd=self.quote(cwd), d=self.quote(run_dir), command=command, return "run.sh", POSIX_RUNNER.format(cwd=self.quote(cwd), d=self.quote(run_dir), command=command,
@@ -323,13 +496,15 @@ class PosixWarpgateTransport:
"echo $! > launch.pid\n" "echo $! > launch.pid\n"
"echo LAUNCHED=$!\n" % (d, DETACH[self.os] % self.quote(script_name))) "echo LAUNCHED=$!\n" % (d, DETACH[self.os] % self.quote(script_name)))
@operation("launch")
def launch(self, task, script_path, run_dir): def launch(self, task, script_path, run_dir):
self.sh(self.launch_script(run_dir, posixpath.basename(script_path)), marker="LAUNCHED=") self.sh(self.launch_script(run_dir, posixpath.basename(script_path)), marker="LAUNCHED=")
@operation("start")
def start_run(self, task, run_dir, files, script_name): def start_run(self, task, run_dir, files, script_name):
"""mkdir + clean, write `files` {name: text}, launch `script_name` -- one relay call unless a file is """mkdir + clean, write `files` {name: text}, launch `script_name` -- one call; on the relay a file larger
larger than CHUNK (those go up first through write_text).""" than CHUNK goes up first through write_text."""
big = {n: t for n, t in files.items() if len(self.b64(t)) > self.CHUNK} big = {} if self.channel.stdin else {n: t for n, t in files.items() if len(self.b64(t)) > self.CHUNK}
script = self.mkdir_script(run_dir, clean=True) script = self.mkdir_script(run_dir, clean=True)
if big: if big:
self.mkdir(run_dir, clean=True) self.mkdir(run_dir, clean=True)
@@ -359,23 +534,26 @@ class PosixWarpgateTransport:
"kill -KILL -- \"-$p\" 2>/dev/null; pkill -KILL -s \"$p\" 2>/dev/null\n" "kill -KILL -- \"-$p\" 2>/dev/null; pkill -KILL -s \"$p\" 2>/dev/null\n"
"sleep 1; echo \"KILLED=ok left=$(pgrep -s \"$p\" 2>/dev/null | wc -l | tr -d ' ')\"\n" % d) "sleep 1; echo \"KILLED=ok left=$(pgrep -s \"$p\" 2>/dev/null | wc -l | tr -d ' ')\"\n" % d)
@operation("kill")
def kill(self, task, run_dir): def kill(self, task, run_dir):
self.sh(self.kill_script(run_dir), marker="KILLED=") self.sh(self.kill_script(run_dir), marker="KILLED=")
def forget(self, task): def forget(self, task):
pass pass
@operation("busy")
def process_count(self, name): def process_count(self, name):
try: try:
out = self.sh("echo COUNT=$(pgrep -x %s | wc -l | tr -d ' ')" % self.quote(name), timeout=60) out = self.sh("echo COUNT=$(pgrep -x %s | wc -l | tr -d ' ')" % self.quote(name), timeout=60)
except Unreached:
raise # a direct-ssh terminal retries this on its fallback
except TransportError: except TransportError:
return None return None
m = re.search(r"COUNT=(\d+)", out) m = re.search(r"COUNT=(\d+)", out)
return int(m.group(1)) if m else None return int(m.group(1)) if m else None
def WarpgateSshTransport(cfg, terminal): def WarpgateSshTransport(cfg, terminal, channel=None, log=None, label="warpgate"):
"""Factory: the transport class for a warpgate-ssh terminal follows its `os`.""" """Factory: the transport class for a remote terminal follows its `os`; the channel defaults to Warpgate."""
if cfg.terminals[terminal]["os"] == "windows": cls = WindowsWarpgateTransport if cfg.terminals[terminal]["os"] == "windows" else PosixWarpgateTransport
return WindowsWarpgateTransport(cfg, terminal) return cls(cfg, terminal, channel=channel, log=log, label=label)
return PosixWarpgateTransport(cfg, terminal)
+6 -1
View File
@@ -12,6 +12,8 @@ agentd:
max_rounds_per_thread: 8 # rebuild the executor thread (with a handoff summary) after this many rounds max_rounds_per_thread: 8 # rebuild the executor thread (with a handoff summary) after this many rounds
comment_max_chars: 6000 comment_max_chars: 6000
cli_terminal: oregon # identity used by `agentd new/comment/state/assign` cli_terminal: oregon # identity used by `agentd new/comment/state/assign`
# ssh_key: AGENTD_SSH_KEY # direct-ssh: Vault key / env var of the dedicated private key (default)
# ssh_dir: /run/plane-agentd/ssh # direct-ssh: ControlMaster sockets + generated known_hosts (default state_dir/ssh)
# Plane webhook receiver (`agentd webhook-serve`, docs/runbook.md "Webhook"); leave the section out to run on cron only. # Plane webhook receiver (`agentd webhook-serve`, docs/runbook.md "Webhook"); leave the section out to run on cron only.
webhook: webhook:
@@ -27,7 +29,10 @@ notify: {kind: none} # notification hook: only `none` (no-op) e
terminals: terminals:
win10: win10:
transport: warpgate-ssh transport: warpgate-ssh # or direct-ssh over Tailscale (design §9, docs/deploy-lxc.md "直连 SSH"):
# transport: direct-ssh
# ssh: {host: 100.123.209.19, user: administrator, port: 22, host_key: "ssh-ed25519 AAAA..."} # pinned host key
# fallback: warpgate-ssh # used only when the direct path fails before the command ran
target: win10 target: win10
os: windows os: windows
launch: schtasks launch: schtasks
+19 -7
View File
@@ -1,5 +1,5 @@
# plane-agentd live configuration on the LXC agentd-01 (/opt/plane-agentd, runs as root; docs/deploy-lxc.md). # plane-agentd live configuration on the LXC agentd-01 (/opt/plane-agentd, runs as root; docs/deploy-lxc.md).
# No secrets: token_key values are Vault key names; the tokens are injected by # No secrets: token_key / ssh_key values are Vault key names; the values are injected by
# `agent-access.py exec --secret-env KEY=KEY` (cron/agentd-tick-lxc.sh). Rollback to oregon: config/agentd.oregon.yaml. # `agent-access.py exec --secret-env KEY=KEY` (cron/agentd-tick-lxc.sh). Rollback to oregon: config/agentd.oregon.yaml.
plane: {base_url: https://plane.kappiee.com, workspace: kappiee} plane: {base_url: https://plane.kappiee.com, workspace: kappiee}
vault: {client: ~/.local/share/server-access-client/agent-access.py, config: https://nas.tail9531b.ts.net:8443/identity/agent.json} vault: {client: ~/.local/share/server-access-client/agent-access.py, config: https://nas.tail9531b.ts.net:8443/identity/agent.json}
@@ -12,11 +12,15 @@ agentd:
max_rounds_per_thread: 8 max_rounds_per_thread: 8
comment_max_chars: 6000 comment_max_chars: 6000
cli_terminal: oregon cli_terminal: oregon
# direct-ssh (PAD-12, docs/deploy-lxc.md "直连 SSH"): the dedicated key comes from Vault AGENTD_SSH_KEY via
# --secret-env; ControlMaster sockets and the known_hosts generated from the pinned host keys live in ssh_dir
ssh_key: AGENTD_SSH_KEY
ssh_dir: /run/plane-agentd/ssh
# Plane webhook receiver (`agentd webhook-serve`, systemd plane-agentd-webhook.service; docs/runbook.md "Webhook"). # Plane webhook receiver (`agentd webhook-serve`, systemd plane-agentd-webhook.service; docs/runbook.md "Webhook").
# One workspace-level Plane webhook (work items + comments) -> POST http://10.77.10.249:8790/plane -> one tick. # One workspace-level Plane webhook (work items + comments) -> POST http://10.77.10.249:8790/plane -> one tick.
webhook: webhook:
bind: 10.77.10.249 # LAN only (Plane VM 10.77.10.68 is on the same /24) bind: 10.77.10.249 # LAN only (Plane VM 506 10.77.10.69 is on the same /24)
port: 8790 port: 8790
path: /plane path: /plane
secret_key: PLANE_WEBHOOK_SECRET # Vault key name; the value is injected by agent-access.py exec --secret-env secret_key: PLANE_WEBHOOK_SECRET # Vault key name; the value is injected by agent-access.py exec --secret-env
@@ -30,8 +34,13 @@ notify: {kind: none} # notification hook, no-op (no ntfy servic
terminals: terminals:
win10: win10:
transport: warpgate-ssh # direct over Tailscale (PAD-12); Warpgate only when the direct path fails before a command ran.
target: win10 # Rollback: transport: warpgate-ssh (ssh/fallback may stay, they are ignored then)
transport: direct-ssh
ssh: {host: 100.123.209.19, user: administrator, port: 22,
host_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzLX7QNHDFYwTgY4wNl0ihZryIyKgpp9HPpPDWbrTX3"}
fallback: warpgate-ssh
target: win10 # Warpgate target of the fallback
os: windows os: windows
launch: schtasks launch: schtasks
repo: 'C:\Users\Administrator\Documents\GwentAssistantProject\gwent' repo: 'C:\Users\Administrator\Documents\GwentAssistantProject\gwent'
@@ -44,9 +53,12 @@ terminals:
capabilities: [android, game] capabilities: [android, game]
executors: [codex] executors: [codex]
oregon: oregon:
# the dispatcher (LXC, no agent CLIs) reaches oregon through Warpgate like any other host (design §9) # the dispatcher (LXC, no agent CLIs) reaches oregon directly over Tailscale, Warpgate as fallback (design §9)
transport: warpgate-ssh transport: direct-ssh
target: oregon # agent-access.py target name ssh: {host: 100.127.58.78, user: ubuntu, port: 22,
host_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICUoI6W7npX+h/kHcXX3iy2XAWbNusrOjg28Xed75wWr"}
fallback: warpgate-ssh
target: oregon # agent-access.py target name (fallback)
os: linux os: linux
launch: nohup launch: nohup
repo: /home/ubuntu/Documents/wjj/project/gwent repo: /home/ubuntu/Documents/wjj/project/gwent
+1
View File
@@ -44,4 +44,5 @@ worker 跑在 Claude 所在的 Linux 机(`gwent-design/tools/board-worker-plan
## 通用规则 ## 通用规则
- 消息要短、要有结论;路径用仓库相对路径;大图只贴缩略拼图(≤1600px)。 - 消息要短、要有结论;路径用仓库相对路径;大图只贴缩略拼图(≤1600px)。
- 不在评论里贴 token、密码、完整日志。 - 不在评论里贴 token、密码、完整日志。
- 不在 win10 的 checkout 上执行任何 git 写操作(commit / checkout / reset / pull / stash);产物留在 live 或 stage,入库由 Claude 侧完成。
- 有疑问优先按任务书里的默认值继续并在评论里说明,而不是停下等人。 - 有疑问优先按任务书里的默认值继续并在评论里说明,而不是停下等人。
+16 -2
View File
@@ -33,7 +33,9 @@
"interval": {"type": "integer", "minimum": 10}, "interval": {"type": "integer", "minimum": 10},
"max_rounds_per_thread": {"type": "integer", "minimum": 1}, "max_rounds_per_thread": {"type": "integer", "minimum": 1},
"comment_max_chars": {"type": "integer", "minimum": 500}, "comment_max_chars": {"type": "integer", "minimum": 500},
"cli_terminal": {"type": "string"} "cli_terminal": {"type": "string"},
"ssh_dir": {"type": "string"},
"ssh_key": {"type": "string"}
} }
}, },
"webhook": { "webhook": {
@@ -71,8 +73,20 @@
"required": ["transport", "os", "launch", "repo", "plane_user", "token_key", "executors"], "required": ["transport", "os", "launch", "repo", "plane_user", "token_key", "executors"],
"additionalProperties": false, "additionalProperties": false,
"properties": { "properties": {
"transport": {"enum": ["local", "warpgate-ssh"]}, "transport": {"enum": ["local", "warpgate-ssh", "direct-ssh"]},
"target": {"type": "string"}, "target": {"type": "string"},
"ssh": {
"type": "object",
"required": ["host", "user", "host_key"],
"additionalProperties": false,
"properties": {
"host": {"type": "string"},
"user": {"type": "string"},
"port": {"type": "integer", "minimum": 1},
"host_key": {"type": "string"}
}
},
"fallback": {"enum": ["warpgate-ssh"]},
"os": {"enum": ["linux", "darwin", "windows"]}, "os": {"enum": ["linux", "darwin", "windows"]},
"launch": {"enum": ["subprocess", "nohup", "schtasks"]}, "launch": {"enum": ["subprocess", "nohup", "schtasks"]},
"repo": {"type": "string"}, "repo": {"type": "string"},
+32 -8
View File
@@ -1,13 +1,37 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# LXC agentd-01 cron entry (*/2, root): one agentd tick under server-access exec (Plane tokens only in the # LXC agentd-01 cron entry (*/2, root): one agentd tick under server-access exec (Plane tokens only in the
# process env), with a lock. The outer flock skips a tick while the previous one still runs; agentd also holds # process env), with a lock. The flock skips a tick while the previous one still runs; agentd also holds
# its own state lock. Logs: /var/log/plane-agentd/{agentd.log,cron.log}; state: /var/lib/plane-agentd. # its own state lock. Logs: /var/log/plane-agentd/{agentd.log,cron.log}; state: /var/lib/plane-agentd.
# Also the webhook receiver's tick command (agentd/webhook.py): exit 75 = lock held (flock -E 75, or agentd's own # Also the webhook receiver's tick command (agentd/webhook.py): exit 75 = lock held (flock -E 75, or agentd's own
# state lock), the receiver then retries. # state lock), the receiver then retries.
exec flock -n -E 75 /var/lib/plane-agentd/cron.lock \ # AGENTD_SSH_KEY = the direct-ssh key (PAD-12). agent-access.py has no optional secrets: when Vault cannot hand it
timeout 900 python3 -I /root/.local/share/server-access-client/agent-access.py \ # out (credential_unavailable / credential_invalid, nothing ran) the tick is repeated once without it, so the
--config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \ # direct-ssh terminals use their warpgate fallback instead of every tick failing. The tick's stderr is appended to
--secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \ # cron.log after its stdout (it has to be inspected for that error first).
--secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN --timeout 850 -- \ LOCK=/var/lib/plane-agentd/cron.lock
python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml \ LOG=/var/log/plane-agentd/cron.log
tick --once >> /var/log/plane-agentd/cron.log 2>&1 CLIENT=(python3 -I /root/.local/share/server-access-client/agent-access.py --config https://nas.tail9531b.ts.net:8443/identity/agent.json)
exec 9>>"$LOCK"
flock -n -E 75 9 || exit $?
tick() { # "$@" = extra exec arguments; fd 9 (the lock) is not handed to the tick's children
timeout 900 "${CLIENT[@]}" exec \
--secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \
--secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN \
"$@" --timeout 850 -- \
python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once 9>&-
}
err=$(mktemp) || exit 1
trap 'rm -f "$err"' EXIT
tick --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY >>"$LOG" 2>"$err"
rc=$?
cat "$err" >>"$LOG"
case "$rc:$(cat "$err")" in
"1:access-hub: credential_unavailable" | "1:access-hub: credential_invalid")
echo "$(date -Is) AGENTD_SSH_KEY not available from Vault: tick repeated without it (direct-ssh terminals use their warpgate fallback)" >>"$LOG"
tick >>"$LOG" 2>&1
rc=$? ;;
esac
exit $rc
+78 -5
View File
@@ -1,6 +1,6 @@
# 在精简 LXC 上部署 agentd(design §9 阶段 3) # 在精简 LXC 上部署 agentd(design §9 阶段 3)
调度器跑在一台**不装任何 agent CLI** 的 Debian 12 LXC(`agentd-01`)上;所有终端(win10、oregon、mac…)都经 Warpgate SSH 远程启动(`transport: warpgate-ssh`)。LXC 只做调度、回帖、通知。 调度器跑在一台**不装任何 agent CLI** 的 Debian 12 LXC(`agentd-01`)上;所有终端(win10、oregon、mac…)都是远程启动:经 Warpgate SSH(`transport: warpgate-ssh`),或经 Tailscale 直连并以 Warpgate 兜底(`transport: direct-ssh`,PAD-12,§9)。LXC 只做调度、回帖、通知。
实际部署(2026-10-21,PAD-11):`agentd-01` = PVE CT 600,Debian 12,tailnet `100.69.78.124`,Warpgate target `agentd-01`,登录 **root**(Vault skill `ssh-agentd-01`;没有另建部署用户,agentd 以 root 运行)。下面是实际做过的步骤;从 oregon 发命令都经 `agent-access.py … run --target agentd-01 --timeout N -- bash -c '…'`。 实际部署(2026-10-21,PAD-11):`agentd-01` = PVE CT 600,Debian 12,tailnet `100.69.78.124`,Warpgate target `agentd-01`,登录 **root**(Vault skill `ssh-agentd-01`;没有另建部署用户,agentd 以 root 运行)。下面是实际做过的步骤;从 oregon 发命令都经 `agent-access.py … run --target agentd-01 --timeout N -- bash -c '…'`。
@@ -9,17 +9,25 @@ LXC 交付时已有 `python3`(3.11)、`python3-yaml`、`openssh-client`、`g
```bash ```bash
apt-get update && apt-get install -y python3 python3-yaml openssh-client git cron ca-certificates apt-get update && apt-get install -y python3 python3-yaml openssh-client git cron ca-certificates
``` ```
`jsonschema` 可选,没有就用内置子集校验器。 `jsonschema` 可选,没有就用内置子集校验器。`direct-ssh` 要用的 `ssh`、`ssh-agent`、`ssh-add`、`ssh-keygen` 都在 `openssh-client` 包里(Debian 12:OpenSSH 9.2)。
## 2. 网络 ## 2. 网络
| 去向 | 用途 | | 去向 | 用途 |
|---|---| |---|---|
| `nas.tail9531b.ts.net:8443`(tailnet) | Agent Vault 身份配置 `identity/agent.json`(含 vault token 与 Warpgate 主机公钥)→ LXC 已在 tailnet | | `nas.tail9531b.ts.net:8443`(tailnet) | Agent Vault 身份配置 `identity/agent.json`(含 vault token 与 Warpgate 主机公钥)→ LXC 已在 tailnet |
| Vault(身份配置里的地址) | 取 Plane token、Warpgate 密码 | | Vault(身份配置里的地址) | 取 Plane token、Warpgate 密码 |
| Warpgate SSH `115.28.210.101:2222` | 所有终端的中继(LXC → win10 / oregon) | | Warpgate SSH `115.28.210.101:2222` | `warpgate-ssh` 终端的中继,以及 `direct-ssh` 的兜底(LXC → win10 / oregon) |
| tailnet:win10 `100.123.209.19:22`、oregon `100.127.58.78:22` | `direct-ssh` 直连(PAD-12,§9) |
| `https://plane.kappiee.com` | Plane API | | `https://plane.kappiee.com` | Plane API |
| `https://git.kappiee.com` | `git pull`(公开仓库,无需凭据) | | `https://git.kappiee.com` | `git pull`(公开仓库,无需凭据) |
## 2.1 到 NAS 的直连路径(OPNsense 静态端口 NAT,2026-10-21)
PVE 站点是双层 NAT(OPNsense → 192.168.1.1),默认的随机源端口会让 tailscale 打洞失败、只能走 DERP(此前取 vault 身份端点 30/30 超时)。修法:
- **OPNsense 出站 NAT 规则**(用户在 GUI 里添加,因为 opnsense 中继账号 kappiee 无权限,我们只读):描述 `Tailscale agentd-01 static-port`,接口 WAN,源 `10.77.10.249/32`,目标 any,协议 UDP,勾选"静态端口"(`staticnatport 1`)。从 kpe-edge-01 的 `Tailscale edge static-port`(10.77.10.110/32)规则克隆而来。加规则后要清掉旧状态(Firewall → Diagnostics → States,按源 IP)。
- **LXC tailscaled 用 UDP 41642**:`/etc/default/tailscaled` 里 `PORT="41642"`(备份 `.bak-20261021144838`),必须与 edge 的 41641 不同,否则两台机器的静态端口映射会在同一 NAS 端点上撞元组。
- **验证**(LXC 上):`tailscale ping 100.89.188.28` 应得 `pong from nas … via 223.102.82.206:1549`(不是 `via DERP`);`tailscale netcheck` 应 `UDP: true`、`MappingVariesByDestIP: false`;身份端点循环(注意 `agent.json` 含 bearer token,只看状态码和耗时):`for i in $(seq 30); do curl -s -o /dev/null -w '%{http_code} %{time_total}\n' --max-time 15 https://nas.tail9531b.ts.net:8443/identity/agent.json; done`。2026-10-21 实测 30/30 成功、中位数 0.089 s,ping 5 ms。
- **回滚**:在 OPNsense GUI 删掉该规则(Firewall → NAT → Outbound)并清状态;LXC 上还原 `/etc/default/tailscaled.bak-20261021144838` 后 `systemctl restart tailscaled`。回滚后路径退回 DERP,调度器应同时切回 oregon(runbook)。
## 3. Agent Vault 客户端 ## 3. Agent Vault 客户端
只复制程序文件(客户端没有落盘的 token 缓存;oregon 目录里的 `agent-service-check.py` 是运维工具,不需要): 只复制程序文件(客户端没有落盘的 token 缓存;oregon 目录里的 `agent-service-check.py` 是运维工具,不需要):
```bash ```bash
@@ -41,7 +49,7 @@ mkdir -p /var/lib/plane-agentd /var/log/plane-agentd && chmod 700 /var/lib/plane
python3 -m unittest && python3 cli/agentd check-config python3 -m unittest && python3 cli/agentd check-config
``` ```
不需要本地配置文件,全部在仓库里(部署可复现): 不需要本地配置文件,全部在仓库里(部署可复现):
- `config/agentd.yaml` = agentd-01 的实时配置:`state_dir: /var/lib/plane-agentd`(状态、心跳、锁),`log_dir: /var/log/plane-agentd`(`agentd.log`),`helpers_dir: tools/win10`,`vault.client: ~/.local/share/server-access-client/agent-access.py`(root 下即 `/root/...`)。终端 win10(warpgate-ssh/windows/schtasks)和 oregon(warpgate-ssh/linux/nohup,`run_root: /home/ubuntu/.gwent-board/agentd-runs`,`bins.claude`);没有 local 终端。 - `config/agentd.yaml` = agentd-01 的实时配置:`state_dir: /var/lib/plane-agentd`(状态、心跳、锁),`log_dir: /var/log/plane-agentd`(`agentd.log`),`helpers_dir: tools/win10`,`vault.client: ~/.local/share/server-access-client/agent-access.py`(root 下即 `/root/...`)。终端 win10(direct-ssh + 兜底 warpgate-ssh/windows/schtasks)和 oregon(direct-ssh + 兜底 warpgate-ssh/linux/nohup,`run_root: /home/ubuntu/.gwent-board/agentd-runs`,`bins.claude`);没有 local 终端。`agentd.ssh_key: AGENTD_SSH_KEY`,`agentd.ssh_dir: /run/plane-agentd/ssh`(ControlMaster 套接字和生成的 known_hosts,agentd 自己建,权限 0700)。
- `tools/win10/win10-ps.sh`、`win10-put.sh`:win10 helper(vault 客户端目录取 `AGENT_ACCESS_HOME`,默认 `~/.local/share/server-access-client`;身份取 `AGENT_ACCESS_CONFIG`)。相对 `helpers_dir` 按仓库根解析。 - `tools/win10/win10-ps.sh`、`win10-put.sh`:win10 helper(vault 客户端目录取 `AGENT_ACCESS_HOME`,默认 `~/.local/share/server-access-client`;身份取 `AGENT_ACCESS_CONFIG`)。相对 `helpers_dir` 按仓库根解析。
- `config/protocols/gwent-board-protocol.md`:看板协议(拼进每轮 prompt,LXC 本机读取)。来源是 gwent 仓库 `design/board-protocol.md`,**协议更新时复制过来提交**,然后 LXC `git pull`。 - `config/protocols/gwent-board-protocol.md`:看板协议(拼进每轮 prompt,LXC 本机读取)。来源是 gwent 仓库 `design/board-protocol.md`,**协议更新时复制过来提交**,然后 LXC `git pull`。
- `config/agentd.oregon.yaml`:oregon 作调度器时的配置(回滚用,见 runbook)。 - `config/agentd.oregon.yaml`:oregon 作调度器时的配置(回滚用,见 runbook)。
@@ -60,7 +68,7 @@ python3 -m unittest && python3 cli/agentd check-config
``` ```
*/2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh */2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh
``` ```
`cron/agentd-tick-lxc.sh` = `flock -n /var/lib/plane-agentd/cron.lock timeout 900 python3 -I /root/.local/share/server-access-client/agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once >> /var/log/plane-agentd/cron.log 2>&1`。 `cron/agentd-tick-lxc.sh` = 拿 `/var/lib/plane-agentd/cron.lock`(`flock -n -E 75`)后执行 `timeout 900 python3 -I /root/.local/share/server-access-client/agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once`,输出追加到 `/var/log/plane-agentd/cron.log`;Vault 取不到 `AGENTD_SSH_KEY` 时不带它重跑一次(§9)。
注意:经中继装 crontab 时别在 `set -e` 下写 `(crontab -l | grep -v …; echo …) | crontab -`——没有旧 crontab 时 grep 返回 1,子 shell 提前退出,装进去的是空 crontab。直接 `printf '%s\n' '<行>' | crontab -`。 注意:经中继装 crontab 时别在 `set -e` 下写 `(crontab -l | grep -v …; echo …) | crontab -`——没有旧 crontab 时 grep 返回 1,子 shell 提前退出,装进去的是空 crontab。直接 `printf '%s\n' '<行>' | crontab -`。
5. 验证:`/var/lib/plane-agentd/heartbeat.json` 的 `at` 每 2 分钟更新。 5. 验证:`/var/lib/plane-agentd/heartbeat.json` 的 `at` 每 2 分钟更新。
6. 冒烟经 LXC 调度器:在 oregon 上 `cli/agentd smoke GWENT oregon --watch`(`--watch` = 只建项并等待,不在本机 tick;oregon 上 `cli/agentd` 默认读 `config/agentd.yaml`,CLI 只需要 Plane token)。 6. 冒烟经 LXC 调度器:在 oregon 上 `cli/agentd smoke GWENT oregon --watch`(`--watch` = 只建项并等待,不在本机 tick;oregon 上 `cli/agentd` 默认读 `config/agentd.yaml`,CLI 只需要 Plane token)。
@@ -100,3 +108,68 @@ python3 -m unittest && python3 cli/agentd check-config
远程 shell 是目标账户的登录 shell(bash/zsh 均可,agentd 发的是 `bash -c '<script>'`);终端需有 `bash base64 nohup pgrep pkill`(Linux 另需 `setsid`,macOS 需 `perl`)。 远程 shell 是目标账户的登录 shell(bash/zsh 均可,agentd 发的是 `bash -c '<script>'`);终端需有 `bash base64 nohup pgrep pkill`(Linux 另需 `setsid`,macOS 需 `perl`)。
4. Plane:建用户 `mac`,token 存 Vault 键 `PLANE_MAC_API_TOKEN`;项目 `routing.by_assignee` 加 `mac: mac`,`default_executor` 加 `mac: claude-code`;`cron/agentd-tick-lxc.sh` 加 `--secret-env PLANE_MAC_API_TOKEN=PLANE_MAC_API_TOKEN`。 4. Plane:建用户 `mac`,token 存 Vault 键 `PLANE_MAC_API_TOKEN`;项目 `routing.by_assignee` 加 `mac: mac`,`default_executor` 加 `mac: claude-code`;`cron/agentd-tick-lxc.sh` 加 `--secret-env PLANE_MAC_API_TOKEN=PLANE_MAC_API_TOKEN`。
5. 提交、推送、LXC `git pull`;`cli/agentd check-config`,然后 `cli/agentd smoke GWENT mac --watch`。 5. 提交、推送、LXC `git pull`;`cli/agentd check-config`,然后 `cli/agentd smoke GWENT mac --watch`。
## 8. Webhook 接收端(PAD-7)
1. 网络:Plane VM(PVE VM 506 plane-02,10.77.10.69,prime-cli 安装;没有 VM 505)与 LXC 10.77.10.249 同在 10.77.10.0/24,接收端只听 `10.77.10.249:8790`(LXC 内无防火墙)。**Plane 侧前提**:Plane 默认拒绝内网 webhook 地址("The webhook URL is not allowed. Provide a publicly reachable HTTP or HTTPS URL."),要在 Plane 的 api(创建时检查)与 worker(每次投递检查)容器环境里设 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(逗号分隔 IP/CIDR)并重建相关容器(design §10)。Plane 3.3.3 没有 compose override 文件:prime-cli 由 `/opt/plane/plane.env` 渲染 `/var/plane/docker-compose.yml`,两处都已有空的 `WEBHOOK_ALLOWED_IPS`(compose 里每个服务一行 `WEBHOOK_ALLOWED_IPS: ""`)。做法:备份两个文件为 `.bak-pad7-<时间戳>` → 把 `plane.env` 的 `WEBHOOK_ALLOWED_IPS=` 与渲染 compose 里的空值改为 `10.77.10.249/32` → `docker compose -p plane -f /var/plane/docker-compose.yml up -d --no-deps api worker webhook-consumer outbox-poller automation-consumer beat-worker`(先 `--dry-run`,确认只重建这 6 个;商业版的 v2 投递走 webhook-consumer/worker,创建时校验在 api;不要用 prime-cli,它要 TTY。compose 里 `WEBHOOK_ALLOWED_HOSTS: ${WEB_URL}` 不带 env 文件时为空,prime-cli 起的容器也是空值,属现状)。**状态(2026-10-22):已应用**(备份 `*.bak-pad7-20261022-011453`,6 个容器 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`),webhook `c039c471-e78a-49c9-b009-5cf77a389ad6` 已建并启用(v2;签名与接收端一致,见 design §10),接收端 active/enabled。纯 http 本身是允许的(http/https 都行)。
2. 密钥:Plane 建 webhook 时生成、只显示一次 → `tools/plane-webhook.py create http://10.77.10.249:8790/plane` 直接写进 Vault `PLANE_WEBHOOK_SECRET`(建出来是停用状态;在 oregon 上以 kappiee 管理员会话运行,见 runbook "Webhook")。核对长度:`agent-access.py … exec --secret-env S=PLANE_WEBHOOK_SECRET -- python3 -c 'import os;print(len(os.environ["S"]))'`。
3. 代码与服务(LXC):
```bash
cd /opt/plane-agentd && flock /var/lib/plane-agentd/cron.lock git pull
cp systemd/plane-agentd-webhook.service /etc/systemd/system/ && systemctl daemon-reload
systemctl enable --now plane-agentd-webhook
curl -s http://10.77.10.249:8790/healthz # {"ok": true, ...}
tail /var/log/plane-agentd/webhook.log # listening on http://10.77.10.249:8790/plane (tenants: GWENT)
```
单元文件复用 cron 脚本的 vault 客户端调用方式:`python3 -I /root/.local/share/server-access-client/agent-access.py --config <identity> exec --secret-env PLANE_WEBHOOK_SECRET=PLANE_WEBHOOK_SECRET --timeout 315360000 -- python3 /opt/plane-agentd/cli/agentd … webhook-serve --bind 10.77.10.249 --port 8790`,`Restart=always`。tick 由接收端经 `systemd-run --scope … cron/agentd-tick-lxc.sh` 拉起(同一把 `cron.lock`,`flock -n -E 75`)。
4. 启用 Plane 侧:`tools/plane-webhook.py patch <id> is_active=true`,然后 `systemctl restart plane-agentd-webhook`(若第 2 步之后密钥变过)。
5. 验证:随便改一个 GWENT 工作项 → `webhook.log` 出现 `decision=tick` 和 `tick start`/`tick done`;错签名请求得 401 且没有 tick;`journalctl -u cron` 里 cron 仍每 2 分钟一次;`systemctl restart plane-agentd-webhook` 后 `/healthz` 恢复。
## 9. 直连 SSH(PAD-12)
LXC 经 Tailscale 直连终端的 sshd,不经过 Warpgate(design §9"终端通道")。每个终端单独配置,Warpgate 留作兜底。前提:终端在 tailnet 上,LXC(`100.69.78.124`)能连到它的 22 端口。实测:win10 `100.123.209.19`(Windows OpenSSH,账号 administrator,直连 RTT 27 ms),oregon `100.127.58.78`(账号 ubuntu,RTT 233 ms)。
**上线顺序**:`cron/agentd-tick-lxc.sh` 带着 `--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY`。agent-access.py 没有可选密钥,取不到键时报 `credential_unavailable`,整个 exec 失败;cron 脚本认出这种情况后会去掉这个参数再跑一次(cron.log 记一行 `AGENTD_SSH_KEY not available from Vault …`),所以先 pull 也不会停派单,只是直连终端一直走兜底。仍然建议第 1–3 步做完再 pull,免得每个 tick 多取一次 Vault。
1. **生成专用密钥**(在 LXC 上;私钥只进 Vault,不留在磁盘上):
```bash
umask 077; d=$(mktemp -d -p /dev/shm) # tmpfs:私钥不落到磁盘(磁盘上 shred 在日志式/写时复制文件系统上不可靠)
ssh-keygen -q -t ed25519 -N '' -C plane-agentd@agentd-01 -f "$d/agentd"
cat "$d/agentd.pub" # 公钥,下面两步要用
```
私钥 `$d/agentd` 原样(多行文本)写进 Vault 键 **`AGENTD_SSH_KEY`**:在 Vault UI 里粘贴,或者用与 `tools/plane-webhook.py` 的 `vault_store()` 相同的接口(`POST /v1/credentials`,agent 身份),读文件直接写,不打印。值里的换行被转义成 `\n` 也可以,agentd 会还原。写完删掉:`rm -rf "$d"`(在 tmpfs 上,删掉即可)。核对时只看长度:
`agent-access.py --config <identity> exec --secret-env K=AGENTD_SSH_KEY --timeout 60 -- python3 -c 'import os;print(len(os.environ["K"]))'`(ed25519 私钥约 400 字符)。
2. **oregon**(经 Warpgate 操作):在 `~ubuntu/.ssh/authorized_keys` 追加一行(目录 700、文件 600):
```
restrict,from="100.69.78.124" ssh-ed25519 AAAA…(上一步的公钥) plane-agentd@agentd-01
```
`restrict` 关掉所有转发、pty 和 `~/.ssh/rc`(agentd 用 `-T`,不需要 pty),不必再单独写 `no-port-forwarding` 等;`from=` 只放行 LXC 的 tailnet 地址。sshd 不认识 `restrict`(OpenSSH 7.2 之前)时会把整行判为无效:认证失败、终端走兜底,不会放宽权限;第 5 步的 ping 能看出来(`sshd -V` 或 Windows 上 `sshd.exe -V` 看版本)。可选的再一层:Tailscale ACL 只允许 LXC(`100.69.78.124` 或它的 tag)访问 win10/oregon 的 `:22`,这样密钥泄露了也没用。
3. **win10**(经 Warpgate,`tools/win10/win10-ps.sh`):Administrator 属于 Administrators 组,所以 sshd 读的是 `C:\ProgramData\ssh\administrators_authorized_keys`,不是用户目录下的文件。用 ASCII 追加同样的一行(别覆盖已有的行),再把 ACL 收紧到只有 Administrators 和 SYSTEM(用 SID,不受系统语言影响),否则 sshd 会忽略这个文件:
```powershell
$f = 'C:\ProgramData\ssh\administrators_authorized_keys'
$line = 'restrict,from="100.69.78.124" ssh-ed25519 AAAA… plane-agentd@agentd-01'
if ((Test-Path $f) -and (Get-Content -Raw $f) -and -not (Get-Content -Raw $f).EndsWith("`n")) { $line = "`r`n" + $line } # 原文件末尾没换行时,别把新行粘到最后一把钥匙后面
Add-Content -Path $f -Encoding ascii -Value $line
icacls $f /inheritance:r /grant '*S-1-5-32-544:F' /grant '*S-1-5-18:F'
```
顺便确认 sshd 的默认 shell 是 cmd.exe(`HKLM:\SOFTWARE\OpenSSH` 下没有 `DefaultShell`,或它指向 cmd.exe)。agentd 发的远程命令是 `powershell.exe -NoProfile -NonInteractive -EncodedCommand <引导>`,真正的脚本经 stdin 传过去。外层如果被改成 PowerShell,stdin 可能转不到内层;第 5 步的 ping 能直接发现这个问题。
4. **钉住的主机公钥**已写在 `config/agentd.yaml` 的 `terminals.<t>.ssh.host_key`,经两个通道核对过:Warpgate 读终端上的 `ssh_host_ed25519_key.pub`,与 tailnet 上 `ssh-keyscan -t ed25519` 的结果一致。新加终端照此办理:
```bash
ssh-keyscan -t ed25519 <tailnet ip> # LXC 上,直连
agent-access.py … run --target <t> -- cat /etc/ssh/ssh_host_ed25519_key.pub # 第二通道(Windows:type C:\ProgramData\ssh\ssh_host_ed25519_key.pub)
```
两边一致才能写进配置。agentd 每次调用都用它生成 known_hosts(`<ssh_dir>/known_hosts.<终端>`,别名 `agentd-<终端>`),全局和用户的 known_hosts 都不读。
5. **上线与验证**(在 LXC 上;拿着 cron 锁做,期间 cron 和 webhook 的 tick 会跳过,webhook 会自动重试):
```bash
cd /root/.local/share/server-access-client
flock /var/lib/plane-agentd/cron.lock bash -c '
git -C /opt/plane-agentd pull &&
for t in win10 oregon; do
python3 -I agent-access.py --config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY --timeout 120 -- \
python3 /opt/plane-agentd/cli/agentd ping $t --transport direct-ssh -n 3
done'
```
预期每个终端打印 3 行 `term=<t> via=direct op=ping ms=… rc=0` 和 `ping n: … ms PONG=…`。第 1 次要握手(win10 约 0.5–1.5 s,oregon 约 2–3 s),第 2、3 次复用连接,应明显更快。win10 若是 `timeout`,多半是 stdin 没到内层 PowerShell,见第 3 步。`--transport warpgate-ssh` 可顺带确认兜底仍然通(约 9 s)。
`cli/agentd check-config` 会多出 `direct-ssh …` 行,`cron/exec secret args` 末尾多了 `--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY`。
6. **观察**:下一个 tick 后看 `/var/log/plane-agentd/remote.log`,应该只有 `via=direct` 行;出现 `via=fallback` 说明直连失败了,同时 `agentd.log` 会有一行原因。在 oregon 上 `cli/agentd smoke GWENT win10 --watch`,量"建项 → `[worker] 第 1 轮开始`",目标 ≤ 15 s(原来 91 s)。
7. **回滚**:把终端的 `transport` 改回 `warpgate-ssh`(提交、推送、LXC 上 `git pull`;`ssh`/`fallback` 可以留着)。cron 脚本里的 `--secret-env AGENTD_SSH_KEY=…` 可以保留:agentd 总会把它从子进程环境里去掉。Vault 键被删掉也不会停 tick(脚本会不带它重跑),但每个 tick 会多取一次 Vault,届时最好也删掉这一行。紧急情况下可在 LXC 上 `git -C /opt/plane-agentd checkout 83bc0f2 -- config/agentd.yaml`(旧配置,全部走 Warpgate,新代码照常能用)。要彻底撤销直连授权:删掉两台终端 authorized_keys 里那一行,再在 LXC 上关掉已经建好的 ControlMaster 连接(删 authorized_keys 不会断开它们,紧挨着的 webhook tick 能让它们活过 60 s):`for s in /run/plane-agentd/ssh/*-*; do [ -S "$s" ] && ssh -O exit -o ControlPath="$s" x; done`。
8. **轮换密钥**:照第 1 步生成新钥,先把新公钥加到两台终端,再更新 Vault `AGENTD_SSH_KEY`,然后删掉旧公钥的行,再照第 7 步关掉已有的 ControlMaster 连接(否则它们空闲 60 s 后才退出)。之后新连接就用新钥。改了某个终端钉住的 `host_key` 不需要这一步:套接字名里带着 pin 的指纹,新 pin 自动用新连接。
+34 -14
View File
@@ -3,22 +3,22 @@
Plane 看板驱动的通用 agent 调度服务:任何人/agent 在 Plane 建工作项并指派给一个"终端",agentd 在该终端起对应的 CLI agent(Codex、Claude Code、脚本…)执行,把结果贴回工作项并流转状态。调度是确定性代码,模型只出现在执行器和总指挥里。 Plane 看板驱动的通用 agent 调度服务:任何人/agent 在 Plane 建工作项并指派给一个"终端",agentd 在该终端起对应的 CLI agent(Codex、Claude Code、脚本…)执行,把结果贴回工作项并流转状态。调度是确定性代码,模型只出现在执行器和总指挥里。
## 1. 目标与非目标 ## 1. 目标与非目标
- 目标:多项目隔离(一份配置一段)、多终端(本机/经 Warpgate 的远程机)、多 CLI(执行器适配器)、多轮(线程持久、进程不常驻)、失败可解释可重试、凭据只经 Vault 注入内存、任何会话可接手。 - 目标:多项目隔离(一份配置一段)、多终端(本机/经 Warpgate 或 Tailscale 直连的远程机)、多 CLI(执行器适配器)、多轮(线程持久、进程不常驻)、失败可解释可重试、凭据只经 Vault 注入内存、任何会话可接手。
- 非目标:不是工作流引擎(无 DAG/分支,顺序交给 Plane "blocked by"——**TODO:阶段 1 尚未实现**,调度不读 blocked-by 关系,被阻塞的 Todo 项照样会被拾起);不替代 Plane/Temporal 的界面;不做账号系统;不存密钥。 - 非目标:不是工作流引擎(无 DAG/分支,顺序交给 Plane "blocked by"——**TODO:阶段 1 尚未实现**,调度不读 blocked-by 关系,被阻塞的 Todo 项照样会被拾起);不替代 Plane/Temporal 的界面;不做账号系统;不存密钥。
## 2. 架构 ## 2. 架构
``` ```
Plane ──webhook v2(HMAC)/ 轮询兜底──► agentd(接收端 + 调度核心 + 执行器适配器) Plane ──webhook(HMAC)即时触发 / cron 轮询兜底──► agentd(接收端 + 调度核心 + 执行器适配器)
│ 按 assignee→终端、label→执行器/模型,串行组,重试,超时,心跳 │ 按 assignee→终端、label→执行器/模型,串行组,重试,超时,心跳
┌───────────────┼───────────────┐ ┌───────────────┼───────────────┐
codex(远程/本机) claude-code(本机/远程) shell codex(远程/本机) claude-code(本机/远程) shell
└───────────────┴───────────────┘ └───────────────┴───────────────┘
执行器只产出"最后消息 + 产物路径";agentd 统一回帖、改状态、通知(ntfy) 执行器只产出"最后消息 + 产物路径";agentd 统一回帖、改状态、通知(notify 钩子,暂为空操作)
``` ```
阶段 3 之后调度核心搬到 Temporal(workflow = 一个工作项的生命周期,activity = 起执行器/等结果/回帖),用 Temporal UI 看运行与重试。 阶段 3 之后调度核心搬到 Temporal(workflow = 一个工作项的生命周期,activity = 起执行器/等结果/回帖),用 Temporal UI 看运行与重试。
## 3. 概念与契约 ## 3. 概念与契约
- **终端(terminal)**:怎么到达一台机器 — `transport`(local | warpgate-ssh)、`os`、`launch`(subprocess | schtasks | nohup)、`repo`、`plane_user`(回帖身份,对应 Vault 键)、`concurrency`、`capabilities`、可用 `executors`。 - **终端(terminal)**:怎么到达一台机器 — `transport`(local | warpgate-ssh | direct-ssh,§9)、`os`、`launch`(subprocess | schtasks | nohup)、`repo`、`plane_user`(回帖身份,对应 Vault 键)、`concurrency`、`capabilities`、可用 `executors`。
- **执行器(executor)**:某个 CLI 的适配器 — 起一轮的命令模板、续线程命令模板、`resume: true|false`、可用模型、默认推理强度、沙箱参数、最后消息取法、线程 ID 取法。 - **执行器(executor)**:某个 CLI 的适配器 — 起一轮的命令模板、续线程命令模板、`resume: true|false`、可用模型、默认推理强度、沙箱参数、最后消息取法、线程 ID 取法。
- **路由(routing)**:`by_assignee`(Plane 用户 → 终端)、`by_label`(`exec:*`、`model:*`、`effort:*`、`needs-*` 串行组、`cap:*` 能力、`budget:*` 时限)、每终端默认执行器。 - **路由(routing)**:`by_assignee`(Plane 用户 → 终端)、`by_label`(`exec:*`、`model:*`、`effort:*`、`needs-*` 串行组、`cap:*` 能力、`budget:*` 时限)、每终端默认执行器。
- **工作项契约**:正文 = 任务书;评论 = 消息;执行器最后消息首行 ∈ `[review] | [question] | [blocked] | [progress]`;agentd 回帖格式固定(轮次、run id、退出码、日志路径)。 - **工作项契约**:正文 = 任务书;评论 = 消息;执行器最后消息首行 ∈ `[review] | [question] | [blocked] | [progress]`;agentd 回帖格式固定(轮次、run id、退出码、日志路径)。
@@ -34,7 +34,7 @@ Plane ──webhook v2(HMAC)/ 轮询兜底──► agentd(接收端 + 调
| 环境类(焦点丢失/弹窗/断网) | 当 blocked 处理,暂停同串行组的派发 | Waiting | | 环境类(焦点丢失/弹窗/断网) | 当 blocked 处理,暂停同串行组的派发 | Waiting |
| 运行中工作项被移出 In Progress | 结束任务进程,回帖"运行已取消:工作项状态被改为 X",释放终端槽位/串行组,保留线程 | 不改 | | 运行中工作项被移出 In Progress | 结束任务进程,回帖"运行已取消:工作项状态被改为 X",释放终端槽位/串行组,保留线程 | 不改 |
| 连续 `max_progress_rounds`(默认 12)轮 `[progress]` | 回帖说明,不再续跑 | Waiting → 应答者 | | 连续 `max_progress_rounds`(默认 12)轮 `[progress]` | 回帖说明,不再续跑 | Waiting → 应答者 |
| agentd 自身 | cron 拉起;心跳 10 min 无更新 → ntfy 报警 | — | | agentd 自身 | cron 拉起(webhook 只是提前触发);心跳 10 min 无更新 → 报警(notify 钩子,未实现) | — |
回到 Todo 的工作项无条件重新拾起;逻辑性失败永不自动重试。 回到 Todo 的工作项无条件重新拾起;逻辑性失败永不自动重试。
阶段 1 的具体取舍(均有单测): 阶段 1 的具体取舍(均有单测):
@@ -58,7 +58,6 @@ executors:
projects: projects:
GWENT: GWENT:
id: 79cb5a60-aa4f-4138-9864-1dd70426e665 id: 79cb5a60-aa4f-4138-9864-1dd70426e665
webhook_secret_key: PLANE_WEBHOOK_GWENT
states: {todo: Todo, running: In Progress, waiting: Waiting, review: Review, done: Done} states: {todo: Todo, running: In Progress, waiting: Waiting, review: Review, done: Done}
reviewer: oregon reviewer: oregon
routing: routing:
@@ -67,16 +66,20 @@ projects:
default_executor: {win10: codex, oregon: claude-code} default_executor: {win10: codex, oregon: claude-code}
retry: {transient_max: 3, backoff_minutes: [2, 5, 10]} retry: {transient_max: 3, backoff_minutes: [2, 5, 10]}
timeout_hours: 4 timeout_hours: 4
notify: {ntfy_topic: gwent, on: [waiting, review, failed]} webhook: {bind: 10.77.10.249, port: 8790, path: /plane, secret_key: PLANE_WEBHOOK_SECRET,
tick_command: [systemd-run, --scope, --quiet, --collect, /opt/plane-agentd/cron/agentd-tick-lxc.sh],
debounce_seconds: 3, retry_seconds: 15, retry_window_seconds: 120}
notify: {kind: none} # 钩子:核心每次改状态调用 Agentd.notify(),none = 空操作
``` ```
## 6. 仓库布局 ## 6. 仓库布局
``` ```
agentd/ 调度核心、plane 客户端、vault 注入、状态存储(json/sqlite)、心跳 agentd/ 调度核心、plane 客户端、vault 注入、状态存储(json/sqlite)、心跳
agentd/executors/ codex.py claude_code.py shell.py(适配器契约见 docs/executor-contract.md) agentd/executors/ codex.py claude_code.py shell.py(适配器契约见 docs/executor-contract.md)
agentd/transports/ local.py warpgate_ssh.py(远程 Linux/macOS:bash -c 经中继;Windows:helper 脚本 + schtasks) agentd/transports/ local.py warpgate_ssh.py(远程 Linux/macOS:bash -c;Windows:PowerShell + schtasks;Warpgate 通道)direct_ssh.py(Tailscale 直连通道,PAD-12)
agentd/web/ 配置页面(阶段 2):读写 YAML + schema 校验 + 连通/冒烟测试 + 最近运行 agentd/web/ 配置页面(阶段 2):读写 YAML + schema 校验 + 连通/冒烟测试 + 最近运行
cli/ agentd new|show|comment|state|assign|smoke cli/ agentd tick|new|show|comment|state|assign|smoke|webhook-serve|check-config
systemd/ plane-agentd-webhook.service(LXC 上的 webhook 接收端)
config/ agentd.yaml(示例 + 实际)、schema.json config/ agentd.yaml(示例 + 实际)、schema.json
docs/ design.md executor-contract.md runbook.md docs/ design.md executor-contract.md runbook.md
``` ```
@@ -84,17 +87,34 @@ docs/ design.md executor-contract.md runbook.md
## 7. 阶段 ## 7. 阶段
1. **核心**:配置驱动调度(轮询)、codex-remote(win10) + claude-local(oregon) 两个执行器、线程续接与重建、失败说明与重试、心跳、CLI;gwent 项目作为首个租户切换过来(替换现有 board-worker-plane)。 1. **核心**:配置驱动调度(轮询)、codex-remote(win10) + claude-local(oregon) 两个执行器、线程续接与重建、失败说明与重试、心跳、CLI;gwent 项目作为首个租户切换过来(替换现有 board-worker-plane)。
2. **配置页面**:单页,三个标签(终端/执行器/路由)+ 状态条;保存即校验、提交 Git、热加载;不显示密钥。 2. **配置页面**:单页,三个标签(终端/执行器/路由)+ 状态条;保存即校验、提交 Git、热加载;不显示密钥。
3. **Webhook + ntfy**:Plane webhook v2 签名校验、即时触发;ntfy 通知 Waiting/Review/失败;轮询保留兜底。前提:Plane VM 进 tailnet 或接收端放在可达位置。 3. **Webhook + 通知**:Plane webhook 签名校验、即时触发,轮询保留兜底——**PAD-7 已实现**(§10)。通知 Waiting/Review/失败:只留了空操作钩子 `notify: {kind: none}`(没有 ntfy 服务)。
4. **Temporal**:调度核心迁为 workflow/activity,获得重试/超时/历史/UI;agentd 变为 worker + webhook 入口。 4. **Temporal**:调度核心迁为 workflow/activity,获得重试/超时/历史/UI;agentd 变为 worker + webhook 入口。
5. **Gitea 联动**:执行器产物以分支/PR 提交并回帖链接;可选 Gitea webhook 回写 Plane。 5. **Gitea 联动**:执行器产物以分支/PR 提交并回帖链接;可选 Gitea webhook 回写 Plane。
## 8. 安全 ## 8. 安全
配置里只有 Vault 键名;token 由 `agent-access.py exec --secret-env` 注入 agentd 进程;执行器不持有 Plane 凭据、不能直接改工作项;webhook 按项目签名校验;页面仅 tailnet 可达,登录走既有网关。 配置里只有 Vault 键名;token 由 `agent-access.py exec --secret-env` 注入 agentd 进程;执行器不持有 Plane 凭据、不能直接改工作项;webhook 是工作区级的一个密钥(Vault 键 `PLANE_WEBHOOK_SECRET`,只经 `--secret-env` 进接收端进程,不传给子进程),HMAC 校验失败 401,只认配置里的项目;接收端只听内网地址;页面仅 tailnet 可达,登录走既有网关。
## 9. 部署位置 ## 9. 部署位置
- Plane VM(plane-01)只跑 Plane:它是设备化节点(固定版本、每日备份停写路径),不放 agentd。 - 网络(2026-10-21):LXC 到 NAS 的 tailnet 路径依赖 OPNsense 的静态端口出站 NAT 规则 + LXC tailscaled 固定 UDP 41642,已直连(详见 docs/deploy-lxc.md §2.1);曾因 DERP 导致调度器暂回 oregon,已于同日切回 LXC。
- Plane VM(Plane 3.3.3,PVE VM 506 plane-02)只跑 Plane:它是设备化节点(固定版本、每日备份停写路径),不放 agentd。
- 阶段 1–2:agentd 跑在 oregon(已有 Vault 客户端、Warpgate 中继脚本、Claude Code、现有 worker 状态),cron 拉起;配置页面同进程,只监听 tailnet 地址。 - 阶段 1–2:agentd 跑在 oregon(已有 Vault 客户端、Warpgate 中继脚本、Claude Code、现有 worker 状态),cron 拉起;配置页面同进程,只监听 tailnet 地址。
- 阶段 3 起(**2026-10-21 PAD-11 已切换**):迁到 PVE 上的精简 LXC `agentd-01`(CT 600,root,仓库 `/opt/plane-agentd`,状态 `/var/lib/plane-agentd`,日志 `/var/log/plane-agentd`;1 vCPU / 1–2 GB,tailnet + 内网 10.77.10.0/24,与 Plane/Temporal 同网段,webhook 走内网)。**LXC 上不装任何 agent CLI**,只有 Python 3.10+、PyYAML、Agent Vault 客户端、ssh 客户端;所有终端(win10、oregon、mac…)一律远程启动(`transport: warpgate-ssh`),LXC 只负责调度、回帖、通知。对外访问经 Warpgate HTTP target,复用既有登录。 - 阶段 3 起(**2026-10-21 PAD-11 已切换**):迁到 PVE 上的精简 LXC `agentd-01`(CT 600,root,仓库 `/opt/plane-agentd`,状态 `/var/lib/plane-agentd`,日志 `/var/log/plane-agentd`;1 vCPU / 1–2 GB,tailnet + 内网 10.77.10.0/24,与 Plane/Temporal 同网段,webhook 走内网)。**LXC 上不装任何 agent CLI**,只有 Python 3.10+、PyYAML、Agent Vault 客户端、ssh 客户端;所有终端(win10、oregon、mac…)一律远程启动(`transport: warpgate-ssh` 或 `direct-ssh`,见下文"终端通道"),LXC 只负责调度、回帖、通知。对外访问经 Warpgate HTTP target,复用既有登录。
- 远程启动的通用实现(P1-6,已实现):Linux/macOS 经 Warpgate SSH(`agent-access.py run --target <终端 target> -- bash -c …`)上传 `prompt.md`/`run.sh` 并一次调用内启动,`setsid nohup` 脱离会话(macOS 用 perl `POSIX::setsid`),轮询 `exit.txt`,从终端上的输出文件取回最后消息/线程 ID,kill 按进程组 + 会话;中继目标名由终端配置给出。oregon 已切到这条路径(调度器暂仍在 oregon,启动经 Warpgate 回到本机)。部署步骤:`docs/deploy-lxc.md`。 - 远程启动的通用实现(P1-6,已实现):Linux/macOS 经 Warpgate SSH(`agent-access.py run --target <终端 target> -- bash -c …`)上传 `prompt.md`/`run.sh` 并一次调用内启动,`setsid nohup` 脱离会话(macOS 用 perl `POSIX::setsid`),轮询 `exit.txt`,从终端上的输出文件取回最后消息/线程 ID,kill 按进程组 + 会话;中继目标名由终端配置给出。oregon 已切到这条路径(调度器暂仍在 oregon,启动经 Warpgate 回到本机)。部署步骤:`docs/deploy-lxc.md`。
- 防重复启动(PAD-11):启动调用之前先把计划中的运行(run id、终端、handle、记账字段)作为 `launching` 标记落盘;启动调用失败/超时或 agentd 中途退出时,下一 tick 探测运行目录(`exit.txt`/`launch.pid`/`pid.txt`):有 → 接管;无 → 同一 run id 换新目录 `<run_id>-r2` 重启(不清空旧目录);探测不通 → 保留标记并占住终端槽位。transport 拒绝清空含 pid 文件的运行目录。某终端中继失败时本 tick 余下操作跳过该终端,其他终端照常。 - 防重复启动(PAD-11):启动调用之前先把计划中的运行(run id、终端、handle、记账字段)作为 `launching` 标记落盘;启动调用失败/超时或 agentd 中途退出时,下一 tick 探测运行目录(`exit.txt`/`launch.pid`/`pid.txt`):有 → 接管;无 → 同一 run id 换新目录 `<run_id>-r2` 重启(不清空旧目录);探测不通 → 保留标记并占住终端槽位。transport 拒绝清空含 pid 文件的运行目录。某终端中继失败时本 tick 余下操作跳过该终端,其他终端照常。
- Windows 仍用 helper 脚本 `win10-ps.sh` / `win10-put.sh`(仓库 `tools/win10/`,写死 `--target win10`)+ 计划任务。**TODO(统一)**:Windows 也改走 `relay()`(argv = `powershell.exe -EncodedCommand …`,文件同样 base64 分块),删掉对 helper 脚本与 `helpers_dir` 的依赖,LXC 上就不必再复制这两个脚本。 - Windows 走 Warpgate 时仍用 helper 脚本 `win10-ps.sh` / `win10-put.sh`(仓库 `tools/win10/`,写死 `--target win10`)+ 计划任务:`agent-access.py run` 把 ssh 的 stdin 接到 /dev/null(`run_remote` 里 `stdin=subprocess.DEVNULL`),cmd.exe 命令行又限 8191 字符,文件只能按 2400 字符分块上传。走 `direct-ssh` 时不用 helper 脚本。收尾读取在 Warpgate 上也已合成一次调用(脚本足够短,放得进 `-EncodedCommand`)。
- **终端通道(PAD-12,2026-10-22)**:LXC 到每个远程终端走哪条路,按终端配置 `transport`:
- `warpgate-ssh`(原有,缺省):`agent-access.py run` 经 Warpgate SSH 网关(`115.28.210.101:2222`)。每次调用都要取 Vault 凭据、与网关握手、密码认证、网关再连终端,LXC 上实测中位数约 9 s(客户端 + vault 0.8 s、网关 banner/kex 1.8 s、密码认证 2.2 s、二跳 + 命令 2–5 s)。客户端写死 `ControlMaster=no`,连接不能复用,数据只能放 argv。一次 win10 启动约 10 次串行调用(busy 检查、mkdir/清空、prompt.md 分块上传 + 重置 + 解码、run.ps1、schtasks):GW-26-1 用了 91 s;收尾约 7 次,约 60 s。
- `direct-ssh`(新):LXC 经 Tailscale 直连终端的 sshd(`ssh: {host, user, port, host_key}`)。认证用一把专用 ed25519 密钥:Vault 键 `AGENTD_SSH_KEY` 经 `--secret-env` 进 tick 进程,载入本进程私有的 ssh-agent(0700 临时目录里的套接字,`ssh-add -` 从 stdin 读)后立即从环境变量删掉,子进程拿不到;不落盘,不进 argv、日志、Plane 评论或异常信息。主机公钥钉死在配置里,`StrictHostKeyChecking=yes`,known_hosts 由钉住的公钥生成,只含这一把。`ControlMaster=auto`,`ControlPersist=60`,套接字放在只有 root 可进的 `agentd.ssh_dir`(`/run/plane-agentd/ssh`),名字是 `<host_key 指纹 8 位>-%C`:pin 改了就换新连接,不会沿用按旧 pin 校验过的主连接。密钥在 CLI 启动时就从环境变量里拿走(第一次直连之前起的子进程也继承不到)。脚本走 stdin:Windows 是一段固定的 PowerShell 引导(`-EncodedCommand`),从 stdin 读真正脚本的 base64;Linux 是固定的 bash 加载器。所以每个操作恰好一次调用,不受 argv 长度限制。win10 启动(建目录、清空并遵守 HASPID 拒绝、写 UTF-8 无 BOM 的 prompt.md/run.ps1、注册并启动计划任务)一次;收尾一次(exit、最后消息、线程 ID、stderr 尾、事件尾)加删计划任务一次。oregon 启动一次,收尾一次。busy 检查和 poll 仍各一次。实测直连路径:win10 RTT 27 ms(握手到认证 0.24 s),oregon RTT 233 ms(握手 2.2 s)。目标:派单延迟 ≤ 15 s。
- `fallback: warpgate-ssh`(可选,需要 `target`):直连**在远程命令运行之前**失败时(ssh 退出 255、没有任何 stdout,且 stderr 里有 ssh 自己在行首打印的连接超时/拒绝/无路由/解析失败/认证失败/agent 错误,或者本进程没拿到密钥。远程加载器把脚本自己的 255 改报 254,所以 255 只可能来自 ssh;有 stdout 就说明命令已经跑过),这个操作改走 Warpgate 重做,本 tick 里该终端余下的操作也都走 Warpgate,`agentd.log` 记一行。**主机公钥不符(包括终端不提供 ssh-ed25519 主机密钥)绝不回退**:按安全事件处理,`agentd.log` 记 `SECURITY …` 行,该终端本 tick 标为 down。命令可能已经运行过的失败(超时、会话中途断开、认不出的 255)不会换路径重试。启动这种非幂等操作仍靠 `launching` 标记加下一 tick 的探测/接管(`resolve_launch`),不存在双启动的路径。直连的每个操作都只有一次远程调用,所以"整个操作换路径重做"不会重复任何副作用。
- **取舍**:直连绕过了 Warpgate 的会话录像和集中授权。访问控制改由终端自己负责:authorized_keys 只认这把专用密钥,并限定来源 `from="100.69.78.124"`(LXC 的 tailnet 地址)。用户已逐终端批准(win10、oregon,2026-10-22)。回滚:把该终端的 `transport` 改回 `warpgate-ssh`(`ssh`/`fallback` 可以留着,此时被忽略)。
- 观测:每次远程调用在 `<log_dir>/remote.log` 记一行(`term= via=direct|warpgate|fallback op= ms= rc=`)。部署后用 `agentd ping <终端> [--transport direct-ssh|warpgate-ssh]` 验证。步骤见 `docs/deploy-lxc.md` §9。
## 10. Webhook 触发(PAD-7)
- **形态**:`agentd webhook-serve`(`agentd/webhook.py`,只用标准库)常驻在 LXC 上(systemd `plane-agentd-webhook`),Plane 工作区级 webhook(事件 `issue` + `issue_comment`)投到 `http://10.77.10.249:8790/plane`。它不自己调度,只是**提前跑一次 cron 的 tick 脚本**:调度仍然只有一份代码、一把锁、一个状态文件;cron 每 2 分钟照跑,webhook 丢了/接收端挂了只是退回到轮询延迟。
- **校验**:`X-Plane-Signature` = HMAC-SHA256(密钥, 请求体) 的十六进制(Plane 签 `json.dumps(payload)` 并原样发送);常量时间比较,先比原始字节、再比重新序列化的结果。缺失/不符 → 401,不触发。
- **路由**:`data.project` ∈ 配置的项目 → 200 并触发;否则(别的项目、删除事件没有项目)→ 202 忽略。不按事件类型或操作者过滤:agentd 自己回帖/改状态也会产生投递,结果是每次有动作的 tick 之后多一次 tick(空闲 tick 不写 Plane,不会循环)。
- **合并与补跑**:第一条投递后等 3 s,期间的投递合并成一次 tick;tick 运行中又来的投递 → 结束后再跑一次(运行中的 tick 可能在变化之前读过 Plane)。锁被占(cron/手工 tick;脚本 `flock -n -E 75`,`cli tick` 撞上 agentd 状态锁也退出 75)→ 每 15 s 重试,最长 2 min,之后交给 cron。
- **进程隔离**:tick 用 `systemd-run --scope` 跑在独立 cgroup 里,重启/停止接收端不会杀掉正在跑的 tick(tick 可能正在启动远程运行)。
- **延迟构成**:投递 → tick 开始 ≈ 3 s 合并窗口;tick 本身 = vault 取 token(1–2 s)+ 读 Plane + 先 poll 每个在途运行(每个中继调用约 10 s)+ 启动(中继约 12 s;win10 上传更久)。`[worker] 第 k 轮开始` 评论在启动调用返回之后才发,所以"建项 → 开始评论"受中继成本限制,而不是触发方式。PAD-12 之后,直连终端每次调用不再是约 10 s,而是亚秒到约 2 s(§9 终端通道)。
- **Plane 的 SSRF 防护**:Plane(≥1.4 / 商业版 3.x)拒绝解析到私有/CGNAT 地址的 webhook URL(创建时和每次投递时都查),除非 Plane 的环境里有 `WEBHOOK_ALLOWED_IPS`(CIDR 列表)或 `WEBHOOK_ALLOWED_HOSTS`。内网接收端需要在 Plane 的 api 与 worker 容器环境里设 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(改 `/opt/plane/plane.env` 与渲染的 `/var/plane/docker-compose.yml`,重建 api/worker,见 deploy-lxc §8)。2026-10-22 已应用(同时重建了 webhook-consumer、outbox-poller、automation-consumer、beat-worker,见 deploy-lxc §8),webhook 已启用,cron 保留为兜底。Plane 建出的是 v2 webhook(`plane/webhook/bgtasks/webhook_task.py` 的 `send_webhook`,负载 `{version: v2, event, entity_type, data, …}`),签名同为 HMAC-SHA256(secret, json.dumps(payload)),`data.project` 路由照常,接收端无需改动。
- **心跳里的跳过原因**:每个 tick 把没派出去的 Todo 项和原因写进 `heartbeat.json` 的 `skipped`(`concurrency 1/1 (GW-15)`、`serialize_group game held by GW-4`、`busy_process codex (1 running, not started by agentd)`、`relay down`、`no route for assignee kappiee`、`start failed: …`),同一项的原因变化时才写一行日志。
+3 -3
View File
@@ -26,11 +26,11 @@ agentd 不常驻:一轮运行比起它的那次 tick 活得久,所以执行
- `kill(handle)` 既用于超时,也用于取消(工作项在运行中被移出 In Progress);必须能在运行未结束时调用。取消后 core 会尝试 `collect(handle)` 取线程 ID,所以 `collect` 在未结束的运行上也不能抛出致命错误(抛异常会被忽略)。 - `kill(handle)` 既用于超时,也用于取消(工作项在运行中被移出 In Progress);必须能在运行未结束时调用。取消后 core 会尝试 `collect(handle)` 取线程 ID,所以 `collect` 在未结束的运行上也不能抛出致命错误(抛异常会被忽略)。
## RunnerExecutor(现有三个适配器的共同形态) ## RunnerExecutor(现有三个适配器的共同形态)
1. 在终端上建运行目录 `<run_root>/<dir_name>`(清空旧文件;目录里已有 `pid.txt`/`launch.pid` 时拒绝并报 `HASPID`),写 `prompt.md`。transport 有 `start_run` 时(远程 POSIX),建目录、写 `prompt.md` 与 runner、启动合并成一次中继调用。 1. 在终端上建运行目录 `<run_root>/<dir_name>`(清空旧文件;目录里已有 `pid.txt`/`launch.pid` 时拒绝并报 `HASPID`),写 `prompt.md`。transport 有 `start_run` 时(远程终端),建目录、写 `prompt.md` 与 runner、启动由 transport 一起做:直连(direct-ssh)和远程 POSIX 是一次调用,Windows 经 Warpgate 时仍逐步上传。
2. 用配置里的命令模板生成命令:`run`(新线程)或 `resume`(续线程)。占位符:`{repo} {model} {effort} {tier} {out} {thread} {perm} {script} {run_dir}`;未知占位符原样保留。`{tier}` 展开为 `tier_arg`(如 `-c 'service_tier="fast"'`),无 tier 时为空。tier 来源:`tier:<x>` 标签或 `routing.by_label` 的 `tier`,否则执行器的 `tier_defaults[<model>]`;`tier:none` 表示不加档位参数。模板首词若在终端 `bins` 里,替换为绝对路径(Windows 前加 `&`)。参数请用单引号,bash 和 PowerShell 都认。 2. 用配置里的命令模板生成命令:`run`(新线程)或 `resume`(续线程)。占位符:`{repo} {model} {effort} {tier} {out} {thread} {perm} {script} {run_dir}`;未知占位符原样保留。`{tier}` 展开为 `tier_arg`(如 `-c 'service_tier="fast"'`),无 tier 时为空。tier 来源:`tier:<x>` 标签或 `routing.by_label` 的 `tier`,否则执行器的 `tier_defaults[<model>]`;`tier:none` 表示不加档位参数。模板首词若在终端 `bins` 里,替换为绝对路径(Windows 前加 `&`)。参数请用单引号,bash 和 PowerShell 都认。
3. transport 生成 runner 脚本(Linux `run.sh`,Windows `run.ps1`):进入 repo,写 `pid.txt`/`started.txt`,把 `prompt.md` 从 stdin 喂给命令,stdout → `stdout_name`,stderr → `stderr.txt`,结束写 `exit.txt`(`exit=N at …`)。 3. transport 生成 runner 脚本(Linux `run.sh`,Windows `run.ps1`):进入 repo,写 `pid.txt`/`started.txt`,把 `prompt.md` 从 stdin 喂给命令,stdout → `stdout_name`,stderr → `stderr.txt`,结束写 `exit.txt`(`exit=N at …`)。
4. transport 分离启动(本机 `setsid -f bash run.sh`;远程 Linux `setsid nohup bash run.sh &`,macOS 用 `perl POSIX::setsid`,`$!` 记入 `launch.pid`;Windows 一次性计划任务 `agentd-<run_id>`)。 4. transport 分离启动(本机 `setsid -f bash run.sh`;远程 Linux `setsid nohup bash run.sh &`,macOS 用 `perl POSIX::setsid`,`$!` 记入 `launch.pid`;Windows 一次性计划任务 `agentd-<run_id>`)。
5. `poll` = `exit.txt` 是否存在;`collect` 由各适配器从终端上保存的文件解析(远程时经中继读回,不依赖调度器本机有任何 CLI): 5. `poll` = `exit.txt` 是否存在;`collect` 由各适配器从终端上保存的文件解析(远程时读回,不依赖调度器本机有任何 CLI)。适配器把要读的东西列成 `COLLECT`(`("read", 文件)`、`("grep", 文件, 子串)`、`("tail", 文件, n)`),经 `transport.fetch` 一次调用取回;transport 没有 `fetch` 时逐个读:
| 适配器 | stdout | 最后消息 | 线程 ID | | 适配器 | stdout | 最后消息 | 线程 ID |
|---|---|---|---| |---|---|---|---|
@@ -41,7 +41,7 @@ agentd 不常驻:一轮运行比起它的那次 tick 活得久,所以执行
配置里的 `thread_from` / `last_message` 目前只是说明性字段,解析逻辑在适配器里。 配置里的 `thread_from` / `last_message` 目前只是说明性字段,解析逻辑在适配器里。
## Transport 接口(`agentd/transports/`) ## Transport 接口(`agentd/transports/`)
`os`、`join`、`quote`、`default_run_root`、`mkdir(path, clean)`(clean 遇到 pid 文件必须拒绝)、`probe(run_dir, names)`(存在的文件名列表,一次调用)、`write_text`、`read_text`(不存在返回 None)、`grep_first(path, needle)`、`tail(path, n)`、`runner(run_dir, cwd, command, stdout_name)`、`launch(task, script, run_dir)`、`kill(task, run_dir)`、`forget(task)`、`process_count(name)`(None = 不可达);可选 `start_run(task, run_dir, files, script_name)`(一次完成建目录 + 写文件 + 启动)。 `os`、`join`、`quote`、`default_run_root`、`mkdir(path, clean)`(clean 遇到 pid 文件必须拒绝)、`probe(run_dir, names)`(存在的文件名列表,一次调用)、`write_text`、`read_text`(不存在返回 None)、`grep_first(path, needle)`、`tail(path, n)`、`runner(run_dir, cwd, command, stdout_name)`、`launch(task, script, run_dir)`、`kill(task, run_dir)`、`forget(task)`、`process_count(name)`(None = 不可达);可选 `start_run(task, run_dir, files, script_name)`(建目录 + 写文件 + 启动)和 `fetch(run_dir, ops)`(一次读回多个结果,`collect` 用)。远程 transport 的脚本由一个"通道"送达:`warpgate-ssh`(`agent-access.py run`,只能走 argv)或 `direct-ssh`(Tailscale 直连 OpenSSH,脚本走 stdin,每个操作一次调用;可配 `fallback: warpgate-ssh`,见 design §9)。直连在命令运行前失败时抛 `Unreached`(`RoutedTransport` 据此换到兜底通道),主机公钥不符时抛 `HostKeyError`(不回退);`process_count` 不能吞掉 `Unreached`。
- `local`:Linux 本机文件 + `setsid -f`;kill = 对 `pid.txt` 的进程组发 SIGTERM。用于调度器与执行器同机的部署和测试。 - `local`:Linux 本机文件 + `setsid -f`;kill = 对 `pid.txt` 的进程组发 SIGTERM。用于调度器与执行器同机的部署和测试。
- `warpgate-ssh` + `os: linux|darwin`(`PosixWarpgateTransport`):每个操作是一次 `agent-access.py --config <vault.config> run --target <target> -- bash -c '<script>'`(`relay()`,目标名取终端 `target`,缺省为终端名;客户端把 argv `shlex.join` 后交给目标账户的登录 shell,stdin 为 /dev/null)。文件:base64 放进 `printf %s '<b64>' | base64 -d`,超过 60000 字符分块追加再解码,回显字节数校验;读文件/`grep_first`/`tail` 的结果以 `B64=` 标记回传。runner 与本机相同(`run.sh`),`setsid nohup` 脱离 SSH 会话(pid = pgid = sid);kill = 对该进程组 `kill -TERM -- -<pid>` + `pkill -TERM -s <pid>`,最多等 5 s 后 KILL,`exit.txt` 已存在时不发信号(防 pid 复用);`process_count` = `pgrep -x`。`repo`/`run_root` 必须是远程绝对路径。每次调用约 10–14 s,所以启动只用一次调用。 - `warpgate-ssh` + `os: linux|darwin`(`PosixWarpgateTransport`):每个操作是一次 `agent-access.py --config <vault.config> run --target <target> -- bash -c '<script>'`(`relay()`,目标名取终端 `target`,缺省为终端名;客户端把 argv `shlex.join` 后交给目标账户的登录 shell,stdin 为 /dev/null)。文件:base64 放进 `printf %s '<b64>' | base64 -d`,超过 60000 字符分块追加再解码,回显字节数校验;读文件/`grep_first`/`tail` 的结果以 `B64=` 标记回传。runner 与本机相同(`run.sh`),`setsid nohup` 脱离 SSH 会话(pid = pgid = sid);kill = 对该进程组 `kill -TERM -- -<pid>` + `pkill -TERM -s <pid>`,最多等 5 s 后 KILL,`exit.txt` 已存在时不发信号(防 pid 复用);`process_count` = `pgrep -x`。`repo`/`run_root` 必须是远程绝对路径。每次调用约 10–14 s,所以启动只用一次调用。
- `warpgate-ssh` + `os: windows`(`WindowsWarpgateTransport`):经 `win10-ps.sh`(PowerShell `-EncodedCommand`)和 `win10-put.sh`(分块上传 + SHA-256 校验);文件读取用 base64 标记回传;kill = `taskkill /T /F /PID <pid.txt>` + `schtasks /End /Delete`。脚本目录由 `agentd.helpers_dir`(或终端 `helpers_dir`)配置;脚本目前写死 `--target win10`(TODO:改走同一个 `relay()`,argv 为 `powershell.exe -NoProfile -NonInteractive -EncodedCommand <b64>`,去掉 helper 脚本)。路径都放进 PowerShell 单引号字面量(`'` 双写);`schtasks /TR` 里脚本路径用 `\"…\"` 包裹,依赖 Windows PowerShell 5.1 的旧式原生参数传递(PowerShell 7.3+ 的 Standard 模式下需改写)。runner 进不了 repo 时写 `exit=97` 结束。 - `warpgate-ssh` + `os: windows`(`WindowsWarpgateTransport`):经 `win10-ps.sh`(PowerShell `-EncodedCommand`)和 `win10-put.sh`(分块上传 + SHA-256 校验);文件读取用 base64 标记回传;kill = `taskkill /T /F /PID <pid.txt>` + `schtasks /End /Delete`。脚本目录由 `agentd.helpers_dir`(或终端 `helpers_dir`)配置;脚本目前写死 `--target win10`(TODO:改走同一个 `relay()`,argv 为 `powershell.exe -NoProfile -NonInteractive -EncodedCommand <b64>`,去掉 helper 脚本)。路径都放进 PowerShell 单引号字面量(`'` 双写);`schtasks /TR` 里脚本路径用 `\"…\"` 包裹,依赖 Windows PowerShell 5.1 的旧式原生参数传递(PowerShell 7.3+ 的 Standard 模式下需改写)。runner 进不了 repo 时写 `exit=97` 结束。
+79 -4
View File
@@ -19,13 +19,15 @@ cd /root/.local/share/server-access-client && python3 -I agent-access.py \
--config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \ --config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
--secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \ --secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \
--secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN \ --secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN \
--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY \
--timeout 900 -- python3 /opt/plane-agentd/cli/agentd tick --once --timeout 900 -- python3 /opt/plane-agentd/cli/agentd tick --once
``` ```
`AGENTD_SSH_KEY` 是 direct-ssh 终端的专用密钥(PAD-12,design §9)。手工运行时不加这个参数不会出错,只是直连终端整次 tick 都走 Warpgate 兜底(慢),`agentd.log` 里会有一行 `direct-ssh unavailable in this process`。但加了参数而 Vault 里没有这个键,agent-access.py 会以 `access-hub: credential_unavailable` 直接失败,agentd 根本不运行(它没有可选密钥);cron 脚本遇到这种情况会去掉这个参数再跑一次,手工运行则要自己去掉。
crontab(LXC root,每 2 分钟): crontab(LXC root,每 2 分钟):
``` ```
*/2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh */2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh
``` ```
`cron/agentd-tick-lxc.sh` = `flock -n /var/lib/plane-agentd/cron.lock timeout 900 python3 -I agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once >> /var/log/plane-agentd/cron.log 2>&1`。空闲 tick 不打印任何东西(cron 日志为空是正常的),活着与否看 `heartbeat.json` 的 `at`;`skip: another tick holds …` 表示与手工 tick 撞锁,无害。 `cron/agentd-tick-lxc.sh` = 拿 `/var/lib/plane-agentd/cron.lock`(`flock -n -E 75`,撞锁退出 75)后执行 `timeout 900 python3 -I agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once`,输出追加到 `/var/log/plane-agentd/cron.log`(stderr 排在 stdout 之后)。如果这次 exec 只报了 `access-hub: credential_unavailable`/`credential_invalid`(退出 1,agentd 没运行),脚本在 cron.log 记一行 `AGENTD_SSH_KEY not available from Vault: tick repeated without it …`,去掉 `AGENTD_SSH_KEY` 再跑一次:直连终端走兜底,派单不中断。空闲 tick 不打印任何东西(cron 日志为空是正常的),活着与否看 `heartbeat.json` 的 `at`;`skip: another tick holds …` 表示与手工 tick 撞锁,无害。
实际配置 `config/agentd.yaml`(已提交,不含密钥):`state_dir: /var/lib/plane-agentd`、`log_dir: /var/log/plane-agentd`,所以下表的 `state/…` 在 LXC 上是 `/var/lib/plane-agentd/…`,`agentd.log` 在 `/var/log/plane-agentd/`。改配置/代码 = 在 oregon 提交并推送,然后 LXC 上 `cd /opt/plane-agentd && git pull`(下一个 tick 生效)。 实际配置 `config/agentd.yaml`(已提交,不含密钥):`state_dir: /var/lib/plane-agentd`、`log_dir: /var/log/plane-agentd`,所以下表的 `state/…` 在 LXC 上是 `/var/lib/plane-agentd/…`,`agentd.log` 在 `/var/log/plane-agentd/`。改配置/代码 = 在 oregon 提交并推送,然后 LXC 上 `cd /opt/plane-agentd && git pull`(下一个 tick 生效)。
oregon 上的 `cli/agentd` 只当 CLI 用(new/show/comment/state/assign,`smoke … --watch`);**不要在 oregon 上跑 `tick` 或不带 `--watch` 的 `smoke`**(那是第二个调度器)。 oregon 上的 `cli/agentd` 只当 CLI 用(new/show/comment/state/assign,`smoke … --watch`);**不要在 oregon 上跑 `tick` 或不带 `--watch` 的 `smoke`**(那是第二个调度器)。
暂停一个终端(不再派新运行、也不探测,在途运行照常收尾):把它的 `concurrency` 设为 0。 暂停一个终端(不再派新运行、也不探测,在途运行照常收尾):把它的 `concurrency` 设为 0。
@@ -48,16 +50,40 @@ cli/agentd state GWENT 12 Todo # Plane 状态名或配置键(todo/review
cli/agentd assign GWENT 12 window10 cli/agentd assign GWENT 12 window10
cli/agentd smoke GWENT oregon --watch # 建一个要求回复 "[review] ok" 的工作项,等排程中的调度器跑完,PASS 则移到 Done cli/agentd smoke GWENT oregon --watch # 建一个要求回复 "[review] ok" 的工作项,等排程中的调度器跑完,PASS 则移到 Done
cli/agentd smoke GWENT win10 --labels model:sol --watch # --labels 给冒烟项加标签(路由到模型/档位) cli/agentd smoke GWENT win10 --labels model:sol --watch # --labels 给冒烟项加标签(路由到模型/档位)
cli/agentd ping win10 [--transport direct-ssh|warpgate-ssh] [-n 3] # 在终端上跑一条空命令并计时,每次调用一行
# (direct-ssh 需要 --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY,不需要 Plane token)
# 不带 --watch = 本机自己 tick(只在调度器所在机器上、且它的 cron 停着时用) # 不带 --watch = 本机自己 tick(只在调度器所在机器上、且它的 cron 停着时用)
``` ```
CLI 以 `--as <terminal>` / `agentd.cli_terminal` / 第一个有 token 的终端的身份操作。 CLI 以 `--as <terminal>` / `agentd.cli_terminal` / 第一个有 token 的终端的身份操作。
## Webhook(PAD-7:Plane 事件即时触发 tick)
cron 每 2 分钟一次的 tick 保留作兜底;另有常驻接收端,Plane 一有工作项/评论变化就立刻触发一次 tick。
- **服务**:LXC `plane-agentd-webhook.service`(单元文件在仓库 `systemd/`,`Restart=always`)= `agent-access.py … exec --secret-env PLANE_WEBHOOK_SECRET=PLANE_WEBHOOK_SECRET --timeout 315360000 -- python3 /opt/plane-agentd/cli/agentd webhook-serve --bind 10.77.10.249 --port 8790`。只听内网地址;密钥只在进程环境里,agentd 起的子进程(tick 脚本)环境里去掉了它。
- **端点**:`POST http://10.77.10.249:8790/plane`;`GET /healthz` → `{"ok": true, "pending", "queued", "running", "retrying_since", "ticks", "last": {rc, seconds, deliveries, at}}`。
- **Plane 侧**:工作区级 webhook(kappiee 工作区设置 → Webhooks,或 `tools/plane-webhook.py`),订阅"工作项"(`issue`)和"评论"(`issue_comment`)。Plane 会把所有项目的事件都发来,接收端按 `data.project` 只认配置里的项目。
- **签名**:头 `X-Plane-Signature` = 用 webhook 密钥对请求体做 HMAC-SHA256 的十六进制摘要(Plane 签的是 `json.dumps(payload)`,发出的正是这串字节;接收端先比原始请求体,再比重新序列化的结果,均为常量时间比较)。缺失/不对 → 401,什么都不触发。其他头:`X-Plane-Event`(issue / issue_comment)、`X-Plane-Delivery`(每次投递的 UUID)。请求体:`{"event","action"(created/updated/deleted),"webhook_id","workspace_id","workspace_slug","data":{…,"project":<uuid>},"activity":{field,new_value,old_value,actor…}}`;删除事件的 `data` 只有 `id`(没有项目 → 202 忽略,由 cron 兜底)。
- **触发逻辑**:配置项目的投递 → 200,3 s(`debounce_seconds`)内的投递合并成一次 tick;tick 运行期间又来的投递 → 它结束后再跑一次(正在跑的 tick 可能在变化之前就读过 Plane)。tick 命令就是 cron 脚本(`systemd-run --scope … cron/agentd-tick-lxc.sh`,在独立 scope 里跑,重启接收端不会杀掉正在跑的 tick)。脚本持 `flock -n -E 75`:锁被 cron/手工 tick 占着 → 退出码 75 → 每 15 s 重试,最长 2 min,之后交给下一个 cron tick。tick 本身的输出照旧进 `cron.log`/`agentd.log`。
- **日志** `/var/log/plane-agentd/webhook.log`:每次投递一行(`delivery=<X-Plane-Delivery> event=issue/updated project=GWENT item=<uuid> decision=tick|202 ignored|401 bad signature|401 missing signature`),每次 tick 两行(`tick start (n deliveries: …)`、`tick done rc=0 in 12.3s` / `tick busy …, retry in 15s` / `giving up`)。Plane 侧的投递记录:`tools/plane-webhook.py logs <webhook id>`。
- **查看**:`systemctl status plane-agentd-webhook`、`journalctl -u plane-agentd-webhook`、`curl -s http://10.77.10.249:8790/healthz`、`tail -f /var/log/plane-agentd/webhook.log`。
- **轮换密钥**(在 oregon 上,kappiee 管理员会话,密钥不出进程、直接写进 Vault):
```bash
cd ~/.local/share/server-access-client && python3 -I agent-access.py --config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
--secret-env PLANE_KAPPIEE_PASSWORD=PLANE_KAPPIEE_PASSWORD --timeout 90 -- \
python3 -I /home/ubuntu/Documents/wjj/project/plane-agentd/tools/plane-webhook.py regenerate <webhook id> # id: … plane-webhook.py list
```
然后 LXC 上 `systemctl restart plane-agentd-webhook`(接收端只在启动时取一次密钥;轮换到重启之间的投递会 401,cron 兜底)。核对:`agent-access.py … exec --secret-env S=PLANE_WEBHOOK_SECRET -- python3 -c 'import os;print(len(os.environ["S"]))'`。
- **停用**:只停接收端 `systemctl disable --now plane-agentd-webhook`(Plane 投递会失败并重试,多次失败后 Plane 自动停用该 webhook 并给管理员发邮件);连 Plane 侧一起停:`plane-webhook.py patch <id> is_active=false`。cron 不受影响。
- **通知**:配置里的 `notify: {kind: none}` 是钩子,核心每次改状态都调用 `Agentd.notify()`,目前什么都不做(没有 ntfy 服务)。
## 文件 ## 文件
| 路径 | 内容 | | 路径 | 内容 |
|---|---| |---|---|
| `state/agentd-state.json` | 每项目每工作项:轮次、线程 ID 与所在终端/执行器、线程轮数、各轮标签、已见评论、重试计数/下次重试、在途运行(handle、deadline) | | `state/agentd-state.json` | 每项目每工作项:轮次、线程 ID 与所在终端/执行器、线程轮数、各轮标签、已见评论、重试计数/下次重试、在途运行(handle、deadline) |
| `<log_dir>/agentd.log`(默认 `state/`) | 每次 tick 的动作与错误 | | `<log_dir>/agentd.log`(默认 `state/`) | 每次 tick 的动作与错误 |
| `state/heartbeat.json` | 最近一次 tick 的时间、在途运行、启动未确认的运行(`launching`)、本 tick 跳过的不可达终端(`down`)、错误;超过 10 分钟不更新 = agentd 没在跑 | | `state/heartbeat.json` | 最近一次 tick 的时间、在途运行、启动未确认的运行(`launching`)、本 tick 跳过的不可达终端(`down`)、没派出去的 Todo 项及原因(`skipped`:`[{project, item, terminal, reason}]`)、错误;超过 10 分钟不更新 = agentd 没在跑 |
| `<log_dir>/webhook.log` | webhook 接收端:每次投递的决定、每次触发的 tick(见上节) |
| `<log_dir>/remote.log` | 每次远程调用一行:`term=win10 via=direct\|warpgate\|fallback op=start\|fetch\|read\|busy\|probe\|kill\|forget… ms=… rc=…`,失败时末尾带 `unreached`/`hostkey`/`unknown`(PAD-12)。不回显到 cron.log;没有轮转,需要时手工截断 |
| `agentd.ssh_dir`(LXC `/run/plane-agentd/ssh`) | direct-ssh 的 ControlMaster 套接字(`<host_key 指纹 8 位>-<%C 哈希>`,空闲 60 s 后自动退出;手工关闭:`for s in /run/plane-agentd/ssh/*-*; do [ -S "$s" ] && ssh -O exit -o ControlPath="$s" x; done`)和由钉住公钥生成的 `known_hosts.<终端>`;root 0700,agentd 自己建。ssh-agent 在 `/tmp/agentd-agent-*`(每个 tick 进程一个,退出时删除) |
| `state/agentd.lock` | tick 互斥锁 | | `state/agentd.lock` | tick 互斥锁 |
| `state/runs/<run_id>/` | `transport: local` 终端的运行目录:`prompt.md run.sh pid.txt started.txt exit.txt stderr.txt` + 执行器输出 | | `state/runs/<run_id>/` | `transport: local` 终端的运行目录:`prompt.md run.sh pid.txt started.txt exit.txt stderr.txt` + 执行器输出 |
| oregon `/home/ubuntu/.gwent-board/agentd-runs/<run_id>/` | 远程 Linux 终端(`run_root`):同上,另有 `launch.pid`(启动时的 `$!`,与 `pid.txt` 相同);claude 输出 `out.json` | | oregon `/home/ubuntu/.gwent-board/agentd-runs/<run_id>/` | 远程 Linux 终端(`run_root`):同上,另有 `launch.pid`(启动时的 `$!`,与 `pid.txt` 相同);claude 输出 `out.json` |
@@ -76,7 +102,10 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
| 现象 | 原因 / 处理 | | 现象 | 原因 / 处理 |
|---|---| |---|---|
| `no Plane token in the environment` / tick 报 `no Plane token` | 没在 `agent-access.py exec --secret-env …` 下运行,或键名不对;`check-config` 看 MISSING | | `no Plane token in the environment` / tick 报 `no Plane token` | 没在 `agent-access.py exec --secret-env …` 下运行,或键名不对;`check-config` 看 MISSING |
| `skip: another tick holds state/agentd.lock` | 上一 tick 还在跑(win10 上传慢);正常。持续出现则查是否有卡死的 agentd 进程 | | `skip: another tick holds state/agentd.lock` | 上一 tick 还在跑(win10 上传慢);正常(`tick` 此时退出码 75)。持续出现则查是否有卡死的 agentd 进程 |
| Todo 项一直不动、心跳看着空闲 | 看 `heartbeat.json` 的 `skipped`:`concurrency n/m (GW-x)` 槽位被谁占着;`serialize_group … held by …`;`busy_process codex (… not started by agentd)` = win10 上有不是 agentd 起的(或上一轮还没退干净的)codex.exe;`relay down`;`no route for assignee …` = 指派的用户不是终端。原因第一次出现时 `agentd.log`/`cron.log` 有一行 `GW-n not dispatched (<终端>): <原因>` |
| `webhook.log` 里 `decision=401` | 签名不对:Plane 侧密钥轮换后接收端没重启,或不是 Plane 发的。`systemctl restart plane-agentd-webhook` |
| Plane 建 webhook 报 "The webhook URL is not allowed…" | Plane 的 SSRF 防护拒绝内网地址:Plane api/worker 环境需 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(design §10) |
| `[worker] 第 k 轮失败:类型=crash(临时性)…重试 n/N` | 网络/中继/429/5xx:按 `backoff_minutes` 自动重试,续原线程;超过 `transient_max` 转 Waiting | | `[worker] 第 k 轮失败:类型=crash(临时性)…重试 n/N` | 网络/中继/429/5xx:按 `backoff_minutes` 自动重试,续原线程;超过 `transient_max` 转 Waiting |
| `类型=timeout` | 超过 `timeout_hours`(或 `budget:2h` 标签);进程树已结束,转 Waiting。看运行目录日志决定拆分任务或加预算 | | `类型=timeout` | 超过 `timeout_hours`(或 `budget:2h` 标签);进程树已结束,转 Waiting。看运行目录日志决定拆分任务或加预算 |
| `类型=crash`(非临时)且是续线程 | 自动在新线程重跑一次(附交接摘要);再失败转 Waiting | | `类型=crash`(非临时)且是续线程 | 自动在新线程重跑一次(附交接摘要);再失败转 Waiting |
@@ -88,7 +117,13 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
| `run GW-n-k: nothing at …; relaunching in GW-n-k-r2` | 启动没有落地:同一 run id、新目录 `<run_id>-r2`(再失败 -r3…)重新启动,旧目录不清空。运行目录里已有 pid 文件时 transport 拒绝清空(`HASPID`),下一 tick 探测后接管 | | `run GW-n-k: nothing at …; relaunching in GW-n-k-r2` | 启动没有落地:同一 run id、新目录 `<run_id>-r2`(再失败 -r3…)重新启动,旧目录不清空。运行目录里已有 pid 文件时 transport 拒绝清空(`HASPID`),下一 tick 探测后接管 |
| heartbeat `launching` 一直不消失 | 终端探测不通(中继坏),标记保留并占着该终端的槽位;中继恢复后自动接管或重启。人工处理:停 cron,在终端上看 `<run_root>/<run_id>/`,再从状态文件删掉该项的 `launching` | | heartbeat `launching` 一直不消失 | 终端探测不通(中继坏),标记保留并占着该终端的槽位;中继恢复后自动接管或重启。人工处理:停 cron,在终端上看 `<run_root>/<run_id>/`,再从状态文件删掉该项的 `launching` |
| `upload of … size mismatch` / `staging … size mismatch` | 远程写文件不完整(磁盘满、`run_root` 不可写);看远程 `run_root` | | `upload of … size mismatch` / `staging … size mismatch` | 远程写文件不完整(磁盘满、`run_root` 不可写);看远程 `run_root` |
| 远程 Linux 终端每步都慢 | 每次中继调用约 10–14 s(取 Vault 凭据 + SSH 握手)。一轮:启动 1 次(prompt 超过约 45 KB 时多几次分块上传)、每 tick poll 1 次、收尾 claude 约 3 次 | | 远程终端每步都慢 | `warpgate-ssh`(或 `via=fallback`)每次调用约 9–14 s(取 Vault 凭据 + 网关握手 + 密码认证 + 二跳)。`direct-ssh` 首次握手 win10 约 0.5 s、oregon 约 2 s,之后 60 s 内复用连接。看 `remote.log` 的 `ms=`。一轮的调用次数:启动 1 次(Warpgate 上 prompt 超过约 45 KB 时多几次分块上传;win10 经 Warpgate 约 8–10 次中继调用),每 tick poll 1 次,收尾 1 次(win10 另加删计划任务 1 次) |
| `SECURITY <终端>: … host key verification FAILED …` | 终端出示的主机公钥和配置里钉住的不一致:可能是重装、sshd 重新生成了主机密钥,也可能是中间人。**不会回退到 Warpgate**,该终端本 tick 标为 down(`heartbeat.down`),每个 tick 都会再报一次。先经第二通道核对(Warpgate:`agent-access.py … run --target oregon -- cat /etc/ssh/ssh_host_ed25519_key.pub`,win10 `win10-ps.sh "Get-Content C:\ProgramData\ssh\ssh_host_ed25519_key.pub"`),确认是合法变更后再更新 `terminals.<t>.ssh.host_key`(deploy-lxc §9 第 4 步)。新的 pin 会换一个 ControlMaster 套接字名(`<pin 指纹>-%C`),不会复用按旧 pin 校验过的连接。终端根本不提供 ssh-ed25519 主机密钥(`no matching host key type`)也按这一行处理。等核对期间要继续派单:把 `transport` 改回 `warpgate-ssh` |
| `<终端>: direct-ssh failed before the command ran (…); using the warpgate-ssh fallback for the rest of this tick` | 直连在命令运行前就失败了(tailscale 断、sshd 停、authorized_keys 缺或 ACL 不对、密钥被拒)。这个操作已在 Warpgate 上重做,本 tick 余下的也走 Warpgate,下个 tick 再试直连。`remote.log` 里会出现 `via=fallback`。查原因:`cli/agentd ping <终端> --transport direct-ssh -n 1`,括号里是 ssh 的原话(如 `Permission denied (publickey)`、`connect to host … Connection timed out`) |
| `direct-ssh unavailable in this process: AGENTD_SSH_KEY is not set …` / `ssh-add failed …` | 进程没拿到密钥(手工运行没加 `--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY`,或 cron 脚本因为 Vault 取不到键而去掉了它),或 Vault 里的值不是无口令的 OpenSSH 私钥。直连终端整次都走兜底;没配 `fallback` 的终端本 tick down |
| cron.log:`AGENTD_SSH_KEY not available from Vault: tick repeated without it …` | Vault 键 `AGENTD_SSH_KEY` 被删、改名或不可读,agent-access.py 拒绝了整个 exec。tick 已不带密钥重跑,直连终端每个 tick 都走 Warpgate 兜底(慢,但不停派单)。修复:按 deploy-lxc §9 第 1 步恢复这个键;`agent-access.py … list` 能看到键名是否还在 |
| 手工 `exec … --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY` 只打印 `access-hub: credential_unavailable` | 同上,Vault 里没有这个键(也可能是某个 Plane token 键)。手工运行时去掉 `AGENTD_SSH_KEY` 那个参数即可 |
| `ssh administrator@… timeout after 150s` / `ssh … exit 255: …`(无 fallback 行) | 直连的命令**可能已经运行**(超时、会话中途断开、认不出的 255),按规则不换路径重试:终端本 tick down;启动的话保留 `launching` 标记,下一 tick 探测后接管或在 `-r2` 重启。win10 上每次都超时:多半是 stdin 没到 PowerShell(sshd 默认 shell 被改成了 PowerShell),见 deploy-lxc §9 第 3 步 |
| 工作项在运行中被人改了状态 | 运行还没结束:下一 tick 取消(kill + `[worker] 运行已取消…` 评论),不改状态。已经结束(`exit.txt` 已写):结果照常回帖,但不再改状态/重试 | | 工作项在运行中被人改了状态 | 运行还没结束:下一 tick 取消(kill + `[worker] 运行已取消…` 评论),不改状态。已经结束(`exit.txt` 已写):结果照常回帖,但不再改状态/重试 |
| Plane 回帖失败 | 在途记录保留,下一 tick 重新 collect 并回帖(可能重复一条评论) | | Plane 回帖失败 | 在途记录保留,下一 tick 重新 collect 并回帖(可能重复一条评论) |
| 想手工终止一轮 | 把工作项移出 In Progress(如 Waiting)即可,下一 tick agentd 自己 kill 并释放槽位。中继不通、kill 一直失败时:在终端上手工结束(Linux 终端 `kill -TERM -- -$(cat pid.txt); pkill -TERM -s $(cat pid.txt)`,win10 `taskkill /T /F /PID <pid.txt>`),停 cron 后从 `state/agentd-state.json` 删掉该项的 `inflight` | | 想手工终止一轮 | 把工作项移出 In Progress(如 Waiting)即可,下一 tick agentd 自己 kill 并释放槽位。中继不通、kill 一直失败时:在终端上手工结束(Linux 终端 `kill -TERM -- -$(cat pid.txt); pkill -TERM -s $(cat pid.txt)`,win10 `taskkill /T /F /PID <pid.txt>`),停 cron 后从 `state/agentd-state.json` 删掉该项的 `inflight` |
@@ -100,6 +135,35 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
3. oregon 恢复 cron:`crontab ~/.gwent-board/crontab.backup-20261021-pad11`(即 `*/2 * * * * /home/ubuntu/Documents/wjj/project/plane-agentd/cron/agentd-tick.sh`)。`cron/agentd-tick.sh` 用 `config/agentd.oregon.yaml`(`state_dir: ~/.gwent-board/agentd`,日志 `~/.gwent-board/agentd/agentd.log`、`~/.gwent-board/agentd-cron.log`),与 LXC 配置只差路径。 3. oregon 恢复 cron:`crontab ~/.gwent-board/crontab.backup-20261021-pad11`(即 `*/2 * * * * /home/ubuntu/Documents/wjj/project/plane-agentd/cron/agentd-tick.sh`)。`cron/agentd-tick.sh` 用 `config/agentd.oregon.yaml`(`state_dir: ~/.gwent-board/agentd`,日志 `~/.gwent-board/agentd/agentd.log`、`~/.gwent-board/agentd-cron.log`),与 LXC 配置只差路径。
4. 看 oregon 的 `~/.gwent-board/agentd/heartbeat.json` 更新。再切回 LXC = 反过来做一遍(docs/deploy-lxc.md §5)。 4. 看 oregon 的 `~/.gwent-board/agentd/heartbeat.json` 更新。再切回 LXC = 反过来做一遍(docs/deploy-lxc.md §5)。
## 上线记录(2026-10-22,PAD-12:直连 SSH + 单次调用派单)
- 代码 5192e04:direct-ssh 传输(会话建立前失败才回落 Warpgate,主机公钥不符绝不回落)、主机公钥固定、Vault 密钥载入 ssh-agent、复用连接、win10 一次调用启动和一次调用收尾、`remote.log`、`agentd ping`。148 个单测通过;经正确性、安全两轮 opus 审查后修正。
- 密钥:在 LXC 的 `/dev/shm` 生成 ed25519,私钥只存在 Vault `AGENTD_SSH_KEY`(指纹 `SHA256:+6uooRFFB3tX4fQ+Sfl+3FmLm9KJpPYpSPzoTcCq/2M`),临时文件已删。公钥行 `restrict,from="100.69.78.124" ssh-ed25519 … plane-agentd@agentd-01`:
- oregon:`~ubuntu/.ssh/authorized_keys`,备份 `.bak-pad12-20261022-025955`;
- win10:`C:\ProgramData\ssh\administrators_authorized_keys`,备份 `.bak-pad12-20261022-110041`,ACL 未变(SYSTEM 和 Administrators 完全控制)。win10 的 sshd 是 OpenSSH_9.5p1,DefaultShell 未设(即 cmd.exe)。
- 两台的主机公钥经 ssh-keyscan 和经 Warpgate 读公钥文件两路核对一致后,固定写进配置。
- 部署 03:01(持 `cron.lock` 执行 git pull 和 ping)。
- ping:oregon 首次 3.9 s(含建立主连接),复用 0.77 s;win10 首次 0.52 s,复用 0.17 s。
- 部署时 win10 的第一次调用有一次 60 s 超时。随后 3 轮新建主连接的首次调用都是 0.52 s,复现不出,推测是 tailnet 线路空闲后刚恢复。这类超时不回落,该终端本 tick 算不可用、下个 tick 重试;在 `remote.log` 里继续观察。
- 冒烟(真实 webhook):
- GW-27 win10:建项 03:06:17.9 → 开工评论 03:06:28.4,**10.5 s**(之前 95 s)。其中 busy 0.22 s、start 0.32 s;收尾 read、fetch、forget 三次调用共 0.9 s。
- GW-28 oregon:建项 03:10:09.5 → 开工 03:10:23.6,**14.1 s**(start 4.1 s,含新建主连接);03:10:32.6 结果已贴回。
- 剩下的延迟在收尾:运行结束后要等下一次 tick(cron 2 分钟,或别的投递触发)才收。迁 Temporal(PAD-8)时用定时器轮询解决。
- 回滚:把该终端的 `transport` 改回 `warpgate-ssh` 再 pull。撤销访问:删掉两台 authorized_keys 里的那一行,再对 `/run/plane-agentd/ssh/` 下的主连接执行 `ssh -O exit`(见 deploy-lxc §9)。
## 上线记录(2026-10-22,PAD-7:Plane 侧 webhook 启用)
- 01:14 Plane VM 506:`plane.env` 与渲染 compose 设 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(备份 `*.bak-pad7-20261022-011453`),`--dry-run` 确认后只重建 api、worker、webhook-consumer、outbox-poller、automation-consumer、beat-worker(01:15:43–01:16:33);01:18:26 实测 API 已返回 200(之前未测,停机上限约 2.5 分钟),api 容器 4 分钟内转 healthy。
- 01:19 `tools/plane-webhook.py create http://10.77.10.249:8790/plane` → 201(白名单生效),密钥写入 Vault `PLANE_WEBHOOK_SECRET`(替换临时值);01:22:30 `systemctl enable --now plane-agentd-webhook`(LXC 代码 1b4d965,单元文件与仓库一致);01:22:57 `patch c039c471-… is_active=true`。
- Plane 建出的是 **v2** webhook:负载 `{version: v2, event, entity_type, data, …}`,由 `plane/webhook/bgtasks/webhook_task.py` 的 `send_webhook` 发出,签名 `HMAC-SHA256(secret, json.dumps(payload))`,与接收端一致;`data.project` 路由到 GWENT 正常。
- 实测(真实投递):评论 GW-25 → 01:24:34 投递到达 → 01:24:37 tick → 4.6 s 完成;GW-25 改 Done 同样 3 s 后 tick。建项 GW-26(01:25:14)→ 01:25:15 投递 → 01:25:18 tick 开始 → 01:26:49 `[worker] 第 1 轮开始`(win10 codex/astra):**建项 → 开工 95 s,其中 tick 本身 91 s**(中继、上传任务书、计划任务启动);只靠 cron 时在这之前还要平均多等约 60 s。剩下的成本都在中继调用上(PAD-12)。
- cron `*/2` 保留为兜底(同一把 `cron.lock`);oregon crontab 仍为空,只有一个调度器。
## 上线记录(2026-10-21,PAD-7:webhook 接收端)
- 13:25 LXC 上 `plane-agentd-webhook` 启用(`6dc7e72`),cron 不变。**Plane 侧 webhook 未建成**:Plane 3.3.3 对 `http://10.77.10.249:8790/plane` 返回 400 "The webhook URL is not allowed. Provide a publicly reachable HTTP or HTTPS URL."(SSRF 防护拒绝内网地址,见 design §10);需要在 Plane 的 api/worker 环境加 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`,之后按 deploy-lxc §8 第 2–5 步建 webhook。在此之前 Vault 里的 `PLANE_WEBHOOK_SECRET` 是临时随机值(`plane_wh_interim_…`),`plane-webhook.py create` 会用 Plane 生成的密钥覆盖它。
- 验收用的是在 LXC 上按 Plane 格式签名的合成投递(`/root/lxc_send.sh good|bad|missing|other`):错签名、缺签名 → 401,不触发;别的项目 → 202;正确签名 → 200,3 s 后 tick 开始。tick 运行中又来一条 → 结束后补跑一次(13:27 两条投递:224 s 的 tick——收尾 GW-15、在 win10 启动 GW-5——之后接着 24 s 的补跑);那段时间 cron 的 tick 撞锁立即退出,cron 本身照常每 2 分钟触发。
- 冒烟 GW-17(oregon claude-code):建项 13:43:15.8 → 合成投递 13:43:41(经 oregon→LXC 中继发出,人为多了约 25 s)→ tick 开始 13:43:44 → `[worker] 第 1 轮开始` 13:44:56 → `[review] ok` 13:47:54,PASS 并移到 Done。**投递 → tick 开始 = 3 s;tick 开始 → 开始评论 = 72 s**(这一 tick 先 poll win10 上在途的 GW-5,再经中继启动 oregon;同期 cron 的启动 tick 也要 70–77 s)。真实 Plane 投递约 1 s 到达时,建项 → 开始评论预计约 75 s,而只靠 cron 是平均约 60 s 等待 + 同样的 tick 时长。≤10 s 的目标在"开始评论"这个口径上达不到:瓶颈是中继调用(每次约 10 s,vault + Warpgate SSH)和启动流程,不是触发方式。
- GW-16(第一次冒烟):webhook 触发的 tick 里 LXC→中继报 `access-hub: config_private_transport_failed`(LXC 经 tailnet 取 `nas…/identity/agent.json` 15 s 超时;13:38 的 cron tick 也撞上一次),启动失败;下一 cron tick 探测无运行目录 → `GW-16-1-r2` 重新启动,PASS(防重复启动机制按设计工作)。
- 重启:tick 运行中 `systemctl restart plane-agentd-webhook`,tick(在 `systemd-run --scope` 里)没被杀、照常结束(heartbeat 13:50:10,errors=[]);接收端约 19 s 后重新监听(取 vault 密钥 + 启动)。
## 切换记录(2026-10-21,PAD-11:调度器迁到 LXC agentd-01) ## 切换记录(2026-10-21,PAD-11:调度器迁到 LXC agentd-01)
- 代码:防重复启动(`launching` 标记 + 探测接管/换目录重启)与"中继不通的终端本 tick 跳过"随本次一起上线(4f45d36);LXC 用仓库里的 `config/agentd.yaml`,oregon 回滚用 `config/agentd.oregon.yaml`。 - 代码:防重复启动(`launching` 标记 + 探测接管/换目录重启)与"中继不通的终端本 tick 跳过"随本次一起上线(4f45d36);LXC 用仓库里的 `config/agentd.yaml`,oregon 回滚用 `config/agentd.oregon.yaml`。
- 切换前 LXC 手工 `tick --once`(oregon cron 仍在跑,Plane 上无 Todo 项):空闲、无错误;LXC 经 helper 探测 win10(`codex` 计数)与经中继探测 oregon 运行目录均正常。 - 切换前 LXC 手工 `tick --once`(oregon cron 仍在跑,Plane 上无 Todo 项):空闲、无错误;LXC 经 helper 探测 win10(`codex` 计数)与经中继探测 oregon 运行目录均正常。
@@ -120,3 +184,14 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
- 冒烟(经 Warpgate 远程启动):GW-10 第 1 轮 claude-code/sonnet PASS(`[review] ok`,启动 08:08:54 → 结束 08:10:17,session `fa3be0f7…`);评论"请回复 `[review] ok 2`"并移回 Todo 后第 2 轮续同一 session(`out.json` 的 `session_id` 不变),回 `[review] ok 2` 转 Review。启动只用 1 次中继调用(约 12 s);72 KB 的 prompt 走分块上传(约 75 s)。 - 冒烟(经 Warpgate 远程启动):GW-10 第 1 轮 claude-code/sonnet PASS(`[review] ok`,启动 08:08:54 → 结束 08:10:17,session `fa3be0f7…`);评论"请回复 `[review] ok 2`"并移回 Todo 后第 2 轮续同一 session(`out.json` 的 `session_id` 不变),回 `[review] ok 2` 转 Review。启动只用 1 次中继调用(约 12 s);72 KB 的 prompt 走分块上传(约 75 s)。
- kill:直接经中继对 oregon 起 `sleep 600 & sleep 500` 的运行再 kill,两个 sleep 都被结束(进程组 + 会话),`pid.txt` 与 `launch.pid` 相同(setsid 没有再 fork)。经 Plane 的取消冒烟(运行中移到 Waiting)因 Plane 当时 502(08:14 起)未做,待补。 - kill:直接经中继对 oregon 起 `sleep 600 & sleep 500` 的运行再 kill,两个 sleep 都被结束(进程组 + 会话),`pid.txt` 与 `launch.pid` 相同(setsid 没有再 fork)。经 Plane 的取消冒烟(运行中移到 Waiting)因 Plane 当时 502(08:14 起)未做,待补。
## 回滚记录(2026-10-21:调度器回到 oregon)
- 原因:LXC agentd-01 经 tailnet 到 NAS 的路径走 DERP 中继,取 vault 身份端点(`nas…/identity/agent.json`)TCP 连接 30/30 超时,tick 里的中继调用不稳定。网络修好后再切回 LXC(docs/deploy-lxc.md §5)。
- 做法:LXC 删 cron 行、停用 `plane-agentd-webhook`、等 `agentd.lock`/`cron.lock` 空闲;`agentd-state.json`(sha256 7201077b…eb87,两端一致)拷到 oregon `~/.gwent-board/agentd/`(旧文件备份为 `.bak-<时间戳>`);oregon 恢复 `*/2` 的 `cron/agentd-tick.sh`。LXC 上的文件、服务单元均未删除。`config/agentd.oregon.yaml` 无需改动(`check-config` 通过)。
- 验证:oregon 首个 tick 14:20:01 接管,在途的 GW-5-1(win10 Codex)已结束,结果回帖并进入 Review;heartbeat `errors=[]`、`inflight=[]`,下一 tick(14:22)同样。
- 重新切回 LXC 之前:先 `crontab -r`(oregon)并等 tick 结束,再按 deploy-lxc §5 把状态拷回 LXC,并重新 enable webhook 服务;任何时刻只能有一个调度器。
## 切换记录(2026-10-21,网络修复后重新切回 LXC)
- 网络:OPNsense 增加出站 NAT 静态端口规则 `Tailscale agentd-01 static-port`(10.77.10.249/32,UDP,WAN),LXC tailscaled 改听 UDP 41642;LXC 到 NAS 直连(`tailscale ping` 5 ms via 223.102.82.206:1549,身份端点 30/30 成功、中位 0.089 s)。详见 deploy-lxc §2.1。
- 切换:oregon crontab 清空(备份 `~/.gwent-board/crontab.backup-20261021-152423-recut`),状态文件(sha256 `54b747db…7155`,两端一致;LXC 旧文件备份 `agentd-state.json.bak-20261021-152504`)拷到 LXC,装 LXC crontab `*/2 … cron/agentd-tick-lxc.sh`。在途的 GW-18 带过去(LXC 上为 GW-18-2),之后的 run-1 结果已回帖。heartbeat errors=[]、down=[]、skipped=[],只有一个调度器。
- Webhook(PAD-7)仍未启用:`plane-agentd-webhook` inactive/disabled;Plane 的 `WEBHOOK_ALLOWED_IPS` 尚未设置(见 deploy-lxc §8 的具体改法,Plane 实为 VM 506 / 10.77.10.69)。PLANE_WEBHOOK_SECRET 仍是临时值。待办:应用该改动后做 §8 第 2–5 步;考虑轮换 Plane 的 AES_SECRET_KEY(排查时被打印进了一次会话记录)。
+105
View File
@@ -0,0 +1,105 @@
# 使用说明:在 Plane 上派活给各机器上的 agent
核对依据:`agentd/core.py`、`agentd/webhook.py`、`config/agentd.yaml`、LXC 上的实际状态、Plane GWENT 项目的标签和成员(2026-10-22)。代码或配置改了要同步改这里。
## 1. 派单需要的字段
- **必填**:状态设为 `Todo`,指派人里至少有一个终端账号(`window10` 或 `oregon`)。标题加描述就是任务书,原样进入 prompt。标签可选,模块与调度无关。
- 有多个指派人时,按 Plane 返回的顺序,第一个能对上终端的账号生效(`core.py` 的 `route()`)。所以"kappiee + window10"会派到 win10。
- **只指派 `kappiee`、指派 `mac`(暂无终端)或不指派**:`route()` 返回空,这一项被静默跳过。Plane 上不会有任何评论,只在 `heartbeat.json` 的 `skipped` 里记一条 `no route for assignee kappiee`,`agentd.log` 里在原因第一次出现时记一行。
- 只有 `Todo` 状态的项会被拾起(`pick()`)。Backlog、Waiting、Review 永远不会自动执行。唯一例外是处于 In Progress、等待临时性重试的项。
## 2. 终端、执行器和默认值
| Plane 指派人 | 终端 | 执行器 | 默认模型 / effort / 档位 | 运行目录(日志和输出) |
|---|---|---|---|---|
| `window10` | win10(Tailscale 直连 SSH,Warpgate 兜底;Windows 计划任务) | codex(`codex exec`,`workspace-write`) | `gpt-6.1-sol` / `medium` / `fast` | `C:\…\gwent\runtime\agentd\<run_id>\`(`prompt.md run.ps1 events.jsonl last.md stderr.txt exit.txt`) |
| `oregon` | oregon(Tailscale 直连 SSH,Warpgate 兜底;`setsid nohup`) | claude-code(`claude -p --permission-mode acceptEdits`) | `sonnet` / (不传 effort)/ — | `/home/ubuntu/.gwent-board/agentd-runs/<run_id>/`(`out.json` 等) |
| `kappiee` / `mac` / 无人 | 不派 | — | — | — |
- 两个终端的并发都是 1,按终端计数,跨项目共享。
- win10 设了 `busy_processes: [codex]`:agentd 自己在 win10 上没有在途运行、却发现有 `codex.exe` 在跑时,不派单(`busy_process codex (… not started by agentd)`)。
- oregon 的运行直接在 `/home/ubuntu/Documents/wjj/project/gwent` 里改文件。这正是指挥所用的 checkout。
## 3. 代码里实际生效的标签
Plane 里目前只建了 4 个标签:`model:astra`、`model:sol`、`effort:high`、`needs-game`。其余前缀的标签要先在 Plane 里建出来才能挂上。
| 标签 | 效果(`route()`) |
|---|---|
| `model:astra` | 改用 gpt-6-astra,effort high(来自 by_label 配置) |
| `model:sol` | 改用 gpt-6.1-sol,tier fast |
| `model:<x>`(其他任意) | 模型名原样改成 x,例如 oregon 上的 `model:opus`。模型名与执行器相关:`model:astra` 挂在 oregon 的项上会让 claude 失败 |
| `effort:<x>` | 只对 codex 生效。claude 的命令模板里没有 `{effort}`,所以被忽略 |
| `tier:<x>` / `tier:none` | 只对 codex 生效,改档位或去掉档位参数 |
| `needs-game` | 串行组 `game`:同组同时只跑一项;env 类失败会暂停整组,直到该项离开 Waiting。**`needs-*` 不是通用前缀**,只有配置里写了的才生效 |
| `exec:<executor>` | 要求同名执行器存在且终端支持。现在每个终端只有一个执行器,例如 `exec:codex` 挂在 oregon 上只会得到 `executor codex not available` 跳过,等于没用 |
| `cap:<x>` | 终端的 `capabilities` 里必须有 x。win10 有 `[android, game]`,oregon 没有,不满足就跳过 |
| `budget:2h` / `budget:90m` | 覆盖超时时间,默认 4 h |
多个标签按顺序处理,后面的覆盖前面的。
## 4. 一轮的完整流程
1. **触发**:Plane 的工作区级 webhook(`issue` 和 `issue_comment` 事件)约 1 s 内送到 `http://10.77.10.249:8790/plane`。
- 接收端用 HMAC 校验签名,只认 GWENT 项目,其他项目返回 202 忽略。
- 等 3 s,把这期间的投递合并成一次 tick。tick 跑的就是 cron 脚本。
- tick 运行中又来投递时,结束后再补跑一次。锁被占用(退出码 75)时每 15 s 重试,最多 2 min。
- 兜底:cron 每 2 分钟一次 tick,同一把 `cron.lock`。
- agentd 自己回帖、改状态也会产生投递,所以每个有动作的 tick 后面都会多一次空 tick(实测 11.1 s)。
2. **tick 的步骤**:先 poll 所有在途运行(每个一次远程调用:直连亚秒到约 2 s,走 Warpgate 时约 10 s),再按编号从小到大挑 Todo 项。依次检查:终端是否可达、并发槽位、串行组、busy 进程。
- 启动前先把 `launching` 标记落盘,防止重复启动。
- 启动后把项改为 In Progress 并指派给终端账号,以终端账号发评论 `[worker] 第 k 轮开始:codex / gpt-6-astra / high @ win10,新线程,run=GW-26-1。`
3. **实测耗时**(runbook 记录和 LXC 的 webhook.log):
- 建项到投递约 1 s,再加 3 s 合并窗口。
- **win10 启动的那次 tick 91.4 s**(GW-26:建项 01:25:14,开工评论 01:26:49)。
- oregon 启动的 tick 约 72 s。
- 空 tick 4.6–4.8 s。
- 以上是 Warpgate 时期的数字,瓶颈在中继调用,与触发方式无关。
- PAD-12 起两个终端都走 Tailscale 直连:win10 启动由约 10 次中继调用变成 busy 检查 + 1 次启动调用,收尾由约 7 次变成 2 次。目标是建项到开工评论 ≤ 15 s,上线后在这里补实测。直连不通时这一轮自动走 Warpgate,只是慢,派单行为不变(design §9)。
4. **结束时**:以终端账号回帖。
- 第一行是 `[codex] 第 k 轮结束(exit=…,run=…,终端 win10,日志 <运行目录>)`,下面是 agent 的最后消息原文(`<pre>` 包裹,超过 6000 字截断,并注明全文所在路径)。
- 再按最后消息首行的标签流转(`transition()`):
| 首行 | 状态 / 指派人 |
|---|---|
| `[review]` | Review,指派 `oregon`(reviewer) |
| `[question]` / `[blocked]` | Waiting,指派 `oregon`(answerer) |
| `[progress]` | 回到 Todo,留在原终端,下一 tick 续同一线程。连续 12 轮后转 Waiting |
| 没有标签但 exit=0 | 按 review 处理,并注明"未按协议标注首行" |
注意:Waiting 和 Review 都指派给 **oregon,不是 kappiee**。没有任何东西会自动处理这两个状态,由指挥(或你)介入。
5. **失败**:回帖 `[worker] 第 k 轮失败:类型=crash|timeout|env|logic,run,exit,重试 n/3。…` 并附上最后一帧输出。
- 临时性错误(stderr 尾部匹配超时、429、5xx、连接被拒等)自动重试,退避 2/5/10 min,续原线程,状态保持 In Progress。
- 续线程那一轮出现非临时性失败时,立即在新线程重跑一次,并附交接摘要。
- 其余情况转 Waiting,指派 oregon。超时会先 kill 进程树。逻辑类失败永不自动重试。
## 5. 回答问题和续跑
1. 在项上发评论回答,开头不要写 `[worker]`,这类评论永远会被跳过。
2. **把状态改回 Todo,并把指派人改回原终端**(例如 `window10`)。这一步是坑:如果指派人还是 oregon 就改 Todo,会派给 oregon 的 Claude。因为终端或执行器变了,会**重建线程**并附交接摘要。
3. 同一终端、同一执行器、线程续了不到 8 轮时,走 `codex exec resume` 或 `claude --resume`,只发送上次之后的新评论(按评论 ID 判断)。
4. 单发评论只会触发一次空 tick,**不会**让 agent 动起来,必须改回 Todo。运行期间发的评论,跑着的那一轮看不到,下一轮才带上。
## 6. 取消和停止
- **把运行中的项移出 In Progress**(改成 Waiting、Backlog、Cancelled 都可以)。下一 tick 会 kill 进程树,回帖 `[worker] 运行已取消:工作项状态被改为 X(run=…)`,释放槽位和串行组,保留线程 ID(`cancel()`)。
- 如果移到的是 **Todo**:同一 tick 里先取消,再立刻按新一轮重新派发(续原线程),相当于"带上新评论重启"。
- 运行已经结束时(`exit.txt` 已写)再移动:结果照常回帖,但不再改状态,也不重试。
- 暂停整个终端:把它的 `concurrency` 设为 0(需要改配置,不能在 Plane 里操作)。
## 7. 日志和状态的位置(LXC)
- `/var/log/plane-agentd/{agentd.log, cron.log, webhook.log}`
- `/var/lib/plane-agentd/{agentd-state.json, heartbeat.json}`:heartbeat 里有 `skipped` 及原因、`inflight`、`down`。
- 健康检查:`GET http://10.77.10.249:8790/healthz`
- 任务产出本身在终端的 repo 里,gwent 的惯例是 win10 的 `runtime/edits/GW-k/`。
## 8. 用手机试一下(冒烟测试,5 步)
1. GWENT 新建工作项,标题 `冒烟 oregon 往返`。描述:`只回复一行 [review] ok;不要读写或修改任何文件,不要运行命令。` oregon 的 Claude 是 acceptEdits,并且在指挥用的 checkout 里运行,所以这句必须写。
2. 指派人**只选 oregon**,不加标签(默认 sonnet),状态设为 **Todo**。
3. 约 1–1.5 分钟后:状态变成 In Progress,出现 oregon 发的评论 `[worker] 第 1 轮开始:claude-code / sonnet @ oregon,新线程,run=GW-n-1。`
4. 再过约 1–3 分钟(历次冒烟 1.5–3 min):出现评论 `[claude-code] 第 1 轮结束(exit=0 …)` 加 `[review] ok`,状态变为 Review、指派 oregon。
5. 可选,测续线程:评论 `请只回复 [review] ok 2`,状态改回 Todo(指派保持 oregon)。开始评论会写"续线程 xxxx",结束后回 `[review] ok 2`。最后手动改成 Done。
- 超过 3 分钟没动静:看 heartbeat 的 `skipped`,常见原因是 oregon 槽位被占,或指派人不对。
## 9. 通知和手机 App(2026-10-22 核实)
- Plane 官方手机 App 可以登录自建的 Commercial 版(选 "Self hosted sign in",填 `https://plane.kappiee.com`)。plane.kappiee.com 只对内网和 tailnet 开放,手机要开 Tailscale。
- **自建实例没有手机推送**:Plane 文档原文 "Push notifications are currently available only for Plane Cloud users."(docs.plane.so/devices/mobile)。App 里只能在收件箱看到通知。
- 收件箱只收你被指派、被 @、创建或订阅的项。Waiting 和 Review 现在都指派给 oregon,不会进你的收件箱。需要你处理的项,由指挥改成指派 kappiee 或 @你。
- 真正推到手机要走外部渠道:给 Plane 配 SMTP 发邮件,或用 ntfy 之类。计划放在最后(PAD-13)。agentd 里现在的 `notify: {kind: none}` 是空钩子。
+827
View File
@@ -0,0 +1,827 @@
"""direct-ssh (PAD-12): config validation, channel selection and fallback rules, the ssh command line, the Windows
bootstrap / payload, single-call launch and collect. Nothing here opens a network connection: the POSIX direct
channel is replaced by a local `bash -c <loader>` fed on stdin, so the real loader and scripts run."""
import base64
import hashlib
import io
import json
import os
import shutil
import subprocess
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from unittest import mock
from agentd import ROOT, cli, config
from agentd.config import Config
from agentd.executors import Run, make_executor
from agentd.transports import (CallLog, HostKeyError, RoutedTransport, TransportError, Unreached, make_transport,
parse_fetch)
from agentd.transports import direct_ssh
from agentd.transports.direct_ssh import (BOOTSTRAP, PS_COMMAND, SH_COMMAND, DirectChannel, KeyAgent, classify,
ps_payload)
from agentd.transports.warpgate_ssh import PosixWarpgateTransport, WarpgateSshTransport, WindowsWarpgateTransport
from tests.fakes import config_data
SCHEMA = json.loads((ROOT / "config" / "schema.json").read_text())
WIN_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzLX7QNHDFYwTgY4wNl0ihZryIyKgpp9HPpPDWbrTX3"
LIN_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICUoI6W7npX+h/kHcXX3iy2XAWbNusrOjg28Xed75wWr"
SECRET = "-----BEGIN OPENSSH PRIVATE KEY-----\nNOT-A-REAL-KEY-but-must-never-leak\n-----END OPENSSH PRIVATE KEY-----\n"
def direct_data(root="/srv/x"):
d = config_data()
d["vault"] = {"client": "/opt/vault/agent-access.py", "config": "https://vault.example/identity.json"}
d["agentd"]["ssh_dir"] = str(Path(root) / "ssh")
d["terminals"]["win10"].update(transport="direct-ssh", fallback="warpgate-ssh",
ssh={"host": "100.123.209.19", "user": "administrator", "port": 22, "host_key": WIN_KEY})
d["terminals"]["lin"] = {"transport": "direct-ssh", "target": "lin", "os": "linux", "launch": "nohup",
"repo": str(Path(root) / "repo"), "run_root": str(Path(root) / "runs"), "plane_user": "oregon",
"token_key": "TOK_LIN", "concurrency": 1, "executors": ["claude-code", "shell"],
"ssh": {"host": "100.127.58.78", "user": "ubuntu", "host_key": LIN_KEY},
"fallback": "warpgate-ssh"}
return d
class FakeAgent:
sock, pub, error = "/tmp/agentd-agent-x/agent.sock", "/tmp/agentd-agent-x/agentd.pub", None
class ConfigTest(unittest.TestCase):
def errs(self, d):
return config.validate(d)
def test_direct_config_is_valid_and_live_config_uses_it(self):
self.assertEqual(self.errs(direct_data()), [])
self.assertEqual(config.mini_validate(direct_data(), SCHEMA), [])
cfg = config.load(ROOT / "config" / "agentd.yaml")
for term in ("win10", "oregon"):
t = cfg.terminals[term]
self.assertEqual((t["transport"], t["fallback"], t["target"]), ("direct-ssh", "warpgate-ssh", term))
self.assertIsNone(config.host_key_error(t["ssh"]["host_key"]))
self.assertIn("--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY", cfg.secret_env_args())
config.load(ROOT / "config" / "agentd.oregon.yaml") # rollback config still valid
def test_bad_direct_settings_have_clear_errors(self):
cases = [
(lambda d: d["terminals"]["lin"].pop("ssh"), "direct-ssh needs ssh:"),
(lambda d: d["terminals"]["lin"]["ssh"].update(host_key="ssh-rsa AAAAB3Nza"), "must be 'ssh-ed25519 <base64>'"),
(lambda d: d["terminals"]["lin"]["ssh"].update(host_key=WIN_KEY[:-4]), "not an ssh-ed25519 public key"),
(lambda d: d["terminals"]["lin"]["ssh"].update(host_key="ssh-ed25519 !!!"), "not an ssh-ed25519 public key"),
(lambda d: d["terminals"]["lin"].pop("target"), "needs the Warpgate 'target'"),
(lambda d: d["terminals"]["lin"]["ssh"].update(host="-oProxyCommand=x"), "host and user must be non-empty"),
(lambda d: d["terminals"]["lin"]["ssh"].update(port=70000), "> 65535"),
(lambda d: d["terminals"]["lin"].update(run_root="~/runs"), "must be an absolute path"),
(lambda d: d["terminals"]["win10"].update(launch="nohup"), "direct-ssh on windows needs launch=schtasks"),
(lambda d: d["terminals"]["win10"].update(target="win11"), "(fallback) only reach target 'win10'"),
(lambda d: d["vault"].pop("config"), "fallback warpgate-ssh needs vault.config"),
(lambda d: d["terminals"]["oregon"].update(fallback="warpgate-ssh"), "only applies to transport direct-ssh"),
(lambda d: d["agentd"].update(ssh_dir="/" + "x" * 60), "too long for the ControlMaster socket"),
]
for mutate, fragment in cases:
d = direct_data()
mutate(d)
errs = self.errs(d)
self.assertTrue(any(fragment in e for e in errs), (fragment, errs))
def test_schema_rejects_unknown_ssh_key_and_fallback_value(self):
d = direct_data()
d["terminals"]["lin"]["ssh"]["password"] = "x"
self.assertTrue(any("password" in e for e in config.mini_validate(d, SCHEMA)))
d = direct_data()
d["terminals"]["lin"]["fallback"] = "local"
self.assertTrue(config.mini_validate(d, SCHEMA))
def test_key_env_is_always_scrubbed_and_only_requested_when_used(self):
cfg = Config(config_data()) # no direct-ssh terminal
self.assertNotIn("--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY", cfg.secret_env_args())
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET}):
self.assertNotIn("AGENTD_SSH_KEY", cfg.scrubbed_env())
d = direct_data()
d["agentd"]["ssh_key"] = "OTHER_KEY"
self.assertEqual(Config(d).secret_env_args()[-1], "--secret-env OTHER_KEY=OTHER_KEY")
class SelectionTest(unittest.TestCase):
def setUp(self):
self.cfg = Config(direct_data())
def test_direct_terminal_gets_direct_primary_and_warpgate_fallback(self):
t = make_transport(self.cfg, "win10")
self.assertIsInstance(t, RoutedTransport)
self.assertIsInstance(t.primary, WindowsWarpgateTransport)
self.assertIsInstance(t.primary.channel, DirectChannel)
self.assertEqual((t.fallback.channel.via, t.fallback.channel.stdin, t.fallback.target), ("fallback", False, "win10"))
self.assertEqual(t.os, "windows")
self.assertTrue(hasattr(t, "start_run"))
def test_forced_channel_has_no_fallback(self):
self.assertIsNone(make_transport(self.cfg, "lin", force="direct-ssh").fallback)
t = make_transport(self.cfg, "lin", force="warpgate-ssh")
self.assertIsInstance(t, PosixWarpgateTransport)
self.assertEqual(t.channel.via, "warpgate")
def test_rollback_to_warpgate_keeps_ssh_section(self):
d = direct_data()
d["terminals"]["lin"]["transport"] = "warpgate-ssh" # ssh/fallback stay, ignored
self.assertEqual(config.validate(d), [])
self.assertIsInstance(make_transport(Config(d), "lin"), PosixWarpgateTransport)
class ClassifyTest(unittest.TestCase):
def test_classification(self):
self.assertEqual(classify(0, ""), "ok")
self.assertEqual(classify(1, "ssh: connect to host x port 22: Connection timed out"), "ok") # command ran
for err in ("ssh: connect to host 100.1.2.3 port 22: Connection timed out",
"ssh: connect to host 100.1.2.3 port 22: Connection refused",
"ssh: connect to host 100.1.2.3 port 22: No route to host",
"ssh: Could not resolve hostname x: Name or service not known",
"Connection timed out during banner exchange",
"kex_exchange_identification: read: Connection reset by peer",
"Connection closed by 100.1.2.3 port 22",
"ubuntu@100.1.2.3: Permission denied (publickey).",
"sign_and_send_pubkey: signing failed for ED25519 \"x\" from agent: agent refused operation"):
self.assertEqual(classify(255, err), "unreached", err)
for err in ("@@@@@@@@@@@\n@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @\nHost key verification failed.",
"No ED25519 host key is known for agentd-win10 and you have requested strict checking.\n"
"Host key verification failed."):
self.assertEqual(classify(255, err), "hostkey")
for err in ("Connection to 100.1.2.3 closed by remote host.", "Timeout, server 100.1.2.3 not responding.",
"client_loop: send disconnect: Connection reset by peer", ""):
self.assertEqual(classify(255, err), "unknown", err) # the command may have run: no fallback
class Prim:
"""Stand-in transport recording calls; `fail` = exception raised by every call."""
os = "linux"
def __init__(self, fail=None):
self.fail, self.calls = fail, []
def join(self, *p):
return "/".join(p)
def start_run(self, *a):
self.calls.append(("start_run", a))
if self.fail:
raise self.fail
return "ok"
def process_count(self, name):
self.calls.append(("process_count", name))
if self.fail:
raise self.fail
return 0
class RoutedTest(unittest.TestCase):
def setUp(self):
self.events = []
self.log = CallLog(event=self.events.append)
def test_unreached_falls_back_once_and_sticks(self):
p, f = Prim(Unreached("ssh: connect to host x: Connection timed out")), Prim()
t = RoutedTransport("lin", p, f, self.log)
self.assertEqual(t.start_run("task", "/r", {}, "run.sh"), "ok")
self.assertEqual(t.process_count("codex"), 0)
self.assertEqual(len(p.calls), 1) # direct not retried within the tick
self.assertEqual([c[0] for c in f.calls], ["start_run", "process_count"])
self.assertEqual(len(self.events), 1)
self.assertIn("fallback", self.events[0])
self.assertEqual(t.join("a", "b"), "a/b") # pure helpers come from the primary
def test_host_key_and_after_command_failures_never_fall_back(self):
for exc in (HostKeyError("host key verification FAILED"), TransportError("ssh timeout after 150s")):
p, f = Prim(exc), Prim()
t = RoutedTransport("lin", p, f, self.log)
with self.assertRaises(type(exc)):
t.start_run("task", "/r", {}, "run.sh")
self.assertEqual(f.calls, [])
def test_no_fallback_configured_raises(self):
with self.assertRaises(Unreached):
RoutedTransport("lin", Prim(Unreached("down")), None, self.log).start_run("t", "/r", {}, "run.sh")
def test_busy_check_unreached_is_not_swallowed(self):
cfg = Config(direct_data())
for cls, term in ((PosixWarpgateTransport, "lin"), (WindowsWarpgateTransport, "win10")):
ch = mock.Mock(stdin=True)
ch.sh.side_effect = ch.ps.side_effect = Unreached("down")
with self.assertRaises(Unreached):
cls(cfg, term, channel=ch).process_count("codex")
ch.sh.side_effect = ch.ps.side_effect = TransportError("exit 1")
self.assertIsNone(cls(cfg, term, channel=ch).process_count("codex"))
class MissingKeyTest(unittest.TestCase):
def setUp(self):
direct_ssh._AGENTS.clear()
direct_ssh._KEYS.clear()
self.tmp = tempfile.TemporaryDirectory()
self.cfg = Config(direct_data(self.tmp.name))
def tearDown(self):
direct_ssh._AGENTS.clear()
self.tmp.cleanup()
def test_missing_key_makes_direct_unavailable_and_uses_the_fallback(self):
events = []
log = CallLog(event=events.append)
env = {k: v for k, v in os.environ.items() if k != "AGENTD_SSH_KEY"}
with mock.patch.dict(os.environ, env, clear=True), \
mock.patch("agentd.transports.direct_ssh.subprocess.run") as run:
t = make_transport(self.cfg, "lin", log=log)
t.fallback = Prim()
self.assertEqual(t.process_count("claude"), 0)
self.assertEqual(t.process_count("claude"), 0)
run.assert_not_called() # no ssh attempted at all
self.assertTrue(any("AGENTD_SSH_KEY is not set" in e for e in events))
self.assertEqual(sum("unavailable in this process" in e for e in events), 1) # once per process
self.assertEqual(sum("using the warpgate-ssh fallback" in e for e in events), 1) # once per tick
def test_empty_key_is_popped_and_reported(self):
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": " "}):
a = KeyAgent("AGENTD_SSH_KEY").start()
self.assertNotIn("AGENTD_SSH_KEY", os.environ)
self.assertIn("not set", a.error)
@unittest.skipUnless(all(shutil.which(b) for b in ("ssh-agent", "ssh-add", "ssh-keygen", "ssh")), "needs OpenSSH")
def test_real_agent_loads_key_from_env_and_cleans_up(self):
keyfile = Path(self.tmp.name) / "k"
subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "agentd-test", "-f", str(keyfile)], check=True)
key = keyfile.read_text()
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": key.replace("\n", "\\n")}): # Vault-style escapes
a = KeyAgent("AGENTD_SSH_KEY").start()
self.assertNotIn("AGENTD_SSH_KEY", os.environ)
try:
self.assertIsNone(a.error)
self.assertEqual(oct(os.stat(a.dir).st_mode & 0o777), "0o700")
self.assertEqual(Path(a.pub).read_text().split()[:2], (keyfile.with_suffix(".pub")).read_text().split()[:2])
for f in Path(a.dir).iterdir(): # nothing on disk holds the private key
if f.is_file():
self.assertNotIn("PRIVATE KEY", f.read_text())
finally:
a.close()
self.assertFalse(os.path.exists(a.dir))
self.assertIsNotNone(a.proc.poll())
def test_bad_key_reports_without_leaking(self):
if not all(shutil.which(b) for b in ("ssh-agent", "ssh-add", "ssh")):
self.skipTest("needs OpenSSH")
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET}):
a = KeyAgent("AGENTD_SSH_KEY").start()
self.assertIn("ssh-add failed", a.error)
self.assertNotIn("NOT-A-REAL-KEY", a.error)
self.assertFalse(os.path.exists(a.dir)) # agent stopped, socket dir gone
class SshCommandTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.cfg = Config(direct_data(self.tmp.name))
self.events, self.lines = [], []
self.log = CallLog(event=self.events.append)
self.log.call = lambda *a, **k: self.lines.append(a)
def tearDown(self):
self.tmp.cleanup()
def run_with(self, rc, err=b"", out=b"PONG=x\n"):
seen = {}
def fake_run(argv, input=None, stdout=None, stderr=None, env=None, timeout=None):
seen.update(argv=argv, input=input, env=env, timeout=timeout)
stdout.write(out)
stderr.write(err)
return mock.Mock(returncode=rc)
return seen, fake_run
def test_command_line_options_and_no_secret(self):
ch = DirectChannel(self.cfg, "lin", self.log)
seen, fake = self.run_with(0)
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET, "TOK_LIN": "plane-token", "SSH_AUTH_SOCK": "/x"}), \
mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake):
rc, out, _ = ch.sh("echo hi", 30, "ping")
argv = seen["argv"]
opts = {argv[i + 1] for i, a in enumerate(argv) if a == "-o"}
ssh_dir = Path(self.tmp.name) / "ssh"
for o in ("BatchMode=yes", "IdentitiesOnly=yes", "IdentityAgent=" + FakeAgent.sock, "IdentityFile=" + FakeAgent.pub,
"StrictHostKeyChecking=yes", "UserKnownHostsFile=%s/known_hosts.lin" % ssh_dir,
"GlobalKnownHostsFile=/dev/null", "HostKeyAlias=agentd-lin", "HostKeyAlgorithms=ssh-ed25519",
"ConnectTimeout=8", "ServerAliveInterval=15", "ServerAliveCountMax=3", "ForwardAgent=no",
"ForwardX11=no", "LogLevel=ERROR", "ControlMaster=auto", "ControlPersist=60",
"ControlPath=%s/%s-%%C" % (ssh_dir, ch.pin), "PasswordAuthentication=no"):
self.assertIn(o, opts)
self.assertEqual(argv[:3], ["ssh", "-F", "/dev/null"])
self.assertEqual(argv[-3:], ["--", "100.127.58.78", SH_COMMAND])
self.assertEqual(argv[argv.index("-l") + 1], "ubuntu")
self.assertEqual(seen["input"], b"echo hi") # the script travels on stdin
self.assertFalse(any("NOT-A-REAL-KEY" in a or "plane-token" in a for a in argv))
self.assertNotIn("AGENTD_SSH_KEY", seen["env"])
self.assertNotIn("TOK_LIN", seen["env"])
self.assertNotIn("SSH_AUTH_SOCK", seen["env"])
self.assertEqual((Path(ssh_dir) / "known_hosts.lin").read_text(), "agentd-lin %s\n" % LIN_KEY)
self.assertEqual(oct(os.stat(ssh_dir).st_mode & 0o777), "0o700")
self.assertEqual((rc, out), (0, "PONG=x\n"))
self.assertEqual(self.lines[-1][:3], ("lin", "direct", "ping"))
self.assertEqual(self.lines[-1][4], 0)
def test_host_key_failure_is_loud_sticky_and_not_unreached(self):
ch = DirectChannel(self.cfg, "lin", self.log)
seen, fake = self.run_with(255, b"No ED25519 host key is known for agentd-lin and you have requested strict "
b"checking.\r\nHost key verification failed.\r\n", b"")
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
mock.patch("agentd.transports.direct_ssh.subprocess.run", side_effect=fake) as run:
with self.assertRaises(HostKeyError) as cm:
ch.sh("true", 30)
self.assertNotIsInstance(cm.exception, Unreached)
with self.assertRaises(HostKeyError):
ch.sh("true", 30)
self.assertEqual(run.call_count, 1) # no second attempt in this tick
self.assertTrue(self.events and self.events[0].startswith("SECURITY lin:"))
def test_connect_failure_is_unreached_and_timeout_is_not(self):
ch = DirectChannel(self.cfg, "lin", self.log)
_, fake = self.run_with(255, b"ssh: connect to host 100.127.58.78 port 22: Connection timed out\n", b"")
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake):
with self.assertRaises(Unreached):
ch.sh("true", 30)
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
mock.patch("agentd.transports.direct_ssh.subprocess.run",
side_effect=subprocess.TimeoutExpired("ssh", 30)):
with self.assertRaises(TransportError) as cm:
ch.sh("true", 30)
self.assertNotIsInstance(cm.exception, Unreached) # may have run: resolve_launch probes instead
def test_windows_ps_uses_bootstrap_and_stdin(self):
ch = DirectChannel(self.cfg, "win10", self.log)
seen, fake = self.run_with(0, out=b"X=ok\r\n")
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake):
self.assertEqual(ch.ps("'X=ok'", 30), "X=ok\n")
self.assertEqual(seen["argv"][-2:], ["100.123.209.19", PS_COMMAND])
self.assertEqual(base64.b64decode(seen["input"]).decode("utf-8"), "'X=ok'")
class BootstrapTest(unittest.TestCase):
def test_encoded_command_round_trip_and_length(self):
value = PS_COMMAND.split()[-1]
self.assertEqual(base64.b64decode(value).decode("utf-16-le"), BOOTSTRAP)
self.assertLess(len(PS_COMMAND), 2000) # cmd.exe limit is 8191
self.assertIn("[Console]::In.ReadToEnd()", BOOTSTRAP)
script = "'é 中文 ''q'''\n"
self.assertTrue(ps_payload(script).isascii())
self.assertEqual(base64.b64decode(ps_payload(script)).decode("utf-8"), script)
class FakeWinChannel:
"""Direct-like Windows channel: answers the start / fetch scripts the way PowerShell would."""
stdin, name, via = True, "ssh fake", "direct"
def __init__(self, files=None, haspid=False):
self.calls, self.files, self.haspid = [], files or {}, haspid
def ps(self, script, timeout, op=None):
self.calls.append((op, script))
if op == "start":
if self.haspid:
return "HASPID\n"
out = ["WROTE=" + hashlib.sha256(base64.b64decode(b)).hexdigest()
for b in __import__("re").findall(r"FromBase64String\('([A-Za-z0-9+/=]*)'\)", script)]
return "\n".join(out + ["LAUNCHED=0"]) + "\n"
if op == "fetch":
ops = [l.split()[:4] for l in script.splitlines() if l.split(" ")[0] in ("AdRead", "AdGrep", "AdTail")]
out = []
for fn, i, name, *rest in ops:
text = self.files.get(name.strip("'"))
if fn == "AdRead":
out.append("M%s" % i if text is None else "F%s=%s" % (i, base64.b64encode(text.encode()).decode()))
else:
if fn == "AdGrep":
text = next((l for l in (text or "").splitlines() if "thread.started" in l), "")
else:
text = "\n".join((text or "").splitlines()[-int(rest[0]):])
out.append("F%s=%s" % (i, base64.b64encode(text.encode()).decode()))
return "\n".join(out) + "\n"
return "FORGOT=ok\n"
class WindowsDirectTest(unittest.TestCase):
def setUp(self):
self.cfg = config.load(ROOT / "config" / "agentd.yaml")
def transport(self, ch):
return WarpgateSshTransport(self.cfg, "win10", channel=ch)
def test_launch_is_one_call_with_exact_utf8_files(self):
ch = FakeWinChannel()
t = self.transport(ch)
ex = make_executor(self.cfg, "win10", "codex", t)
prompt = "中文 'quoted' \"dq\" $env:X `tick`\r\n" * 2000 # ~70 KB: far beyond any argv limit
h = ex.start(Run("GW-26-1", t.repo, prompt, "gpt-6-astra", "high"))
self.assertEqual([c[0] for c in ch.calls], ["start"])
script = ch.calls[0][1]
blobs = __import__("re").findall(r"FromBase64String\('([A-Za-z0-9+/=]*)'\)", script)
self.assertEqual(base64.b64decode(blobs[0]), prompt.encode("utf-8")) # UTF-8, no BOM, bytes exact
self.assertFalse(base64.b64decode(blobs[1]).startswith(b"\xef\xbb\xbf"))
self.assertIn("Set-Location -LiteralPath", base64.b64decode(blobs[1]).decode("utf-8"))
self.assertLess(script.index("'HASPID'; return"), script.index("WriteAllBytes"))
self.assertLess(script.index("WriteAllBytes"), script.index("schtasks /Run /TN 'agentd-GW-26-1'"))
self.assertTrue(h["run_dir"].endswith("\\runtime\\agentd\\GW-26-1"))
def test_launch_refuses_a_started_directory(self):
t = self.transport(FakeWinChannel(haspid=True))
with self.assertRaises(TransportError) as cm:
t.start_run("agentd-GW-1-1", "C:\\r\\GW-1-1", {"prompt.md": "p", "run.ps1": "r"}, "run.ps1")
self.assertIn("HASPID", str(cm.exception))
def test_collect_is_one_call_plus_forget(self):
files = {"exit.txt": "\ufeffexit=0 at 2026-10-22T10:00:00", "last.md": "[review] done\n",
"events.jsonl": '{"type":"x"}\n{"type":"thread.started","thread_id":"th-42"}\n{"type":"turn.completed"}',
"stderr.txt": "warn 1\nwarn 2"}
ch = FakeWinChannel(files)
ex = make_executor(self.cfg, "win10", "codex", self.transport(ch))
h = {"run_dir": "C:\\r\\GW-1-1", "task": "agentd-GW-1-1", "thread_id": None}
res = ex.collect(h)
ex.forget(h)
self.assertEqual([c[0] for c in ch.calls], ["fetch", "forget"])
self.assertEqual((res.last_message, res.exit_code, res.thread_id), ("[review] done", 0, "th-42"))
self.assertEqual(res.tail, 'warn 1\nwarn 2\n{"type":"turn.completed"}')
def test_warpgate_fetch_is_one_call_when_it_fits(self):
cfg = config.load(ROOT / "config" / "agentd.example.yaml")
t = WindowsWarpgateTransport(cfg, "win10")
self.assertFalse(t.channel.stdin)
ops = make_executor(cfg, "win10", "codex", t).COLLECT
script = t.fetch_script(t.join(t.default_run_root(), "GW-26-1-r2"), ops)
self.assertTrue(t.channel.fits(script))
calls = []
t.ps = lambda s, timeout=150: (calls.append(s), "\n".join("M%d" % i for i in range(len(ops))))[1]
self.assertEqual(t.fetch("C:\\r", ops), [None, None, None, "", ""])
self.assertEqual(len(calls), 1)
self.assertFalse(t.channel.fits("x" * 3000))
class LocalDirect:
"""POSIX direct channel stand-in: runs the real loader (SH_COMMAND) through a local login-style shell with the
script on stdin, exactly what sshd does with the remote command."""
stdin, name, via = True, "ssh local", "direct"
def __init__(self):
self.calls = []
def sh(self, script, timeout, op=None):
self.calls.append(op)
p = subprocess.run(["sh", "-c", SH_COMMAND], input=script.encode("utf-8"), # as sshd: $SHELL -c "<command>"
capture_output=True, timeout=timeout)
return p.returncode, p.stdout.decode("utf-8", "replace"), p.stderr.decode("utf-8", "replace")
class PosixDirectTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
root = Path(self.tmp.name)
(root / "repo").mkdir()
self.cfg = Config(direct_data(root))
self.ch = LocalDirect()
self.t = PosixWarpgateTransport(self.cfg, "lin", channel=self.ch)
def tearDown(self):
self.tmp.cleanup()
def test_loader_quoting_matches_sh_quote(self):
self.assertEqual(SH_COMMAND, "bash -c 'exec 3<&0 </dev/null; s=$(cat <&3); exec 3<&-; (eval \"$s\"); r=$?; "
"[ $r = 255 ] && r=254; exit $r'")
def test_loader_never_exits_255(self):
self.assertEqual(self.ch.sh("echo x >&2; exit 255", 10)[0], 254) # 255 = ssh's own "no session" code
self.assertEqual(self.ch.sh("exit 3", 10)[0], 3)
self.assertEqual(self.ch.sh("true", 10)[0], 0)
def test_big_prompt_launch_and_collect_are_one_call_each(self):
ex = make_executor(self.cfg, "lin", "shell", self.t)
prompt = ("ünïcode 'q' \"d\" $(nope) `x`\n" * 6000) # ~200 KB > CHUNK and > MAX_ARG_STRLEN
h = ex.start(Run("GW-7-1", self.cfg.terminals["lin"]["repo"], prompt, script="wc -c"))
self.assertEqual(self.ch.calls, ["start"])
self.assertEqual(Path(h["run_dir"], "prompt.md").read_text(encoding="utf-8"), prompt)
for _ in range(100):
if ex.poll(h) == "finished":
break
__import__("time").sleep(0.1)
n = len(self.ch.calls)
res = ex.collect(h)
self.assertEqual(self.ch.calls[n:], ["fetch"])
self.assertEqual((res.last_message, res.exit_code), (str(len(prompt.encode("utf-8"))), 0))
with self.assertRaises(TransportError): # pid.txt present: never wiped, never relaunched there
ex.start(Run("GW-7-1", self.cfg.terminals["lin"]["repo"], "again", script="true"))
self.assertEqual(Path(h["run_dir"], "prompt.md").read_text(encoding="utf-8"), prompt)
def test_fetch_missing_files_and_stdin_is_not_the_scripts(self):
d = Path(self.tmp.name) / "runs" / "x"
d.mkdir(parents=True)
(d / "a.txt").write_text("l1\nl2 needle\nl3\n")
ops = [("read", "a.txt"), ("read", "none"), ("grep", "a.txt", "needle"), ("grep", "none", "x"),
("tail", "a.txt", 2), ("tail", "none", 3)]
self.assertEqual(self.t.fetch(str(d), ops), ["l1\nl2 needle\nl3\n", None, "l2 needle", None, "l2 needle\nl3", ""])
rc, out, _ = self.ch.sh("cat; echo done", 10) # the script's own stdin is /dev/null
self.assertEqual((rc, out), (0, "done\n"))
def test_parse_fetch_rejects_garbage(self):
with self.assertRaises(TransportError):
parse_fetch("nothing here", [("read", "x")])
class CallLogTest(unittest.TestCase):
def test_line_format_and_once(self):
with tempfile.TemporaryDirectory() as tmp:
events = []
log = CallLog(Path(tmp) / "sub" / "remote.log", events.append)
log.call("win10", "direct", "start", 812.4, 0)
log.call("win10", "fallback", "busy", 9100, 255, "unreached")
lines = (Path(tmp) / "sub" / "remote.log").read_text().splitlines()
self.assertTrue(lines[0].endswith("term=win10 via=direct op=start ms=812 rc=0"))
self.assertTrue(lines[1].endswith("via=fallback op=busy ms=9100 rc=255 unreached"))
log.event("a", once="k")
log.event("a", once="k")
self.assertEqual(events, ["a"])
class PingCliTest(unittest.TestCase):
def test_ping_prints_timing_and_forces_channel(self):
cfg = Config(direct_data())
seen = {}
def fake_make(cfg_, term, log=None, force=None):
seen.update(term=term, force=force)
return mock.Mock(ping=mock.Mock(return_value="PONG=Linux 6.8 oregon"))
buf = io.StringIO()
with mock.patch("agentd.cli.make_transport", fake_make), redirect_stdout(buf), self.assertRaises(SystemExit) as cm:
cli.cmd_ping(cfg, cli.parser().parse_args(["ping", "lin", "--transport", "warpgate-ssh", "-n", "2"]))
self.assertEqual(cm.exception.code, 0)
self.assertEqual(seen, {"term": "lin", "force": "warpgate-ssh"})
self.assertEqual(buf.getvalue().count("PONG=Linux 6.8 oregon"), 2)
class ReviewFixesTest(unittest.TestCase):
"""Follow-ups of the PAD-12 review: classification limits, socket and known_hosts names, key handling."""
def setUp(self):
direct_ssh._KEYS.clear()
self.tmp = tempfile.TemporaryDirectory()
self.cfg = Config(direct_data(self.tmp.name))
def tearDown(self):
direct_ssh._KEYS.clear()
self.tmp.cleanup()
def test_output_or_remote_stderr_never_counts_as_unreached(self):
self.assertEqual(classify(255, "Connection reset by 100.1.2.3 port 22", "WROTE=ab\n"), "unknown")
self.assertEqual(classify(255, "git: ssh: connect to host github.com port 22: Connection timed out"), "unknown")
self.assertEqual(classify(255, "fatal: Could not read from remote repository"), "unknown")
def test_no_ed25519_offered_is_a_host_key_failure(self):
self.assertEqual(classify(255, "Unable to negotiate with 100.1.2.3 port 22: no matching host key type found. "
"Their offer: ssh-rsa"), "hostkey")
self.assertEqual(classify(255, "Unable to negotiate with 100.1.2.3 port 22: no matching cipher found. "
"Their offer: aes128-cbc"), "unreached")
def test_reset_after_output_is_not_retried_elsewhere(self):
ch = DirectChannel(self.cfg, "lin")
def fake_run(argv, input=None, stdout=None, stderr=None, env=None, timeout=None):
stdout.write(b"WROTE=1\n")
stderr.write(b"Connection reset by 100.127.58.78 port 22\r\n")
return mock.Mock(returncode=255)
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake_run):
with self.assertRaises(TransportError) as cm:
ch.sh("true", 30)
self.assertNotIsInstance(cm.exception, Unreached)
def test_control_path_follows_the_pinned_key(self):
a = DirectChannel(self.cfg, "lin")
d = direct_data(self.tmp.name)
d["terminals"]["lin"]["ssh"]["host_key"] = WIN_KEY
b = DirectChannel(Config(d), "lin")
path = lambda ch: [x for x in ch.argv(FakeAgent, "true") if x.startswith("ControlPath=")][0] # noqa: E731
self.assertNotEqual(path(a), path(b))
self.assertEqual(path(a), "ControlPath=%s/%s-%%C" % (self.cfg.ssh_dir, a.pin))
self.assertEqual(len(a.pin), 8)
long_dir = direct_data("/" + "x" * 40)
self.assertTrue(any("too long" in e for e in config.validate(long_dir))) # 41 + 67 > 104
def test_known_hosts_temp_files_are_per_terminal_and_process(self):
d = direct_ssh.control_dir(self.cfg)
tmps = []
real = os.replace
with mock.patch("agentd.transports.direct_ssh.os.replace",
side_effect=lambda a, b: (tmps.append(str(a)), real(a, b))):
DirectChannel(self.cfg, "win10").known_hosts(d)
DirectChannel(self.cfg, "lin").known_hosts(d)
self.assertEqual(len(set(tmps)), 2)
self.assertTrue(all(str(os.getpid()) in t for t in tmps))
self.assertEqual((d / "known_hosts.win10").read_text(), "agentd-win10 %s\n" % WIN_KEY)
self.assertEqual((d / "known_hosts.lin").read_text(), "agentd-lin %s\n" % LIN_KEY)
self.assertEqual(sorted(f.name for f in d.iterdir()), ["known_hosts.lin", "known_hosts.win10"])
def test_key_leaves_the_environment_at_cli_start(self):
buf = io.StringIO()
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET}), redirect_stdout(buf):
cli.main(["--config", str(ROOT / "config" / "agentd.yaml"), "check-config"])
self.assertNotIn("AGENTD_SSH_KEY", os.environ)
self.assertIn("direct-ssh key AGENTD_SSH_KEY: set", buf.getvalue())
self.assertNotIn("NOT-A-REAL-KEY", buf.getvalue())
self.assertTrue(direct_ssh.key_present("AGENTD_SSH_KEY"))
with mock.patch("agentd.transports.direct_ssh.shutil.which", return_value=None):
a = KeyAgent("AGENTD_SSH_KEY").start() # the agent takes the key over from the CLI
self.assertIn("not installed", a.error)
self.assertFalse(direct_ssh.key_present("AGENTD_SSH_KEY"))
def test_local_busy_check_gets_the_scrubbed_env(self):
cfg = Config(config_data())
from agentd.transports.local import LocalTransport
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET, "TOK_OR": "plane-token"}), \
mock.patch("agentd.transports.local.subprocess.run", return_value=mock.Mock(stdout="2\n")) as run:
self.assertEqual(LocalTransport(cfg, "oregon").process_count("codex"), 2)
env = run.call_args.kwargs["env"]
self.assertNotIn("AGENTD_SSH_KEY", env)
self.assertNotIn("TOK_OR", env)
def test_task_names_are_quoted_in_kill_and_forget(self):
ch = mock.Mock(stdin=True)
ch.ps.return_value = "KILLED=ok"
t = WindowsWarpgateTransport(self.cfg, "win10", channel=ch)
t.kill("agentd-a'b", "C:\\runs\\x")
t.forget("agentd-a'b")
for c in ch.ps.call_args_list:
self.assertIn("'agentd-a''b'", c.args[0])
self.assertNotIn("'agentd-a'b'", c.args[0])
class FakeTerm:
"""Remote transport stand-in for core-level routing tests: pure helpers from the real Windows transport,
start_run / probe on a dict of run directories shared by the direct and fallback channels. `fail` = exceptions
for the next calls: Unreached and HostKeyError strike before the command ran, any other after it (the run
exists). A host-key failure sticks, like DirectChannel's."""
def __init__(self, cfg, dirs):
self.real = WindowsWarpgateTransport(cfg, "win10", channel=mock.Mock(stdin=True))
self.dirs, self.fail, self.calls, self.hostkey = dirs, [], [], None
def __getattr__(self, name):
return getattr(self.real, name)
def _next(self, name):
self.calls.append(name)
if self.hostkey:
raise self.hostkey
exc = self.fail.pop(0) if self.fail else None
if isinstance(exc, HostKeyError):
self.hostkey = exc
return exc
def start_run(self, task, run_dir, files, script_name):
exc = self._next("start_run")
if isinstance(exc, (Unreached, HostKeyError)):
raise exc
if run_dir in self.dirs:
raise TransportError("HASPID")
self.dirs[run_dir] = ["launch.pid"]
if exc:
raise exc
def probe(self, run_dir, names):
exc = self._next("probe")
if exc:
raise exc
return list(self.dirs.get(run_dir, ()))
class CoreRoutingTest(unittest.TestCase):
"""Agentd.start -> codex executor -> RoutedTransport: the double-launch guarantee end to end."""
def setUp(self):
from tests.fakes import Harness
self.h = Harness(direct_data())
self.dirs, self.events = {}, []
self.direct, self.fallback = FakeTerm(self.h.cfg, self.dirs), FakeTerm(self.h.cfg, self.dirs)
log = CallLog(event=self.events.append)
self.h.d.executor_factory = lambda term, name: make_executor( # a fresh routed transport per tick
self.h.cfg, term, name, RoutedTransport(term, self.direct, self.fallback, log))
def tearDown(self):
self.h.close()
def test_unreached_launch_moves_to_the_fallback_once(self):
iid = self.h.plane.add_issue(1)
self.direct.fail = [Unreached("ssh: connect to host 100.123.209.19 port 22: Connection timed out")]
self.h.tick()
rec = self.h.rec(iid)
self.assertNotIn("launching", rec)
self.assertEqual(rec["inflight"]["run_id"], "GW-1-1")
self.assertEqual((self.direct.calls, self.fallback.calls), (["start_run"], ["start_run"]))
self.assertEqual(len(self.dirs), 1)
self.assertEqual(self.h.plane.state_of(iid), "In Progress")
self.assertTrue(any("warpgate-ssh fallback" in e for e in self.events))
def test_failure_after_the_command_ran_is_probed_and_adopted_not_relaunched(self):
iid = self.h.plane.add_issue(1)
self.direct.fail = [TransportError("ssh 100.123.209.19 timeout after 150s")]
self.h.tick()
rec = self.h.rec(iid)
self.assertIn("launching", rec)
self.assertNotIn("inflight", rec)
self.assertEqual(self.fallback.calls, [])
self.h.tick() # probe finds launch.pid -> adopt
rec = self.h.rec(iid)
self.assertNotIn("launching", rec)
self.assertEqual(rec["inflight"]["handle"]["run_dir"], list(self.dirs)[0])
self.assertEqual(self.direct.calls, ["start_run", "probe"])
self.assertEqual((self.fallback.calls, len(self.dirs)), ([], 1))
def test_host_key_failure_keeps_the_marker_and_never_falls_back(self):
iid = self.h.plane.add_issue(1)
self.direct.fail = [HostKeyError("host key verification FAILED")]
self.h.tick()
self.assertIn("launching", self.h.rec(iid))
self.h.tick() # the probe fails the same way: marker kept, slot held
rec = self.h.rec(iid)
self.assertIn("launching", rec)
self.assertNotIn("inflight", rec)
self.assertEqual(self.direct.calls, ["start_run", "probe"])
self.assertEqual((self.fallback.calls, self.dirs), ([], {}))
self.assertEqual(self.h.plane.state_of(iid), "Todo")
FAKE_CLIENT = """import json, os, sys
argv = sys.argv[1:]
with open(os.environ["FAKE_CALLS"], "a") as fh:
fh.write(json.dumps(argv) + "\\n")
if os.environ.get("FAKE_MISSING") and "AGENTD_SSH_KEY=AGENTD_SSH_KEY" in argv:
print("access-hub: credential_unavailable", file=sys.stderr)
sys.exit(1)
print("ticked")
print("agentd stderr line", file=sys.stderr)
sys.exit(int(os.environ.get("FAKE_RC", "0")))
"""
@unittest.skipUnless(all(shutil.which(b) for b in ("bash", "flock", "timeout")), "needs bash, flock, timeout")
class CronScriptTest(unittest.TestCase):
"""cron/agentd-tick-lxc.sh with a fake agent-access.py: a missing AGENTD_SSH_KEY must not stop the tick."""
def setUp(self):
import re
self.tmp = tempfile.TemporaryDirectory()
d = Path(self.tmp.name)
(d / "client.py").write_text(FAKE_CLIENT)
text = (ROOT / "cron" / "agentd-tick-lxc.sh").read_text()
for var, value in (("LOCK", d / "cron.lock"), ("LOG", d / "cron.log"), ("CLIENT", "(python3 -I %s)" % (d / "client.py"))):
text, n = re.subn(r"(?m)^%s=.*$" % var, "%s=%s" % (var, value), text)
self.assertEqual(n, 1, var)
self.script, self.calls, self.log = d / "tick.sh", d / "calls", d / "cron.log"
self.script.write_text(text)
def tearDown(self):
self.tmp.cleanup()
def run_tick(self, **env):
e = dict(os.environ, FAKE_CALLS=str(self.calls), **env)
rc = subprocess.run(["bash", str(self.script)], env=e, timeout=60).returncode
calls = [json.loads(l) for l in self.calls.read_text().splitlines()] if self.calls.exists() else []
return rc, calls, self.log.read_text() if self.log.exists() else ""
def test_normal_tick_passes_the_key(self):
rc, calls, log = self.run_tick()
self.assertEqual((rc, len(calls)), (0, 1))
self.assertIn("--secret-env", calls[0])
self.assertIn("AGENTD_SSH_KEY=AGENTD_SSH_KEY", calls[0])
self.assertEqual(calls[0][-3:], ["/opt/plane-agentd/config/agentd.yaml", "tick", "--once"])
self.assertIn("ticked", log)
self.assertIn("agentd stderr line", log)
def test_missing_vault_key_repeats_the_tick_without_it(self):
rc, calls, log = self.run_tick(FAKE_MISSING="1")
self.assertEqual((rc, len(calls)), (0, 2))
self.assertNotIn("AGENTD_SSH_KEY=AGENTD_SSH_KEY", calls[1])
self.assertIn("PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN", calls[1])
self.assertIn("AGENTD_SSH_KEY not available from Vault", log)
self.assertIn("ticked", log)
def test_a_failing_tick_is_not_repeated(self):
rc, calls, _ = self.run_tick(FAKE_RC="1")
self.assertEqual((rc, len(calls)), (1, 1))
def test_lock_held_exits_75(self):
import fcntl
with open(Path(self.tmp.name) / "cron.lock", "a") as fh:
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
rc, calls, _ = self.run_tick()
self.assertEqual((rc, calls), (75, []))
if __name__ == "__main__":
unittest.main()
+5 -3
View File
@@ -80,9 +80,11 @@ class PosixRemoteTest(unittest.TestCase):
def test_live_config_runs_oregon_remotely(self): def test_live_config_runs_oregon_remotely(self):
cfg = config.load(ROOT / "config" / "agentd.yaml") cfg = config.load(ROOT / "config" / "agentd.yaml")
t = make_transport(cfg, "oregon") t = make_transport(cfg, "oregon") # PAD-12: direct-ssh over Tailscale, Warpgate relay as fallback
self.assertIsInstance(t, PosixWarpgateTransport) self.assertIsInstance(t.primary, PosixWarpgateTransport)
self.assertEqual((t.target, t.os), ("oregon", "linux")) self.assertIsInstance(t.fallback, PosixWarpgateTransport)
self.assertEqual((t.primary.channel.stdin, t.fallback.channel.stdin), (True, False))
self.assertEqual((t.fallback.target, t.os), ("oregon", "linux"))
self.assertEqual(cfg.executors["codex"]["tier_defaults"], {"gpt-6.1-sol": "fast"}) self.assertEqual(cfg.executors["codex"]["tier_defaults"], {"gpt-6.1-sol": "fast"})
def test_relay_argv_and_scrubbed_env(self): def test_relay_argv_and_scrubbed_env(self):