Compare commits
9 Commits
6dc7e72ac8
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 94a09e3bf0 | |||
| 5192e04088 | |||
| 83bc0f243c | |||
| ccbc68369c | |||
| 709d65ef83 | |||
| 57887a90f2 | |||
| 1b4d965baf | |||
| 4ba6d0ca5f | |||
| f6c4221f0e |
@@ -2,11 +2,12 @@
|
|||||||
|
|
||||||
Plane 看板驱动的通用 agent 调度服务:在 Plane 建工作项并指派给某个终端,agentd 在该终端起对应的 CLI agent(Codex / Claude Code / 脚本)执行,把结果贴回工作项并流转状态。调度是确定性代码;模型只出现在执行器和总指挥里。
|
Plane 看板驱动的通用 agent 调度服务:在 Plane 建工作项并指派给某个终端,agentd 在该终端起对应的 CLI agent(Codex / Claude Code / 脚本)执行,把结果贴回工作项并流转状态。调度是确定性代码;模型只出现在执行器和总指挥里。
|
||||||
|
|
||||||
|
- 使用说明(在 Plane 上怎么派活、回答、取消,标签和默认值):`docs/usage.md`
|
||||||
- 设计:`docs/design.md`(架构、契约、失败策略、配置、阶段)
|
- 设计:`docs/design.md`(架构、契约、失败策略、配置、阶段)
|
||||||
- 首个租户:GWENT 项目(看板 https://plane.kappiee.com/kappiee/projects/79cb5a60-aa4f-4138-9864-1dd70426e665/issues/ )
|
- 首个租户:GWENT 项目(看板 https://plane.kappiee.com/kappiee/projects/79cb5a60-aa4f-4138-9864-1dd70426e665/issues/ )
|
||||||
- 凭据全部经 Agent Vault 注入内存,仓库和配置里只有键名。
|
- 凭据全部经 Agent Vault 注入内存,仓库和配置里只有键名。
|
||||||
|
|
||||||
## 使用
|
## 使用
|
||||||
- 契约与扩展:`docs/executor-contract.md`;安装、cron、日志、故障:`docs/runbook.md`;LXC 部署(调度器 2026-10-21 起跑在 `agentd-01`):`docs/deploy-lxc.md`
|
- 契约与扩展:`docs/executor-contract.md`;安装、cron、日志、故障:`docs/runbook.md`;LXC 部署(调度器 2026-10-21 起跑在 `agentd-01`):`docs/deploy-lxc.md`
|
||||||
- 入口:`cli/agentd tick|new|show|comment|state|assign|smoke|check-config`(在 `agent-access.py exec --secret-env …` 下运行)
|
- 入口:`cli/agentd tick|new|show|comment|state|assign|smoke|ping|check-config`(在 `agent-access.py exec --secret-env …` 下运行;`ping` 验证到终端的 direct-ssh / warpgate-ssh 路径)
|
||||||
- 测试:`python3 -m unittest`(离线,不访问 Plane/win10)
|
- 测试:`python3 -m unittest`(离线,不访问 Plane/win10)
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from . import config as config_mod
|
|||||||
from .core import Agentd
|
from .core import Agentd
|
||||||
from .plane import Plane, brief_to_html, html_to_text, text_to_html
|
from .plane import Plane, brief_to_html, html_to_text, text_to_html
|
||||||
from .store import Busy
|
from .store import Busy
|
||||||
|
from .transports import CallLog, TransportError, direct_ssh, make_transport
|
||||||
|
|
||||||
|
|
||||||
def identity(cfg, wanted=None):
|
def identity(cfg, wanted=None):
|
||||||
@@ -159,6 +160,31 @@ def cmd_smoke(cfg, a):
|
|||||||
sys.exit(0 if ok else 1)
|
sys.exit(0 if ok else 1)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_ping(cfg, a):
|
||||||
|
"""Trivial command on a terminal, timed per call (verifies direct-ssh / warpgate-ssh after a deployment). The
|
||||||
|
direct-ssh key comes from the environment like the Plane tokens (agent-access.py exec --secret-env)."""
|
||||||
|
if a.terminal not in cfg.terminals:
|
||||||
|
sys.exit("unknown terminal %s" % a.terminal)
|
||||||
|
t = cfg.terminals[a.terminal]
|
||||||
|
if (a.transport or t["transport"]) == "local":
|
||||||
|
sys.exit("%s is a local terminal: nothing to ping" % a.terminal)
|
||||||
|
if a.transport == "direct-ssh" and "ssh" not in t:
|
||||||
|
sys.exit("terminals.%s has no ssh section" % a.terminal)
|
||||||
|
log = CallLog(event=lambda m: print("event: " + m, flush=True), echo=True)
|
||||||
|
tr = make_transport(cfg, a.terminal, log=log, force=a.transport)
|
||||||
|
ok = True
|
||||||
|
for i in range(a.count):
|
||||||
|
t0 = time.monotonic()
|
||||||
|
try:
|
||||||
|
pong = tr.ping()
|
||||||
|
except TransportError as exc:
|
||||||
|
ok = False
|
||||||
|
print("ping %d: FAILED after %d ms: %s" % (i + 1, (time.monotonic() - t0) * 1000, exc), flush=True)
|
||||||
|
continue
|
||||||
|
print("ping %d: %d ms %s" % (i + 1, (time.monotonic() - t0) * 1000, pong), flush=True)
|
||||||
|
sys.exit(0 if ok else 1)
|
||||||
|
|
||||||
|
|
||||||
def cmd_check_config(cfg, a):
|
def cmd_check_config(cfg, a):
|
||||||
print("config %s is valid" % cfg.path)
|
print("config %s is valid" % cfg.path)
|
||||||
print("projects: %s" % ", ".join(cfg.projects))
|
print("projects: %s" % ", ".join(cfg.projects))
|
||||||
@@ -166,6 +192,13 @@ def cmd_check_config(cfg, a):
|
|||||||
env = cfg.token_env(name)
|
env = cfg.token_env(name)
|
||||||
print("terminal %-10s %-13s %-8s executors=%s token %s: %s" % (
|
print("terminal %-10s %-13s %-8s executors=%s token %s: %s" % (
|
||||||
name, t["transport"], t["os"], ",".join(t["executors"]), env, "set" if os.environ.get(env) else "MISSING"))
|
name, t["transport"], t["os"], ",".join(t["executors"]), env, "set" if os.environ.get(env) else "MISSING"))
|
||||||
|
if t["transport"] == "direct-ssh":
|
||||||
|
s = t["ssh"]
|
||||||
|
print(" direct-ssh %s@%s:%s, fallback %s" % (s["user"], s["host"], s.get("port", 22), t.get("fallback", "none")))
|
||||||
|
if cfg.uses_direct_ssh():
|
||||||
|
print("direct-ssh key %s: %s; ssh_dir %s" % (
|
||||||
|
cfg.ssh_key_env, "set" if direct_ssh.key_present(cfg.ssh_key_env) else "MISSING (direct terminals use their fallback)",
|
||||||
|
cfg.ssh_dir))
|
||||||
print("cron/exec secret args: %s" % " ".join(cfg.secret_env_args()))
|
print("cron/exec secret args: %s" % " ".join(cfg.secret_env_args()))
|
||||||
if cfg.webhook:
|
if cfg.webhook:
|
||||||
k = cfg.webhook["secret_key"]
|
k = cfg.webhook["secret_key"]
|
||||||
@@ -221,6 +254,12 @@ def parser():
|
|||||||
s.add_argument("--port", type=int, help="port (default: webhook.port)")
|
s.add_argument("--port", type=int, help="port (default: webhook.port)")
|
||||||
s.set_defaults(fn=cmd_webhook_serve)
|
s.set_defaults(fn=cmd_webhook_serve)
|
||||||
sub.add_parser("check-config", help="validate the config and list token env vars").set_defaults(fn=cmd_check_config)
|
sub.add_parser("check-config", help="validate the config and list token env vars").set_defaults(fn=cmd_check_config)
|
||||||
|
s = sub.add_parser("ping", help="time a trivial command on a terminal (direct-ssh needs the key env var, see check-config)")
|
||||||
|
s.add_argument("terminal")
|
||||||
|
s.add_argument("--transport", choices=["direct-ssh", "warpgate-ssh"],
|
||||||
|
help="force one channel, no fallback (default: the terminal's transport with its fallback)")
|
||||||
|
s.add_argument("-n", "--count", type=int, default=3, help="calls (default 3: the 2nd+ reuse the SSH connection)")
|
||||||
|
s.set_defaults(fn=cmd_ping)
|
||||||
return ap
|
return ap
|
||||||
|
|
||||||
|
|
||||||
@@ -230,4 +269,5 @@ def main(argv=None):
|
|||||||
cfg = config_mod.load(a.config)
|
cfg = config_mod.load(a.config)
|
||||||
except config_mod.ConfigError as exc:
|
except config_mod.ConfigError as exc:
|
||||||
sys.exit(str(exc))
|
sys.exit(str(exc))
|
||||||
|
direct_ssh.take_key(cfg.ssh_key_env) # out of os.environ before any child process starts
|
||||||
a.fn(cfg, a)
|
a.fn(cfg, a)
|
||||||
|
|||||||
+93
-17
@@ -2,8 +2,10 @@
|
|||||||
|
|
||||||
The config only ever holds Vault *key names* (`token_key`). The token itself is read from the environment
|
The config only ever holds Vault *key names* (`token_key`). The token itself is read from the environment
|
||||||
variable of the same name, which `agent-access.py exec --secret-env KEY=KEY` injects into the process."""
|
variable of the same name, which `agent-access.py exec --secret-env KEY=KEY` injects into the process."""
|
||||||
|
import base64
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import struct
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
@@ -11,6 +13,7 @@ import yaml
|
|||||||
from . import ROOT
|
from . import ROOT
|
||||||
|
|
||||||
SCHEMA_PATH = ROOT / "config" / "schema.json"
|
SCHEMA_PATH = ROOT / "config" / "schema.json"
|
||||||
|
DEFAULT_SSH_KEY = "AGENTD_SSH_KEY" # env var (== Vault key name) carrying the direct-ssh private key
|
||||||
DEFAULT_PATHS = (ROOT / "config" / "agentd.local.yaml", ROOT / "config" / "agentd.yaml")
|
DEFAULT_PATHS = (ROOT / "config" / "agentd.local.yaml", ROOT / "config" / "agentd.yaml")
|
||||||
|
|
||||||
|
|
||||||
@@ -83,6 +86,48 @@ def schema_errors(data, schema):
|
|||||||
for e in validator.iter_errors(data)]
|
for e in validator.iter_errors(data)]
|
||||||
|
|
||||||
|
|
||||||
|
def host_key_error(value):
|
||||||
|
"""None if `value` is a usable pinned host key ('ssh-ed25519 <base64>' whose blob names the same type)."""
|
||||||
|
parts = str(value).split()
|
||||||
|
if len(parts) < 2 or parts[0] != "ssh-ed25519":
|
||||||
|
return "must be 'ssh-ed25519 <base64>' (the terminal's /etc/ssh/ssh_host_ed25519_key.pub, verified out of band)"
|
||||||
|
try:
|
||||||
|
blob = base64.b64decode(parts[1], validate=True)
|
||||||
|
n = struct.unpack(">I", blob[:4])[0]
|
||||||
|
ok = blob[4:4 + n] == b"ssh-ed25519" and len(blob) == 4 + n + 4 + 32
|
||||||
|
except (ValueError, struct.error):
|
||||||
|
ok = False
|
||||||
|
return None if ok else "the base64 part is not an ssh-ed25519 public key (copy/paste damage?)"
|
||||||
|
|
||||||
|
|
||||||
|
def ssh_dir_of(data):
|
||||||
|
d = (data.get("agentd") or {}).get("ssh_dir")
|
||||||
|
if d:
|
||||||
|
p = Path(os.path.expanduser(d))
|
||||||
|
return p if p.is_absolute() else ROOT / p
|
||||||
|
s = Path(os.path.expanduser((data.get("agentd") or {}).get("state_dir", "state")))
|
||||||
|
return (s if s.is_absolute() else ROOT / s) / "ssh"
|
||||||
|
|
||||||
|
|
||||||
|
def remote_errors(data, tname, t, kind):
|
||||||
|
"""Rules every remote transport (warpgate-ssh, direct-ssh, and the fallback channel) shares."""
|
||||||
|
errs = []
|
||||||
|
if t.get("os") == "windows":
|
||||||
|
if t.get("launch") != "schtasks":
|
||||||
|
errs.append("terminals.%s: %s on windows needs launch=schtasks" % (tname, kind))
|
||||||
|
if kind == "warpgate-ssh" and t.get("target", tname) != "win10":
|
||||||
|
errs.append("terminals.%s: the Windows helper scripts only reach target 'win10' for now" % tname)
|
||||||
|
else:
|
||||||
|
if t.get("launch") != "nohup":
|
||||||
|
errs.append("terminals.%s: %s on %s needs launch=nohup" % (tname, kind, t.get("os")))
|
||||||
|
for key in ("repo", "run_root"):
|
||||||
|
if key in t and not str(t[key]).startswith("/"):
|
||||||
|
errs.append("terminals.%s.%s: must be an absolute path on the remote (no ~)" % (tname, key))
|
||||||
|
if kind == "warpgate-ssh" and not data.get("vault", {}).get("config"):
|
||||||
|
errs.append("terminals.%s: remote POSIX terminals need vault.config (relay identity)" % tname)
|
||||||
|
return errs
|
||||||
|
|
||||||
|
|
||||||
def semantic_errors(data):
|
def semantic_errors(data):
|
||||||
"""Cross-references the schema cannot express."""
|
"""Cross-references the schema cannot express."""
|
||||||
errs = []
|
errs = []
|
||||||
@@ -91,21 +136,36 @@ def semantic_errors(data):
|
|||||||
for ex in t.get("executors", []):
|
for ex in t.get("executors", []):
|
||||||
if ex not in execs:
|
if ex not in execs:
|
||||||
errs.append("terminals.%s: unknown executor '%s'" % (tname, ex))
|
errs.append("terminals.%s: unknown executor '%s'" % (tname, ex))
|
||||||
if t.get("transport") == "warpgate-ssh" and t.get("os") == "windows":
|
kind = t.get("transport")
|
||||||
if t.get("launch") != "schtasks":
|
if kind == "warpgate-ssh":
|
||||||
errs.append("terminals.%s: warpgate-ssh on windows needs launch=schtasks" % tname)
|
errs += remote_errors(data, tname, t, kind)
|
||||||
if t.get("target", tname) != "win10":
|
elif kind == "direct-ssh":
|
||||||
errs.append("terminals.%s: the Windows helper scripts only reach target 'win10' for now" % tname)
|
errs += remote_errors(data, tname, t, kind)
|
||||||
elif t.get("transport") == "warpgate-ssh":
|
s = t.get("ssh")
|
||||||
if t.get("launch") != "nohup":
|
if not s:
|
||||||
errs.append("terminals.%s: warpgate-ssh on %s needs launch=nohup" % (tname, t.get("os")))
|
errs.append("terminals.%s: direct-ssh needs ssh: {host, user, port, host_key}" % tname)
|
||||||
for key in ("repo", "run_root"):
|
else:
|
||||||
if key in t and not str(t[key]).startswith("/"):
|
if not s.get("host") or str(s["host"]).startswith("-") or not s.get("user"):
|
||||||
errs.append("terminals.%s.%s: must be an absolute path on the remote (no ~)" % (tname, key))
|
errs.append("terminals.%s.ssh: host and user must be non-empty (host not starting with '-')" % tname)
|
||||||
if not data.get("vault", {}).get("config"):
|
if s.get("port", 22) > 65535:
|
||||||
errs.append("terminals.%s: remote POSIX terminals need vault.config (relay identity)" % tname)
|
errs.append("terminals.%s.ssh.port: %r > 65535" % (tname, s["port"]))
|
||||||
if t.get("transport") == "local" and t.get("os") != "linux":
|
why = host_key_error(s.get("host_key", ""))
|
||||||
|
if why:
|
||||||
|
errs.append("terminals.%s.ssh.host_key: %s" % (tname, why))
|
||||||
|
if t.get("fallback") == "warpgate-ssh":
|
||||||
|
if "target" not in t:
|
||||||
|
errs.append("terminals.%s: fallback warpgate-ssh needs the Warpgate 'target'" % tname)
|
||||||
|
elif t.get("os") == "windows" and t["target"] != "win10":
|
||||||
|
errs.append("terminals.%s: the Windows helper scripts (fallback) only reach target 'win10' for now" % tname)
|
||||||
|
if t.get("os") != "windows" and not data.get("vault", {}).get("config"):
|
||||||
|
errs.append("terminals.%s: fallback warpgate-ssh needs vault.config (relay identity)" % tname)
|
||||||
|
elif t.get("fallback"):
|
||||||
|
errs.append("terminals.%s: 'fallback' only applies to transport direct-ssh" % tname)
|
||||||
|
if kind == "local" and t.get("os") != "linux":
|
||||||
errs.append("terminals.%s: local transport supports only os=linux" % tname)
|
errs.append("terminals.%s: local transport supports only os=linux" % tname)
|
||||||
|
ctl = ssh_dir_of(data) # "/<pin8>-<40-char %C>", plus ".<16 chars>" while ssh sets the socket up
|
||||||
|
if any(t.get("transport") == "direct-ssh" for t in terms.values()) and len(str(ctl)) + 1 + 9 + 40 + 17 > 104:
|
||||||
|
errs.append("agentd.ssh_dir: %s is too long for the ControlMaster socket path (UNIX limit 104)" % ctl)
|
||||||
for ename, e in execs.items():
|
for ename, e in execs.items():
|
||||||
adapter = e.get("adapter", ename)
|
adapter = e.get("adapter", ename)
|
||||||
if adapter not in ("codex", "claude-code", "shell"):
|
if adapter not in ("codex", "claude-code", "shell"):
|
||||||
@@ -156,6 +216,19 @@ class Config:
|
|||||||
p = Path(os.path.expanduser(self.agentd.get("state_dir", "state")))
|
p = Path(os.path.expanduser(self.agentd.get("state_dir", "state")))
|
||||||
return p if p.is_absolute() else ROOT / p
|
return p if p.is_absolute() else ROOT / p
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ssh_dir(self):
|
||||||
|
"""direct-ssh ControlMaster sockets + generated known_hosts (default <state_dir>/ssh)."""
|
||||||
|
return ssh_dir_of(self.data)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ssh_key_env(self):
|
||||||
|
"""Env var (== Vault key name) holding the direct-ssh private key."""
|
||||||
|
return self.agentd.get("ssh_key") or DEFAULT_SSH_KEY
|
||||||
|
|
||||||
|
def uses_direct_ssh(self):
|
||||||
|
return any(t["transport"] == "direct-ssh" for t in self.terminals.values())
|
||||||
|
|
||||||
def helpers_dir(self, terminal):
|
def helpers_dir(self, terminal):
|
||||||
d = self.terminals[terminal].get("helpers_dir") or self.agentd.get("helpers_dir") or "tools/win10"
|
d = self.terminals[terminal].get("helpers_dir") or self.agentd.get("helpers_dir") or "tools/win10"
|
||||||
p = Path(os.path.expanduser(d))
|
p = Path(os.path.expanduser(d))
|
||||||
@@ -185,13 +258,16 @@ class Config:
|
|||||||
return keys
|
return keys
|
||||||
|
|
||||||
def scrubbed_env(self):
|
def scrubbed_env(self):
|
||||||
"""os.environ without the secrets above, for every child process agentd starts."""
|
"""os.environ without the secrets above, for every child process agentd starts. The direct-ssh key variable
|
||||||
drop = self.secret_keys()
|
is always dropped, even with no direct-ssh terminal (a rollback may leave the cron script injecting it)."""
|
||||||
|
drop = self.secret_keys() | {self.ssh_key_env}
|
||||||
return {k: v for k, v in os.environ.items() if k not in drop}
|
return {k: v for k, v in os.environ.items() if k not in drop}
|
||||||
|
|
||||||
def secret_env_args(self):
|
def secret_env_args(self):
|
||||||
"""`--secret-env` arguments for agent-access.py exec, one per distinct token key."""
|
"""`--secret-env` arguments for agent-access.py exec: one per distinct token key, plus the direct-ssh key."""
|
||||||
keys = sorted({t["token_key"] for t in self.terminals.values()})
|
keys = sorted({t["token_key"] for t in self.terminals.values()})
|
||||||
|
if self.uses_direct_ssh():
|
||||||
|
keys.append(self.ssh_key_env)
|
||||||
return ["--secret-env %s=%s" % (k, k) for k in keys]
|
return ["--secret-env %s=%s" % (k, k) for k in keys]
|
||||||
|
|
||||||
def project(self, name):
|
def project(self, name):
|
||||||
|
|||||||
+4
-2
@@ -21,7 +21,7 @@ from .executors import ExecResult, Run, make_executor
|
|||||||
from .executors.base import fill
|
from .executors.base import fill
|
||||||
from .plane import Plane, html_to_text, text_to_html
|
from .plane import Plane, html_to_text, text_to_html
|
||||||
from .store import Store
|
from .store import Store
|
||||||
from .transports import TransportError, make_transport
|
from .transports import CallLog, TransportError, make_transport
|
||||||
|
|
||||||
TAGS = ("review", "question", "blocked", "progress")
|
TAGS = ("review", "question", "blocked", "progress")
|
||||||
WORKER = "[worker]"
|
WORKER = "[worker]"
|
||||||
@@ -152,9 +152,11 @@ class Agentd:
|
|||||||
self.store = store or Store(cfg.state_dir, log_dir=cfg.log_dir)
|
self.store = store or Store(cfg.state_dir, log_dir=cfg.log_dir)
|
||||||
self.now = now
|
self.now = now
|
||||||
self.log = self.store.log
|
self.log = self.store.log
|
||||||
|
# one line per remote call (terminal, channel, operation, ms, rc) -> remote.log; fallback/host-key -> agentd.log
|
||||||
|
self.calllog = CallLog(self.store.log_dir / "remote.log", self.log)
|
||||||
self.plane_factory = plane_factory or (lambda term: Plane(
|
self.plane_factory = plane_factory or (lambda term: Plane(
|
||||||
cfg.plane["base_url"], cfg.plane["workspace"], cfg.token(term), cfg.plane.get("timeout", 60)))
|
cfg.plane["base_url"], cfg.plane["workspace"], cfg.token(term), cfg.plane.get("timeout", 60)))
|
||||||
self.transport_factory = transport_factory or (lambda term: make_transport(cfg, term))
|
self.transport_factory = transport_factory or (lambda term: make_transport(cfg, term, log=self.calllog))
|
||||||
self.executor_factory = executor_factory or (lambda term, name: make_executor(cfg, term, name, self.transport(term)))
|
self.executor_factory = executor_factory or (lambda term, name: make_executor(cfg, term, name, self.transport(term)))
|
||||||
self._reset_caches()
|
self._reset_caches()
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import re
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from ..transports import fetch_each
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Run:
|
class Run:
|
||||||
@@ -55,6 +57,11 @@ class Executor:
|
|||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
def parse_exit(text):
|
||||||
|
m = re.search(r"exit=(-?\d+)", text or "")
|
||||||
|
return int(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
def fill(template, params):
|
def fill(template, params):
|
||||||
"""Replace {name} placeholders that exist in params; leave anything else untouched."""
|
"""Replace {name} placeholders that exist in params; leave anything else untouched."""
|
||||||
return re.sub(r"\{(\w+)\}", lambda m: str(params[m.group(1)]) if m.group(1) in params else m.group(0), template)
|
return re.sub(r"\{(\w+)\}", lambda m: str(params[m.group(1)]) if m.group(1) in params else m.group(0), template)
|
||||||
@@ -101,7 +108,7 @@ class RunnerExecutor(Executor):
|
|||||||
run_dir, task = h["run_dir"], h["task"]
|
run_dir, task = h["run_dir"], h["task"]
|
||||||
script_name, script = t.runner(run_dir, run.repo, self.command(run, run_dir), self.stdout_name)
|
script_name, script = t.runner(run_dir, run.repo, self.command(run, run_dir), self.stdout_name)
|
||||||
files = {"prompt.md": run.prompt, script_name: script}
|
files = {"prompt.md": run.prompt, script_name: script}
|
||||||
if hasattr(t, "start_run"): # remote POSIX: stage + launch in one relay round trip
|
if hasattr(t, "start_run"): # remote: stage + launch in one call (direct-ssh; POSIX also on the relay)
|
||||||
t.start_run(task, run_dir, files, script_name)
|
t.start_run(task, run_dir, files, script_name)
|
||||||
else:
|
else:
|
||||||
t.mkdir(run_dir, clean=True)
|
t.mkdir(run_dir, clean=True)
|
||||||
@@ -121,9 +128,14 @@ class RunnerExecutor(Executor):
|
|||||||
return "finished" if text and "exit=" in text else "running"
|
return "finished" if text and "exit=" in text else "running"
|
||||||
|
|
||||||
def exit_code(self, handle):
|
def exit_code(self, handle):
|
||||||
text = self.transport.read_text(self.transport.join(handle["run_dir"], "exit.txt")) or ""
|
return parse_exit(self.transport.read_text(self.transport.join(handle["run_dir"], "exit.txt")))
|
||||||
m = re.search(r"exit=(-?\d+)", text)
|
|
||||||
return int(m.group(1)) if m else None
|
def fetch(self, handle, ops):
|
||||||
|
"""Reads of the run directory for `collect`: one call on remote transports (see transports.fetch)."""
|
||||||
|
t = self.transport
|
||||||
|
if hasattr(t, "fetch"):
|
||||||
|
return t.fetch(handle["run_dir"], ops)
|
||||||
|
return fetch_each(t, handle["run_dir"], ops)
|
||||||
|
|
||||||
def kill(self, handle):
|
def kill(self, handle):
|
||||||
self.transport.kill(handle["task"], handle["run_dir"])
|
self.transport.kill(handle["task"], handle["run_dir"])
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
resume with `--resume <session_id>`. The JSON result carries `result` (last message) and `session_id`."""
|
resume with `--resume <session_id>`. The JSON result carries `result` (last message) and `session_id`."""
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from .base import ExecResult, RunnerExecutor
|
from .base import ExecResult, RunnerExecutor, parse_exit
|
||||||
|
|
||||||
|
|
||||||
def parse_output(text):
|
def parse_output(text):
|
||||||
@@ -26,13 +26,14 @@ def parse_output(text):
|
|||||||
class ClaudeCodeExecutor(RunnerExecutor):
|
class ClaudeCodeExecutor(RunnerExecutor):
|
||||||
stdout_name = "out.json"
|
stdout_name = "out.json"
|
||||||
|
|
||||||
|
COLLECT = [("read", "exit.txt"), ("read", "out.json"), ("tail", "stderr.txt", 5)]
|
||||||
|
|
||||||
def collect(self, handle):
|
def collect(self, handle):
|
||||||
t, d = self.transport, handle["run_dir"]
|
exit_text, out, tail = self.fetch(handle, self.COLLECT) # one remote call
|
||||||
obj = parse_output(t.read_text(t.join(d, "out.json")))
|
obj = parse_output(out)
|
||||||
err = bool(obj.get("is_error")) or obj.get("subtype", "success") != "success"
|
err = bool(obj.get("is_error")) or obj.get("subtype", "success") != "success"
|
||||||
last = "" if err else (obj.get("result") or "")
|
last = "" if err else (obj.get("result") or "")
|
||||||
tail = self.stderr_tail(handle)
|
|
||||||
if err:
|
if err:
|
||||||
tail = ("%s %s" % (obj.get("subtype", ""), str(obj.get("result", ""))[:400]) + "\n" + tail).strip()
|
tail = ("%s %s" % (obj.get("subtype", ""), str(obj.get("result", ""))[:400]) + "\n" + tail).strip()
|
||||||
return ExecResult(last.strip(), self.exit_code(handle), obj.get("session_id") or handle.get("thread_id"),
|
return ExecResult(last.strip(), parse_exit(exit_text), obj.get("session_id") or handle.get("thread_id"),
|
||||||
self.logs(handle, "out.json", "stderr.txt"), tail)
|
self.logs(handle, "out.json", "stderr.txt"), tail)
|
||||||
|
|||||||
@@ -2,22 +2,22 @@
|
|||||||
Thread id comes from the `thread.started` event in events.jsonl; the last message from last.md."""
|
Thread id comes from the `thread.started` event in events.jsonl; the last message from last.md."""
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from .base import ExecResult, RunnerExecutor
|
from .base import ExecResult, RunnerExecutor, parse_exit
|
||||||
|
|
||||||
|
|
||||||
class CodexExecutor(RunnerExecutor):
|
class CodexExecutor(RunnerExecutor):
|
||||||
stdout_name = "events.jsonl"
|
stdout_name = "events.jsonl"
|
||||||
|
COLLECT = [("read", "exit.txt"), ("read", "last.md"), ("grep", "events.jsonl", "thread.started"),
|
||||||
|
("tail", "stderr.txt", 5), ("tail", "events.jsonl", 1)]
|
||||||
|
|
||||||
def collect(self, handle):
|
def collect(self, handle):
|
||||||
t, d = self.transport, handle["run_dir"]
|
exit_text, last, line, err_tail, last_event = self.fetch(handle, self.COLLECT) # one remote call
|
||||||
last = t.read_text(t.join(d, "last.md")) or ""
|
|
||||||
thread = handle.get("thread_id")
|
thread = handle.get("thread_id")
|
||||||
line = t.grep_first(t.join(d, "events.jsonl"), "thread.started")
|
|
||||||
if line:
|
if line:
|
||||||
try:
|
try:
|
||||||
thread = json.loads(line).get("thread_id") or thread
|
thread = json.loads(line).get("thread_id") or thread
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
tail = (self.stderr_tail(handle) + "\n" + t.tail(t.join(d, "events.jsonl"), 1)[-600:]).strip()
|
tail = (err_tail + "\n" + last_event[-600:]).strip()
|
||||||
return ExecResult(last.strip(), self.exit_code(handle), thread,
|
return ExecResult((last or "").strip(), parse_exit(exit_text), thread,
|
||||||
self.logs(handle, "events.jsonl", "stderr.txt", "last.md"), tail)
|
self.logs(handle, "events.jsonl", "stderr.txt", "last.md"), tail)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"""Shell/script executor: runs `{script}` with the prompt on stdin. The last message is `last.md` in the run
|
"""Shell/script executor: runs `{script}` with the prompt on stdin. The last message is `last.md` in the run
|
||||||
directory if the script wrote it (its path is passed as {out}), else the tail of stdout. No threads."""
|
directory if the script wrote it (its path is passed as {out}), else the tail of stdout. No threads."""
|
||||||
from .base import ExecResult, RunnerExecutor
|
from .base import ExecResult, RunnerExecutor, parse_exit
|
||||||
|
|
||||||
|
|
||||||
class ShellExecutor(RunnerExecutor):
|
class ShellExecutor(RunnerExecutor):
|
||||||
@@ -10,10 +10,11 @@ class ShellExecutor(RunnerExecutor):
|
|||||||
def resume(self):
|
def resume(self):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
COLLECT = [("read", "exit.txt"), ("read", "last.md"), ("read", "stdout.txt"), ("tail", "stderr.txt", 5)]
|
||||||
|
|
||||||
def collect(self, handle):
|
def collect(self, handle):
|
||||||
t, d = self.transport, handle["run_dir"]
|
exit_text, last, stdout, tail = self.fetch(handle, self.COLLECT) # one remote call
|
||||||
last = t.read_text(t.join(d, "last.md"))
|
|
||||||
if last is None:
|
if last is None:
|
||||||
last = (t.read_text(t.join(d, "stdout.txt")) or "")[-6000:]
|
last = (stdout or "")[-6000:]
|
||||||
return ExecResult(last.strip(), self.exit_code(handle), None,
|
return ExecResult(last.strip(), parse_exit(exit_text), None,
|
||||||
self.logs(handle, "stdout.txt", "stderr.txt"), self.stderr_tail(handle))
|
self.logs(handle, "stdout.txt", "stderr.txt"), tail)
|
||||||
|
|||||||
@@ -4,21 +4,162 @@ Interface (duck-typed, see docs/executor-contract.md):
|
|||||||
os, join(*parts), quote(s), default_run_root()
|
os, join(*parts), quote(s), default_run_root()
|
||||||
mkdir(path, clean=False), write_text(path, text), read_text(path) -> str|None,
|
mkdir(path, clean=False), write_text(path, text), read_text(path) -> str|None,
|
||||||
grep_first(path, needle) -> str|None, tail(path, n) -> str
|
grep_first(path, needle) -> str|None, tail(path, n) -> str
|
||||||
|
fetch(run_dir, ops) -> [result per op] (ops: ("read", name) | ("grep", name, needle) | ("tail", name, n))
|
||||||
runner(run_dir, cwd, command, stdout_name) -> (script_name, script_text)
|
runner(run_dir, cwd, command, stdout_name) -> (script_name, script_text)
|
||||||
launch(task, script_path, run_dir), kill(task, run_dir), forget(task)
|
start_run(task, run_dir, files, script_name) (remote only), launch(task, script_path, run_dir), kill(task, run_dir),
|
||||||
process_count(name) -> int|None (None = unreachable)"""
|
forget(task), process_count(name) -> int|None (None = unreachable)
|
||||||
|
|
||||||
|
Remote terminals are reached through a channel (design §9): `warpgate-ssh` (agent-access.py run through the
|
||||||
|
Warpgate gateway) or `direct-ssh` (OpenSSH over Tailscale with a dedicated key, optional warpgate fallback)."""
|
||||||
|
import base64
|
||||||
|
import functools
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
class TransportError(Exception):
|
class TransportError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def make_transport(cfg, terminal):
|
class Unreached(TransportError):
|
||||||
kind = cfg.terminals[terminal]["transport"]
|
"""The remote command certainly did not run (connect / auth / agent failure, direct path unavailable): the
|
||||||
|
only kind of failure a direct-ssh terminal may repeat on its fallback channel."""
|
||||||
|
|
||||||
|
|
||||||
|
class HostKeyError(TransportError):
|
||||||
|
"""The terminal presented a host key other than the pinned one: a security error, never retried elsewhere."""
|
||||||
|
|
||||||
|
|
||||||
|
class CallLog:
|
||||||
|
"""One line per remote call -> `path` (remote.log); notable events (fallback, host key) also -> `event`."""
|
||||||
|
|
||||||
|
def __init__(self, path=None, event=None, echo=False):
|
||||||
|
self.path, self.event_fn, self.echo = path, event, echo
|
||||||
|
self._once = set()
|
||||||
|
|
||||||
|
def call(self, terminal, via, op, ms, rc, note=""):
|
||||||
|
line = "term=%s via=%s op=%s ms=%d rc=%s%s" % (terminal, via, op or "-", ms, rc, " " + note if note else "")
|
||||||
|
if self.echo:
|
||||||
|
print(line, flush=True)
|
||||||
|
if self.path:
|
||||||
|
try:
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(self.path, "a", encoding="utf-8") as fh:
|
||||||
|
fh.write(time.strftime("%Y-%m-%dT%H:%M:%S") + " " + line + "\n")
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def event(self, msg, once=None):
|
||||||
|
"""`once` = a key: the message is logged only the first time per process."""
|
||||||
|
if once is not None:
|
||||||
|
if once in self._once:
|
||||||
|
return
|
||||||
|
self._once.add(once)
|
||||||
|
if self.event_fn:
|
||||||
|
self.event_fn(msg)
|
||||||
|
|
||||||
|
|
||||||
|
NULL_LOG = CallLog()
|
||||||
|
|
||||||
|
|
||||||
|
def operation(name):
|
||||||
|
"""Method decorator: names the remote operation for the call log (innermost wins, restored on return)."""
|
||||||
|
def deco(fn):
|
||||||
|
@functools.wraps(fn)
|
||||||
|
def wrapper(self, *args, **kwargs):
|
||||||
|
prev, self._op = getattr(self, "_op", None), name
|
||||||
|
try:
|
||||||
|
return fn(self, *args, **kwargs)
|
||||||
|
finally:
|
||||||
|
self._op = prev
|
||||||
|
return wrapper
|
||||||
|
return deco
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_each(t, run_dir, ops):
|
||||||
|
"""`fetch` built from the single-file primitives (one call per op): local transport and fallbacks."""
|
||||||
|
out = []
|
||||||
|
for o in ops:
|
||||||
|
path = t.join(run_dir, o[1])
|
||||||
|
if o[0] == "read":
|
||||||
|
out.append(t.read_text(path))
|
||||||
|
elif o[0] == "grep":
|
||||||
|
out.append(t.grep_first(path, o[2]))
|
||||||
|
elif o[0] == "tail":
|
||||||
|
out.append(t.tail(path, o[2]))
|
||||||
|
else:
|
||||||
|
raise ValueError("unknown fetch op %r" % (o,))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def parse_fetch(out, ops, what=""):
|
||||||
|
"""Decode the `F<i>=<base64>` / `M<i>` (missing) lines of a remote fetch script into results per op:
|
||||||
|
read -> text|None, grep -> line|None, tail -> text ("" when missing)."""
|
||||||
|
res = []
|
||||||
|
for i, o in enumerate(ops):
|
||||||
|
m = re.search(r"^F%d=([A-Za-z0-9+/=]*)$" % i, out, re.M)
|
||||||
|
if m:
|
||||||
|
text = base64.b64decode(m.group(1)).decode("utf-8", "replace") if m.group(1) else ""
|
||||||
|
elif re.search(r"^M%d$" % i, out, re.M):
|
||||||
|
text = None
|
||||||
|
else:
|
||||||
|
raise TransportError("fetch %s: unexpected output: %s" % (what, out.strip()[-200:]))
|
||||||
|
if o[0] == "grep":
|
||||||
|
text = (text or "").rstrip("\n") or None
|
||||||
|
elif o[0] == "tail":
|
||||||
|
text = (text or "").rstrip("\n")
|
||||||
|
res.append(text)
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
class RoutedTransport:
|
||||||
|
"""direct-ssh terminal: every operation runs on the primary (direct) transport; when it fails before the remote
|
||||||
|
command ran (`Unreached`) the operation is repeated on the fallback (warpgate-ssh) transport, which then serves
|
||||||
|
the rest of this transport's life (one tick). Each direct operation is a single remote call, so a repeat never
|
||||||
|
doubles a side effect. Host-key failures and failures after the command may have run propagate unchanged."""
|
||||||
|
PURE = ("join", "quote", "default_run_root", "runner")
|
||||||
|
|
||||||
|
def __init__(self, terminal, primary, fallback=None, log=None):
|
||||||
|
self.terminal, self.primary, self.fallback = terminal, primary, fallback
|
||||||
|
self.log = log or NULL_LOG
|
||||||
|
self.on_fallback = False
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
attr = getattr(self.primary, name)
|
||||||
|
if name in self.PURE or not callable(attr):
|
||||||
|
return attr
|
||||||
|
|
||||||
|
def routed(*args, **kwargs):
|
||||||
|
if self.on_fallback:
|
||||||
|
return getattr(self.fallback, name)(*args, **kwargs)
|
||||||
|
try:
|
||||||
|
return attr(*args, **kwargs)
|
||||||
|
except Unreached as exc:
|
||||||
|
if self.fallback is None:
|
||||||
|
raise
|
||||||
|
self.on_fallback = True
|
||||||
|
self.log.event("%s: direct-ssh failed before the command ran (%s); using the warpgate-ssh fallback "
|
||||||
|
"for the rest of this tick" % (self.terminal, exc))
|
||||||
|
return getattr(self.fallback, name)(*args, **kwargs)
|
||||||
|
return routed
|
||||||
|
|
||||||
|
|
||||||
|
def make_transport(cfg, terminal, log=None, force=None):
|
||||||
|
"""`force` = direct-ssh | warpgate-ssh: that channel only, no fallback (agentd ping)."""
|
||||||
|
t = cfg.terminals[terminal]
|
||||||
|
kind = force or t["transport"]
|
||||||
if kind == "local":
|
if kind == "local":
|
||||||
from .local import LocalTransport
|
from .local import LocalTransport
|
||||||
return LocalTransport(cfg, terminal)
|
return LocalTransport(cfg, terminal)
|
||||||
if kind == "warpgate-ssh":
|
if kind == "warpgate-ssh":
|
||||||
from .warpgate_ssh import WarpgateSshTransport
|
from .warpgate_ssh import WarpgateSshTransport
|
||||||
return WarpgateSshTransport(cfg, terminal)
|
return WarpgateSshTransport(cfg, terminal, log=log)
|
||||||
|
if kind == "direct-ssh":
|
||||||
|
from .direct_ssh import DirectChannel
|
||||||
|
from .warpgate_ssh import WarpgateSshTransport
|
||||||
|
primary = WarpgateSshTransport(cfg, terminal, channel=DirectChannel(cfg, terminal, log))
|
||||||
|
fallback = None
|
||||||
|
if not force and t.get("fallback") == "warpgate-ssh":
|
||||||
|
fallback = WarpgateSshTransport(cfg, terminal, log=log, label="fallback")
|
||||||
|
return RoutedTransport(terminal, primary, fallback, log)
|
||||||
raise TransportError("unknown transport %s" % kind)
|
raise TransportError("unknown transport %s" % kind)
|
||||||
|
|||||||
@@ -0,0 +1,313 @@
|
|||||||
|
"""direct-ssh channel (PAD-12): OpenSSH from the dispatcher straight to the terminal over Tailscale, no Warpgate.
|
||||||
|
|
||||||
|
- Auth: one dedicated agentd key. Its private text arrives in the env var named by `agentd.ssh_key` (default
|
||||||
|
AGENTD_SSH_KEY, injected by `agent-access.py exec --secret-env`), is popped from os.environ (children never see
|
||||||
|
it) and loaded with `ssh-add -` into a per-process ssh-agent on a socket in a private 0700 temp dir. The key is
|
||||||
|
never written to disk, argv, logs or exceptions. No key / agent failure -> the channel is unavailable
|
||||||
|
(`Unreached`), so a terminal with `fallback: warpgate-ssh` uses Warpgate instead.
|
||||||
|
- Host key: pinned per terminal (`ssh.host_key`); a known_hosts file is generated from it (public data) and
|
||||||
|
checked strictly. A mismatch raises HostKeyError: never a fallback.
|
||||||
|
- Reuse: ControlMaster=auto / ControlPersist=60 under `agentd.ssh_dir` (root-only 0700), so the second call of a
|
||||||
|
tick (and the next tick within a minute) skips the handshake. The socket name carries a fingerprint of the pinned
|
||||||
|
host key: a changed pin never rides an open master that was verified against the old one.
|
||||||
|
- Payload on stdin: Windows runs a fixed PowerShell bootstrap (-EncodedCommand) that reads the base64 of the real
|
||||||
|
script from stdin; POSIX runs a fixed bash loader that reads the script from stdin. One call per operation, no
|
||||||
|
argv size limit."""
|
||||||
|
import atexit
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
from . import NULL_LOG, HostKeyError, TransportError, Unreached
|
||||||
|
from .local import sh_quote
|
||||||
|
|
||||||
|
CONNECT_TIMEOUT = 8
|
||||||
|
CONTROL_PERSIST = 60
|
||||||
|
SOCKET_MAX = 104 # sun_path; ssh adds ".<16 random chars>" while it sets up the master socket
|
||||||
|
CTL_NAME = 1 + 9 + 40 + 17 # "/<pin8>-<%C: 40 hex>" + that suffix
|
||||||
|
|
||||||
|
# Fixed PowerShell bootstrap: the operation's script arrives on stdin as base64 of its UTF-8 text (ASCII on the
|
||||||
|
# wire, so the console input code page does not matter).
|
||||||
|
BOOTSTRAP = ("$ProgressPreference='SilentlyContinue'\n"
|
||||||
|
"$ErrorActionPreference='Continue'\n"
|
||||||
|
"[Console]::OutputEncoding=[Text.Encoding]::UTF8\n"
|
||||||
|
"$s=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String([Console]::In.ReadToEnd().Trim()))\n"
|
||||||
|
"& ([ScriptBlock]::Create($s))\n")
|
||||||
|
|
||||||
|
|
||||||
|
def encoded_command(script):
|
||||||
|
"""PowerShell -EncodedCommand value: base64 of the UTF-16LE text."""
|
||||||
|
return base64.b64encode(script.encode("utf-16-le")).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
PS_COMMAND = "powershell.exe -NoProfile -NonInteractive -EncodedCommand " + encoded_command(BOOTSTRAP)
|
||||||
|
# POSIX loader: the script is read from stdin into a variable (no MAX_ARG_STRLEN), then runs in a subshell with
|
||||||
|
# stdin = /dev/null. Its exit 255 is reported as 254: 255 stays ssh's own "no session" code. (The PowerShell
|
||||||
|
# bootstrap exits 0/1 by $?; none of our PowerShell scripts calls `exit`.)
|
||||||
|
SH_COMMAND = "bash -c " + sh_quote('exec 3<&0 </dev/null; s=$(cat <&3); exec 3<&-; (eval "$s"); r=$?; '
|
||||||
|
'[ $r = 255 ] && r=254; exit $r')
|
||||||
|
|
||||||
|
|
||||||
|
def ps_payload(script):
|
||||||
|
return (base64.b64encode(script.encode("utf-8")).decode("ascii") + "\n").encode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- failure classification
|
||||||
|
# A host offering no ssh-ed25519 key at all counts as a host-key failure too (a downgrade is a security signal).
|
||||||
|
HOSTKEY = re.compile(r"Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED|"
|
||||||
|
r"No \S+ host key is known for|host key for \S+ has changed|no matching host key type", re.I)
|
||||||
|
# ssh exit 255 with one of these = no remote session ran the command. Only ssh's own messages, anchored at the start
|
||||||
|
# of a line (stderr also carries the remote command's stderr).
|
||||||
|
UNREACHED = re.compile(r"^(?:ssh: connect to host |ssh: Could not resolve hostname |banner exchange: |"
|
||||||
|
r"Connection timed out during banner exchange|kex_exchange_identification: |"
|
||||||
|
r"Connection (?:closed|reset) by \S+ port \d+\s*$|"
|
||||||
|
r"\S+: Permission denied \((?:publickey|password|keyboard-interactive|hostbased|gssapi)|"
|
||||||
|
r"Received disconnect from .*Too many authentication failures|"
|
||||||
|
r"sign_and_send_pubkey: |error fetching identities|Error connecting to agent|"
|
||||||
|
r"Unable to negotiate with |Session open refused by peer)", re.I | re.M)
|
||||||
|
|
||||||
|
|
||||||
|
def classify(rc, err, out=""):
|
||||||
|
"""-> 'ok' (the remote command ran, whatever its exit code), 'hostkey', 'unreached' (certainly not run) or
|
||||||
|
'unknown' (ssh exit 255 without a recognisable pre-session error: the command may have run). Any stdout proves
|
||||||
|
that a session ran the command, so it is never 'unreached'."""
|
||||||
|
if rc != 255:
|
||||||
|
return "ok"
|
||||||
|
if (out or "").strip():
|
||||||
|
return "unknown"
|
||||||
|
if HOSTKEY.search(err or ""):
|
||||||
|
return "hostkey"
|
||||||
|
if UNREACHED.search(err or ""):
|
||||||
|
return "unreached"
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- per-process key agent
|
||||||
|
def _die_with_parent():
|
||||||
|
"""preexec: the agent gets SIGTERM when agentd dies, even by SIGKILL (Linux PR_SET_PDEATHSIG)."""
|
||||||
|
try:
|
||||||
|
import ctypes
|
||||||
|
ctypes.CDLL("libc.so.6", use_errno=True).prctl(1, signal.SIGTERM)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def normalise_key(text):
|
||||||
|
"""Vault may hand the key with literal `\\n` escapes or without the final newline."""
|
||||||
|
if "\n" not in text.strip() and "\\n" in text:
|
||||||
|
text = text.replace("\\n", "\n")
|
||||||
|
return text.strip() + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
_KEYS = {} # env var name -> key text, taken out of os.environ at CLI start until the agent loads it
|
||||||
|
|
||||||
|
|
||||||
|
def take_key(env_name):
|
||||||
|
"""Move the key out of os.environ right away (called by the CLI before anything else runs), so no child -- not
|
||||||
|
even one started before the first direct call -- can inherit it."""
|
||||||
|
if env_name not in _KEYS:
|
||||||
|
_KEYS[env_name] = os.environ.pop(env_name, None)
|
||||||
|
else:
|
||||||
|
os.environ.pop(env_name, None)
|
||||||
|
|
||||||
|
|
||||||
|
def key_present(env_name):
|
||||||
|
return bool((_KEYS.get(env_name) or os.environ.get(env_name) or "").strip())
|
||||||
|
|
||||||
|
|
||||||
|
class KeyAgent:
|
||||||
|
"""ssh-agent of this process holding the agentd key. `error` set = direct-ssh unavailable here."""
|
||||||
|
|
||||||
|
def __init__(self, env_name):
|
||||||
|
self.env_name = env_name
|
||||||
|
self.dir = self.sock = self.pub = self.proc = None
|
||||||
|
self.error = None
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
key = os.environ.pop(self.env_name, None) # popped even if loading fails: children never inherit it
|
||||||
|
key = _KEYS.pop(self.env_name, None) or key
|
||||||
|
if not key or not key.strip():
|
||||||
|
self.error = "%s is not set (agent-access.py exec --secret-env %s=<vault key>)" % (self.env_name, self.env_name)
|
||||||
|
return self
|
||||||
|
agent_bin, add_bin = shutil.which("ssh-agent"), shutil.which("ssh-add")
|
||||||
|
if not (agent_bin and add_bin and shutil.which("ssh")):
|
||||||
|
self.error = "ssh / ssh-agent / ssh-add not installed"
|
||||||
|
return self
|
||||||
|
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LANG": "C"}
|
||||||
|
try:
|
||||||
|
self.dir = tempfile.mkdtemp(prefix="agentd-agent-") # 0700
|
||||||
|
self.sock = os.path.join(self.dir, "agent.sock")
|
||||||
|
atexit.register(self.close)
|
||||||
|
self.proc = subprocess.Popen([agent_bin, "-D", "-a", self.sock], stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env,
|
||||||
|
close_fds=True, preexec_fn=_die_with_parent)
|
||||||
|
end = time.monotonic() + 5
|
||||||
|
while not os.path.exists(self.sock):
|
||||||
|
if self.proc.poll() is not None or time.monotonic() > end:
|
||||||
|
self.error = "ssh-agent did not start"
|
||||||
|
return self
|
||||||
|
time.sleep(0.02)
|
||||||
|
env["SSH_AUTH_SOCK"] = self.sock
|
||||||
|
add = subprocess.run([add_bin, "-q", "-"], input=normalise_key(key).encode("utf-8"), env=env,
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=15)
|
||||||
|
key = None
|
||||||
|
if add.returncode != 0:
|
||||||
|
why = add.stderr.decode("utf-8", "replace").strip().splitlines()[:1]
|
||||||
|
why = why[0][:120] if why and "-----" not in why[0] else ""
|
||||||
|
self.error = "ssh-add failed (rc %d) %s" % (add.returncode, why)
|
||||||
|
return self
|
||||||
|
listed = subprocess.run([add_bin, "-L"], env=env, capture_output=True, timeout=15)
|
||||||
|
pub = next((l for l in listed.stdout.decode("ascii", "replace").splitlines() if l.startswith("ssh-")), None)
|
||||||
|
if not pub:
|
||||||
|
self.error = "ssh-agent holds no key after ssh-add"
|
||||||
|
return self
|
||||||
|
self.pub = os.path.join(self.dir, "agentd.pub") # public half: selects the agent key (IdentitiesOnly)
|
||||||
|
with open(self.pub, "w", encoding="ascii") as fh:
|
||||||
|
fh.write(pub + "\n")
|
||||||
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
|
self.error = "ssh-agent setup failed: %s" % type(exc).__name__
|
||||||
|
finally:
|
||||||
|
key = None
|
||||||
|
if self.error:
|
||||||
|
self.close()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
if self.proc and self.proc.poll() is None:
|
||||||
|
self.proc.terminate()
|
||||||
|
try:
|
||||||
|
self.proc.wait(3)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
self.proc.kill()
|
||||||
|
if self.dir:
|
||||||
|
shutil.rmtree(self.dir, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
_AGENTS = {}
|
||||||
|
|
||||||
|
|
||||||
|
def key_agent(cfg, log=NULL_LOG):
|
||||||
|
"""The process-wide agent for cfg's key env var, started on first use; raises Unreached when unavailable."""
|
||||||
|
name = cfg.ssh_key_env
|
||||||
|
agent = _AGENTS.get(name)
|
||||||
|
if agent is None:
|
||||||
|
agent = _AGENTS[name] = KeyAgent(name).start()
|
||||||
|
if agent.error:
|
||||||
|
log.event("direct-ssh unavailable in this process: %s" % agent.error, once="agent:" + name)
|
||||||
|
raise Unreached("direct-ssh unavailable: %s" % agent.error)
|
||||||
|
return agent
|
||||||
|
|
||||||
|
|
||||||
|
def control_dir(cfg):
|
||||||
|
"""agentd.ssh_dir: ControlMaster sockets + generated known_hosts; created 0700, must be ours."""
|
||||||
|
d = cfg.ssh_dir
|
||||||
|
try:
|
||||||
|
d.mkdir(parents=True, exist_ok=True)
|
||||||
|
os.chmod(d, 0o700)
|
||||||
|
if os.stat(d).st_uid != os.geteuid():
|
||||||
|
raise OSError("not owned by this user")
|
||||||
|
except OSError as exc:
|
||||||
|
raise Unreached("ssh_dir %s unusable: %s" % (d, exc)) from None
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- channel
|
||||||
|
class DirectChannel:
|
||||||
|
"""One terminal's direct OpenSSH path. ps() / sh() mirror the warpgate channel; payloads go on stdin."""
|
||||||
|
stdin = True
|
||||||
|
|
||||||
|
def __init__(self, cfg, terminal, log=None, label="direct"):
|
||||||
|
self.cfg, self.terminal, self.log, self.via = cfg, terminal, log or NULL_LOG, label
|
||||||
|
s = cfg.terminals[terminal]["ssh"]
|
||||||
|
self.host, self.user, self.port = s["host"], s["user"], int(s.get("port", 22))
|
||||||
|
self.host_key = " ".join(s["host_key"].split()[:2])
|
||||||
|
self.pin = hashlib.sha256(self.host_key.encode("ascii")).hexdigest()[:8]
|
||||||
|
self.alias = "agentd-" + terminal
|
||||||
|
self.hostkey_failed = None
|
||||||
|
self.name = "ssh %s@%s" % (self.user, self.host)
|
||||||
|
|
||||||
|
def known_hosts(self, d):
|
||||||
|
path = d / ("known_hosts." + self.terminal)
|
||||||
|
line = "%s %s\n" % (self.alias, self.host_key)
|
||||||
|
try:
|
||||||
|
if not path.exists() or path.read_text(encoding="ascii") != line:
|
||||||
|
tmp = path.with_name("%s.%d.tmp" % (path.name, os.getpid())) # per terminal and process
|
||||||
|
tmp.write_text(line, encoding="ascii")
|
||||||
|
os.replace(tmp, path)
|
||||||
|
except OSError as exc:
|
||||||
|
raise Unreached("known_hosts %s: %s" % (path, exc)) from None
|
||||||
|
return path
|
||||||
|
|
||||||
|
def argv(self, agent, remote_command):
|
||||||
|
d = control_dir(self.cfg)
|
||||||
|
ctl = str(d / (self.pin + "-%C")) # pin fingerprint: a new pin never reuses a master verified on the old one
|
||||||
|
if len(str(d)) + CTL_NAME > SOCKET_MAX:
|
||||||
|
self.log.event("agentd.ssh_dir %s is too long for a control socket; connection reuse off" % d, once="ctl")
|
||||||
|
mux = ["-o", "ControlMaster=no", "-o", "ControlPath=none"]
|
||||||
|
else:
|
||||||
|
mux = ["-o", "ControlMaster=auto", "-o", "ControlPersist=%d" % CONTROL_PERSIST, "-o", "ControlPath=" + ctl]
|
||||||
|
return (["ssh", "-F", "/dev/null", "-T", "-p", str(self.port), "-l", self.user,
|
||||||
|
"-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
|
||||||
|
"-o", "IdentityAgent=" + agent.sock, "-o", "IdentityFile=" + agent.pub,
|
||||||
|
"-o", "PreferredAuthentications=publickey", "-o", "PasswordAuthentication=no",
|
||||||
|
"-o", "KbdInteractiveAuthentication=no",
|
||||||
|
"-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + str(self.known_hosts(d)),
|
||||||
|
"-o", "GlobalKnownHostsFile=/dev/null", "-o", "HostKeyAlias=" + self.alias,
|
||||||
|
"-o", "HostKeyAlgorithms=" + self.host_key.split()[0], "-o", "UpdateHostKeys=no",
|
||||||
|
"-o", "ConnectTimeout=%d" % CONNECT_TIMEOUT, "-o", "ServerAliveInterval=15",
|
||||||
|
"-o", "ServerAliveCountMax=3", "-o", "ForwardAgent=no", "-o", "ForwardX11=no",
|
||||||
|
"-o", "ClearAllForwardings=yes", "-o", "LogLevel=ERROR", "-o", "EscapeChar=none"]
|
||||||
|
+ mux + ["--", self.host, remote_command])
|
||||||
|
|
||||||
|
def run(self, remote_command, payload, timeout, op):
|
||||||
|
"""-> (rc, stdout, stderr) of a call whose command ran; raises HostKeyError / Unreached / TransportError."""
|
||||||
|
if self.hostkey_failed:
|
||||||
|
raise HostKeyError(self.hostkey_failed)
|
||||||
|
agent = key_agent(self.cfg, self.log)
|
||||||
|
argv = self.argv(agent, remote_command)
|
||||||
|
env = self.cfg.scrubbed_env()
|
||||||
|
env.pop("SSH_AUTH_SOCK", None)
|
||||||
|
t0 = time.monotonic()
|
||||||
|
ms = lambda: (time.monotonic() - t0) * 1000 # noqa: E731
|
||||||
|
# output goes to files, not pipes: a ControlPersist master forked by this call must never hold our pipes open
|
||||||
|
with tempfile.TemporaryFile() as fo, tempfile.TemporaryFile() as fe:
|
||||||
|
try:
|
||||||
|
p = subprocess.run(argv, input=payload, stdout=fo, stderr=fe, env=env, timeout=timeout)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
self.log.call(self.terminal, self.via, op, ms(), "timeout")
|
||||||
|
raise TransportError("%s timeout after %ss" % (self.name, timeout)) from None # may have run
|
||||||
|
except OSError as exc:
|
||||||
|
self.log.call(self.terminal, self.via, op, ms(), "launch-failed")
|
||||||
|
raise Unreached("%s: ssh launch failed: %s" % (self.name, type(exc).__name__)) from None
|
||||||
|
fo.seek(0)
|
||||||
|
fe.seek(0)
|
||||||
|
dec = lambda b: b.decode("utf-8", "replace").replace("\r", "") # noqa: E731
|
||||||
|
out, err = dec(fo.read()), dec(fe.read())
|
||||||
|
kind = classify(p.returncode, err, out)
|
||||||
|
self.log.call(self.terminal, self.via, op, ms(), p.returncode, "" if kind == "ok" else kind)
|
||||||
|
if kind == "hostkey":
|
||||||
|
self.hostkey_failed = ("%s:%d host key verification FAILED (pinned %s...; other key or no such key type "
|
||||||
|
"offered): not falling back; check terminals.%s.ssh.host_key out of band"
|
||||||
|
% (self.host, self.port, self.host_key[:40], self.terminal))
|
||||||
|
self.log.event("SECURITY %s: %s" % (self.terminal, self.hostkey_failed), once="hostkey:" + self.terminal)
|
||||||
|
raise HostKeyError(self.hostkey_failed)
|
||||||
|
if kind == "unreached":
|
||||||
|
raise Unreached("%s exit 255: %s" % (self.name, err.strip()[-300:]))
|
||||||
|
if kind == "unknown":
|
||||||
|
raise TransportError("%s exit 255: %s" % (self.name, err.strip()[-300:] or "(no message)"))
|
||||||
|
return p.returncode, out, err
|
||||||
|
|
||||||
|
def ps(self, script, timeout, op=None):
|
||||||
|
rc, out, err = self.run(PS_COMMAND, ps_payload(script), timeout, op)
|
||||||
|
if rc != 0 and not out.strip():
|
||||||
|
raise TransportError("%s exit %s: %s" % (self.name, rc, err.strip()[-300:]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def sh(self, script, timeout, op=None):
|
||||||
|
return self.run(SH_COMMAND, script.encode("utf-8"), timeout, op)
|
||||||
@@ -91,5 +91,5 @@ class LocalTransport:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def process_count(self, name):
|
def process_count(self, name):
|
||||||
out = subprocess.run(["pgrep", "-c", "-x", name], capture_output=True, text=True)
|
out = subprocess.run(["pgrep", "-c", "-x", name], capture_output=True, text=True, env=self.cfg.scrubbed_env())
|
||||||
return int(out.stdout.strip() or 0)
|
return int(out.stdout.strip() or 0)
|
||||||
|
|||||||
@@ -1,17 +1,25 @@
|
|||||||
"""Terminals behind Warpgate, reached through the server-access relay client:
|
"""Remote terminals (Windows and Linux/macOS) and the Warpgate channel.
|
||||||
|
|
||||||
|
The transport classes build the scripts; a *channel* carries them (design §9):
|
||||||
|
- WarpgateChannel (`transport: warpgate-ssh`, also the direct-ssh fallback): the server-access relay client
|
||||||
|
|
||||||
agent-access.py --config <identity> run --target <target> --timeout N -- <argv...>
|
agent-access.py --config <identity> run --target <target> --timeout N -- <argv...>
|
||||||
|
|
||||||
The client shlex-joins argv and the target account's login shell runs it (cmd.exe on Windows); stdin is
|
The client shlex-joins argv and the target account's login shell runs it (cmd.exe on Windows); stdin is
|
||||||
/dev/null, so everything travels in argv. `relay()` is the single place that builds that call.
|
/dev/null (agent-access.py run_remote uses stdin=DEVNULL), so everything travels in argv. `relay()` is the single
|
||||||
|
place that builds that call.
|
||||||
|
- DirectChannel (`transport: direct-ssh`, agentd/transports/direct_ssh.py): OpenSSH over Tailscale; the script goes
|
||||||
|
on stdin, so every operation (including a launch with its prompt) is one call.
|
||||||
|
|
||||||
- os linux / darwin -> PosixWarpgateTransport: every operation is one `bash -c '<script>'` relay call. Files go up
|
- os linux / darwin -> PosixWarpgateTransport: every operation is one `bash -c '<script>'` call. On the relay files go
|
||||||
as base64 chunks (`printf %s '<b64>' | base64 -d`), the runner starts detached with `setsid nohup bash run.sh`
|
up as base64 chunks (`printf %s '<b64>' | base64 -d`), on the direct channel in the same single call. The runner
|
||||||
(its own session: pid == pgid == sid), `poll` reads exit.txt, `kill` signals the process group and session.
|
starts detached with `setsid nohup bash run.sh` (its own session: pid == pgid == sid), `poll` reads exit.txt,
|
||||||
- os windows -> WindowsWarpgateTransport: still drives the helper scripts win10-ps.sh (PowerShell
|
`fetch` returns everything `collect` needs in one call, `kill` signals the process group and session.
|
||||||
-EncodedCommand) and win10-put.sh (chunked upload, SHA-256 verified), which hard-code `--target win10`;
|
- os windows -> WindowsWarpgateTransport: PowerShell scripts; runs are one-shot scheduled tasks (schtasks) so they
|
||||||
runs are one-shot scheduled tasks (schtasks) so they outlive the SSH session.
|
outlive the SSH session. On the relay it still drives the helper scripts win10-ps.sh (PowerShell -EncodedCommand,
|
||||||
TODO: move Windows onto relay() (`powershell -EncodedCommand` as argv) and drop the helper scripts."""
|
argv-size bound) and win10-put.sh (chunked upload, SHA-256 verified), which hard-code `--target win10`; on the
|
||||||
|
direct channel mkdir + clean + prompt.md + run.ps1 + schtasks is one call.
|
||||||
|
The class names are historical: the channel, not the class, decides the path."""
|
||||||
import base64
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
@@ -20,8 +28,9 @@ import re
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
from . import TransportError
|
from . import NULL_LOG, TransportError, Unreached, fetch_each, operation, parse_fetch
|
||||||
from .local import RUNNER as POSIX_RUNNER, sh_quote
|
from .local import RUNNER as POSIX_RUNNER, sh_quote
|
||||||
|
|
||||||
DEFAULT_CLIENT = "~/.local/share/server-access-client/agent-access.py"
|
DEFAULT_CLIENT = "~/.local/share/server-access-client/agent-access.py"
|
||||||
@@ -55,6 +64,55 @@ def relay(cfg, target, argv, timeout=150):
|
|||||||
return call(cfg, relay_argv(cfg, target, argv, timeout), timeout)
|
return call(cfg, relay_argv(cfg, target, argv, timeout), timeout)
|
||||||
|
|
||||||
|
|
||||||
|
PS_ARGV_MAX = 7600 # win10-ps.sh refuses longer -EncodedCommand values (cmd.exe line limit 8191)
|
||||||
|
PS_PRELUDE = 120 # characters win10-ps.sh puts in front of every script
|
||||||
|
|
||||||
|
|
||||||
|
class WarpgateChannel:
|
||||||
|
"""Calls through agent-access.py run (argv only). Each call is logged with its duration."""
|
||||||
|
stdin = False
|
||||||
|
|
||||||
|
def __init__(self, cfg, terminal, log=None, label="warpgate"):
|
||||||
|
self.cfg, self.terminal, self.log, self.via = cfg, terminal, log or NULL_LOG, label
|
||||||
|
t = cfg.terminals[terminal]
|
||||||
|
self.target = t.get("target", terminal)
|
||||||
|
self.name = "relay %s" % self.target
|
||||||
|
if t["os"] == "windows":
|
||||||
|
helpers = cfg.helpers_dir(terminal)
|
||||||
|
self.ps_script, self.put_script = str(helpers / "win10-ps.sh"), str(helpers / "win10-put.sh")
|
||||||
|
|
||||||
|
def timed(self, op, fn, *args):
|
||||||
|
t0 = time.monotonic()
|
||||||
|
try:
|
||||||
|
res = fn(*args)
|
||||||
|
except TransportError:
|
||||||
|
self.log.call(self.terminal, self.via, op, (time.monotonic() - t0) * 1000, "error")
|
||||||
|
raise
|
||||||
|
self.log.call(self.terminal, self.via, op, (time.monotonic() - t0) * 1000, res[0])
|
||||||
|
return res
|
||||||
|
|
||||||
|
def fits(self, script):
|
||||||
|
"""Whether win10-ps.sh accepts the script (its -EncodedCommand stays under PS_ARGV_MAX)."""
|
||||||
|
units = len(script.encode("utf-16-le")) // 2 + PS_PRELUDE
|
||||||
|
return (2 * units + 2) // 3 * 4 <= PS_ARGV_MAX
|
||||||
|
|
||||||
|
def ps(self, script, timeout, op=None):
|
||||||
|
rc, out, err = self.timed(op, call, self.cfg, [self.ps_script, script], timeout, {"WIN10_TIMEOUT": str(timeout)})
|
||||||
|
if rc != 0 and not out.strip():
|
||||||
|
raise TransportError("relay exit %s: %s" % (rc, err.strip()[-300:]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def put(self, local, remote, op=None):
|
||||||
|
with open(local, "rb") as fh:
|
||||||
|
sha = hashlib.sha256(fh.read()).hexdigest()
|
||||||
|
rc, out, err = self.timed(op, call, self.cfg, [self.put_script, local, remote], 540)
|
||||||
|
if ("remote sha256 " + sha) not in out:
|
||||||
|
raise TransportError("upload of %s failed: %s" % (remote, (out + err).strip()[-300:]))
|
||||||
|
|
||||||
|
def sh(self, script, timeout, op=None):
|
||||||
|
return self.timed(op, relay, self.cfg, self.target, ["bash", "-c", script], timeout)
|
||||||
|
|
||||||
|
|
||||||
RUNNER = r"""$ErrorActionPreference = 'Continue'
|
RUNNER = r"""$ErrorActionPreference = 'Continue'
|
||||||
[Console]::OutputEncoding = [Text.Encoding]::UTF8
|
[Console]::OutputEncoding = [Text.Encoding]::UTF8
|
||||||
$OutputEncoding = [Text.Encoding]::UTF8
|
$OutputEncoding = [Text.Encoding]::UTF8
|
||||||
@@ -68,6 +126,15 @@ Set-Content -Path '{d}\exit.txt' -Value ("exit=" + $LASTEXITCODE + " at " + (Get
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# Windows fetch helpers: F<i>=<base64> per result, M<i> = file missing (parse_fetch). Names must not collide with
|
||||||
|
# PowerShell's built-in aliases (aliases win over functions: `r` is Invoke-History).
|
||||||
|
PS_FETCH = r"""function AdRead($i,$n){$p=Join-Path $d $n; if(Test-Path -LiteralPath $p){"F$i="+[Convert]::ToBase64String([IO.File]::ReadAllBytes($p))}else{"M$i"}}
|
||||||
|
function AdOut($i,$t){"F$i="+[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))}
|
||||||
|
function AdGrep($i,$n,$x){$p=Join-Path $d $n; $m=$null; if(Test-Path -LiteralPath $p){$m=Select-String -LiteralPath $p -SimpleMatch -Pattern $x|Select-Object -First 1}; if($m){AdOut $i $m.Line}else{"F$i="}}
|
||||||
|
function AdTail($i,$n,$k){$p=Join-Path $d $n; $t=''; if(Test-Path -LiteralPath $p){$t=(Get-Content -LiteralPath $p -Encoding UTF8 -Tail $k) -join "`n"}; AdOut $i $t}
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
def _b64_payload(out, marker):
|
def _b64_payload(out, marker):
|
||||||
m = re.search(marker + r"=([A-Za-z0-9+/=]*)", out)
|
m = re.search(marker + r"=([A-Za-z0-9+/=]*)", out)
|
||||||
if not m:
|
if not m:
|
||||||
@@ -78,12 +145,11 @@ def _b64_payload(out, marker):
|
|||||||
class WindowsWarpgateTransport:
|
class WindowsWarpgateTransport:
|
||||||
os = "windows"
|
os = "windows"
|
||||||
|
|
||||||
def __init__(self, cfg, terminal):
|
def __init__(self, cfg, terminal, channel=None, log=None, label="warpgate"):
|
||||||
self.cfg, self.terminal = cfg, terminal
|
self.cfg, self.terminal = cfg, terminal
|
||||||
t = cfg.terminals[terminal]
|
t = cfg.terminals[terminal]
|
||||||
self.target = t.get("target", terminal)
|
self.target = t.get("target", terminal)
|
||||||
helpers = cfg.helpers_dir(terminal)
|
self.channel = channel or WarpgateChannel(cfg, terminal, log, label)
|
||||||
self.ps_script, self.put_script = str(helpers / "win10-ps.sh"), str(helpers / "win10-put.sh")
|
|
||||||
self.repo = t["repo"]
|
self.repo = t["repo"]
|
||||||
|
|
||||||
def default_run_root(self):
|
def default_run_root(self):
|
||||||
@@ -95,32 +161,38 @@ class WindowsWarpgateTransport:
|
|||||||
def quote(self, s):
|
def quote(self, s):
|
||||||
return "'" + str(s).replace("'", "''") + "'"
|
return "'" + str(s).replace("'", "''") + "'"
|
||||||
|
|
||||||
# ---------------------------------------------------------------- relay primitives
|
# ---------------------------------------------------------------- channel primitives
|
||||||
def ps(self, script, timeout=150):
|
def ps(self, script, timeout=150):
|
||||||
rc, out, err = call(self.cfg, [self.ps_script, script], timeout, {"WIN10_TIMEOUT": str(timeout)})
|
return self.channel.ps(script, timeout, getattr(self, "_op", None))
|
||||||
if rc != 0 and not out.strip():
|
|
||||||
raise TransportError("relay exit %s: %s" % (rc, err.strip()[-300:]))
|
|
||||||
return out
|
|
||||||
|
|
||||||
def put(self, local, remote):
|
def put(self, local, remote):
|
||||||
with open(local, "rb") as fh:
|
self.channel.put(local, remote, getattr(self, "_op", None))
|
||||||
sha = hashlib.sha256(fh.read()).hexdigest()
|
|
||||||
rc, out, err = call(self.cfg, [self.put_script, local, remote], 540)
|
@operation("ping")
|
||||||
if ("remote sha256 " + sha) not in out:
|
def ping(self):
|
||||||
raise TransportError("upload of %s failed: %s" % (remote, (out + err).strip()[-300:]))
|
out = self.ps("'PONG=' + $PSVersionTable.PSVersion.ToString() + ' ' + $env:COMPUTERNAME", timeout=60)
|
||||||
|
m = re.search(r"PONG=[^\n]*", out)
|
||||||
|
if not m:
|
||||||
|
raise TransportError("ping %s: unexpected output: %s" % (self.terminal, out.strip()[-200:]))
|
||||||
|
return m.group(0)
|
||||||
|
|
||||||
# ---------------------------------------------------------------- file ops
|
# ---------------------------------------------------------------- file ops
|
||||||
def mkdir(self, path, clean=False):
|
def mkdir_script(self, path, clean=False):
|
||||||
q = self.quote(path)
|
q = self.quote(path)
|
||||||
script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q
|
script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q
|
||||||
if clean: # never wipe a directory a runner has already started in
|
if clean: # never wipe a directory a runner has already started in
|
||||||
script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) "
|
script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) "
|
||||||
"{ 'HASPID'; return }\n" % (q, q))
|
"{ 'HASPID'; return }\n" % (q, q))
|
||||||
script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q
|
script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q
|
||||||
out = self.ps(script + "'MKDIR=ok'")
|
return script
|
||||||
|
|
||||||
|
@operation("mkdir")
|
||||||
|
def mkdir(self, path, clean=False):
|
||||||
|
out = self.ps(self.mkdir_script(path, clean) + "'MKDIR=ok'")
|
||||||
if "MKDIR=ok" not in out:
|
if "MKDIR=ok" not in out:
|
||||||
raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:]))
|
raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:]))
|
||||||
|
|
||||||
|
@operation("probe")
|
||||||
def probe(self, run_dir, names):
|
def probe(self, run_dir, names):
|
||||||
out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')"
|
out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')"
|
||||||
% (self.quote(run_dir), ",".join(self.quote(n) for n in names)))
|
% (self.quote(run_dir), ",".join(self.quote(n) for n in names)))
|
||||||
@@ -129,7 +201,20 @@ class WindowsWarpgateTransport:
|
|||||||
raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:]))
|
raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:]))
|
||||||
return [n for n in m.group(1).strip().split(",") if n]
|
return [n for n in m.group(1).strip().split(",") if n]
|
||||||
|
|
||||||
|
def write_script(self, path, data):
|
||||||
|
"""Write bytes exactly as given (UTF-8 without BOM for our texts), echo their SHA-256."""
|
||||||
|
return ("$p = %s\n[IO.File]::WriteAllBytes($p, [Convert]::FromBase64String('%s'))\n"
|
||||||
|
"'WROTE=' + (Get-FileHash -LiteralPath $p -Algorithm SHA256).Hash.ToLower()\n"
|
||||||
|
% (self.quote(path), base64.b64encode(data).decode("ascii")))
|
||||||
|
|
||||||
|
@operation("write")
|
||||||
def write_text(self, path, text):
|
def write_text(self, path, text):
|
||||||
|
if self.channel.stdin:
|
||||||
|
data = text.encode("utf-8")
|
||||||
|
out = self.ps(self.write_script(path, data))
|
||||||
|
if re.findall(r"WROTE=([0-9a-f]{64})", out) != [hashlib.sha256(data).hexdigest()]:
|
||||||
|
raise TransportError("upload of %s failed: %s" % (path, out.strip()[-300:]))
|
||||||
|
return
|
||||||
fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt")
|
fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt")
|
||||||
try:
|
try:
|
||||||
with os.fdopen(fd, "w", encoding="utf-8", newline="") as fh:
|
with os.fdopen(fd, "w", encoding="utf-8", newline="") as fh:
|
||||||
@@ -138,6 +223,7 @@ class WindowsWarpgateTransport:
|
|||||||
finally:
|
finally:
|
||||||
os.unlink(tmp)
|
os.unlink(tmp)
|
||||||
|
|
||||||
|
@operation("read")
|
||||||
def read_text(self, path):
|
def read_text(self, path):
|
||||||
q = self.quote(path)
|
q = self.quote(path)
|
||||||
out = self.ps("if (Test-Path -LiteralPath %s) { 'B64=' + [Convert]::ToBase64String([IO.File]::ReadAllBytes(%s)) }"
|
out = self.ps("if (Test-Path -LiteralPath %s) { 'B64=' + [Convert]::ToBase64String([IO.File]::ReadAllBytes(%s)) }"
|
||||||
@@ -147,8 +233,9 @@ class WindowsWarpgateTransport:
|
|||||||
text = _b64_payload(out, "B64")
|
text = _b64_payload(out, "B64")
|
||||||
if text is None:
|
if text is None:
|
||||||
raise TransportError("read %s: unexpected relay output" % path)
|
raise TransportError("read %s: unexpected relay output" % path)
|
||||||
return text.lstrip("")
|
return text.lstrip("\ufeff")
|
||||||
|
|
||||||
|
@operation("grep")
|
||||||
def grep_first(self, path, needle):
|
def grep_first(self, path, needle):
|
||||||
q = self.quote(path)
|
q = self.quote(path)
|
||||||
out = self.ps("$m = if (Test-Path -LiteralPath %s) { Select-String -LiteralPath %s -SimpleMatch -Pattern %s | Select-Object -First 1 }\n"
|
out = self.ps("$m = if (Test-Path -LiteralPath %s) { Select-String -LiteralPath %s -SimpleMatch -Pattern %s | Select-Object -First 1 }\n"
|
||||||
@@ -156,12 +243,30 @@ class WindowsWarpgateTransport:
|
|||||||
% (q, q, self.quote(needle)))
|
% (q, q, self.quote(needle)))
|
||||||
return _b64_payload(out, "B64") or None
|
return _b64_payload(out, "B64") or None
|
||||||
|
|
||||||
|
@operation("tail")
|
||||||
def tail(self, path, n):
|
def tail(self, path, n):
|
||||||
q = self.quote(path)
|
q = self.quote(path)
|
||||||
out = self.ps("$t = if (Test-Path -LiteralPath %s) { (Get-Content -LiteralPath %s -Encoding UTF8 -Tail %d) -join \"`n\" } else { '' }\n"
|
out = self.ps("$t = if (Test-Path -LiteralPath %s) { (Get-Content -LiteralPath %s -Encoding UTF8 -Tail %d) -join \"`n\" } else { '' }\n"
|
||||||
"'B64=' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))" % (q, q, n))
|
"'B64=' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))" % (q, q, n))
|
||||||
return _b64_payload(out, "B64") or ""
|
return _b64_payload(out, "B64") or ""
|
||||||
|
|
||||||
|
def fetch_script(self, run_dir, ops):
|
||||||
|
lines = ["$d = %s" % self.quote(run_dir), PS_FETCH.rstrip("\n")]
|
||||||
|
for i, o in enumerate(ops):
|
||||||
|
fn = {"read": "AdRead", "grep": "AdGrep", "tail": "AdTail"}[o[0]]
|
||||||
|
arg = "" if o[0] == "read" else " " + (self.quote(o[2]) if o[0] == "grep" else str(int(o[2])))
|
||||||
|
lines.append("%s %d %s%s" % (fn, i, self.quote(o[1]), arg))
|
||||||
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
@operation("fetch")
|
||||||
|
def fetch(self, run_dir, ops):
|
||||||
|
"""Several reads of one run directory in one call (collect)."""
|
||||||
|
script = self.fetch_script(run_dir, ops)
|
||||||
|
if not self.channel.stdin and not self.channel.fits(script):
|
||||||
|
return fetch_each(self, run_dir, ops)
|
||||||
|
res = parse_fetch(self.ps(script, timeout=200), ops, run_dir)
|
||||||
|
return [r.lstrip("\ufeff") if o[0] == "read" and r else r for o, r in zip(ops, res)]
|
||||||
|
|
||||||
# ---------------------------------------------------------------- processes
|
# ---------------------------------------------------------------- processes
|
||||||
def runner(self, run_dir, cwd, command, stdout_name):
|
def runner(self, run_dir, cwd, command, stdout_name):
|
||||||
return "run.ps1", RUNNER.format(cwd=self.quote(cwd), d=run_dir.replace("'", "''"), command=command, stdout=stdout_name)
|
return "run.ps1", RUNNER.format(cwd=self.quote(cwd), d=run_dir.replace("'", "''"), command=command, stdout=stdout_name)
|
||||||
@@ -176,23 +281,60 @@ class WindowsWarpgateTransport:
|
|||||||
"schtasks /Run /TN '%(t)s' | Out-Null\n"
|
"schtasks /Run /TN '%(t)s' | Out-Null\n"
|
||||||
"'LAUNCHED=' + $LASTEXITCODE" % dict(t=task.replace("'", "''"), s=script_path.replace("'", "''")))
|
"'LAUNCHED=' + $LASTEXITCODE" % dict(t=task.replace("'", "''"), s=script_path.replace("'", "''")))
|
||||||
|
|
||||||
|
@operation("launch")
|
||||||
def launch(self, task, script_path, run_dir):
|
def launch(self, task, script_path, run_dir):
|
||||||
out = self.ps(self.launch_script(task, script_path))
|
out = self.ps(self.launch_script(task, script_path))
|
||||||
if "LAUNCHED=0" not in out:
|
if "LAUNCHED=0" not in out:
|
||||||
raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:]))
|
raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:]))
|
||||||
|
|
||||||
|
def start_script(self, task, run_dir, files, script_name):
|
||||||
|
"""mkdir + clean (HASPID refusal) + every file + schtasks, as one script. -> (script, [sha256 per file])"""
|
||||||
|
script, shas = "$ErrorActionPreference = 'Stop'\n" + self.mkdir_script(run_dir, clean=True), []
|
||||||
|
for name, text in files.items():
|
||||||
|
data = text.encode("utf-8")
|
||||||
|
script += self.write_script(self.join(run_dir, name), data)
|
||||||
|
shas.append(hashlib.sha256(data).hexdigest())
|
||||||
|
script += "$ErrorActionPreference = 'Continue'\n" + self.launch_script(task, self.join(run_dir, script_name))
|
||||||
|
return script, shas
|
||||||
|
|
||||||
|
@operation("start")
|
||||||
|
def start_run(self, task, run_dir, files, script_name):
|
||||||
|
"""Stage the run directory and start the task: one call on the direct channel; on the relay (argv only)
|
||||||
|
mkdir, chunked uploads and the launch go one by one."""
|
||||||
|
if not self.channel.stdin:
|
||||||
|
self.mkdir(run_dir, clean=True)
|
||||||
|
for name, text in files.items():
|
||||||
|
self.write_text(self.join(run_dir, name), text)
|
||||||
|
return self.launch(task, self.join(run_dir, script_name), run_dir)
|
||||||
|
script, shas = self.start_script(task, run_dir, files, script_name)
|
||||||
|
out = self.ps(script)
|
||||||
|
if re.search(r"^HASPID$", out, re.M):
|
||||||
|
raise TransportError("refusing to clean %s: it holds a pid file (HASPID)" % run_dir)
|
||||||
|
wrote = re.findall(r"WROTE=([0-9a-f]{64})", out)
|
||||||
|
if wrote != shas:
|
||||||
|
raise TransportError("staging %s: hash mismatch (%d/%d files): %s" % (run_dir, len(wrote), len(shas),
|
||||||
|
out.strip()[-300:]))
|
||||||
|
if "LAUNCHED=0" not in out:
|
||||||
|
raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:]))
|
||||||
|
|
||||||
|
@operation("kill")
|
||||||
def kill(self, task, run_dir):
|
def kill(self, task, run_dir):
|
||||||
|
t = task.replace("'", "''")
|
||||||
self.ps("$p = Join-Path %s 'pid.txt'\n"
|
self.ps("$p = Join-Path %s 'pid.txt'\n"
|
||||||
"if (Test-Path $p) { taskkill /T /F /PID (Get-Content $p | Select-Object -First 1) 2>&1 | Out-Null }\n"
|
"if (Test-Path $p) { taskkill /T /F /PID (Get-Content $p | Select-Object -First 1) 2>&1 | Out-Null }\n"
|
||||||
"schtasks /End /TN '%s' 2>&1 | Out-Null\n"
|
"schtasks /End /TN '%s' 2>&1 | Out-Null\n"
|
||||||
"schtasks /Delete /F /TN '%s' 2>&1 | Out-Null\n'KILLED=ok'" % (self.quote(run_dir), task, task))
|
"schtasks /Delete /F /TN '%s' 2>&1 | Out-Null\n'KILLED=ok'" % (self.quote(run_dir), t, t))
|
||||||
|
|
||||||
|
@operation("forget")
|
||||||
def forget(self, task):
|
def forget(self, task):
|
||||||
self.ps("schtasks /Delete /F /TN '%s' 2>&1 | Out-Null; 'FORGOT=ok'" % task)
|
self.ps("schtasks /Delete /F /TN '%s' 2>&1 | Out-Null; 'FORGOT=ok'" % task.replace("'", "''"))
|
||||||
|
|
||||||
|
@operation("busy")
|
||||||
def process_count(self, name):
|
def process_count(self, name):
|
||||||
try:
|
try:
|
||||||
out = self.ps("'COUNT=' + @(Get-Process -Name %s -ErrorAction SilentlyContinue).Count" % self.quote(name))
|
out = self.ps("'COUNT=' + @(Get-Process -Name %s -ErrorAction SilentlyContinue).Count" % self.quote(name))
|
||||||
|
except Unreached:
|
||||||
|
raise # a direct-ssh terminal retries this on its fallback
|
||||||
except TransportError:
|
except TransportError:
|
||||||
return None
|
return None
|
||||||
m = re.search(r"COUNT=(\d+)", out)
|
m = re.search(r"COUNT=(\d+)", out)
|
||||||
@@ -210,15 +352,17 @@ DETACH = {
|
|||||||
|
|
||||||
|
|
||||||
class PosixWarpgateTransport:
|
class PosixWarpgateTransport:
|
||||||
"""Linux/macOS terminal through the relay. Each primitive is one `bash -c` round trip (~10 s through
|
"""Linux/macOS terminal. Each primitive is one `bash -c` round trip (~10 s through Vault + Warpgate, well under
|
||||||
Vault + Warpgate), so `start_run` stages the run directory and launches it in a single call."""
|
a second on a reused direct connection), so `start_run` stages the run directory and launches it in a single
|
||||||
CHUNK = 60000 # base64 chars per call: one argv element must stay below 128 KiB (MAX_ARG_STRLEN) on both ends
|
call and `fetch` reads everything `collect` needs in one."""
|
||||||
|
CHUNK = 60000 # relay only: base64 chars per call; one argv element must stay below 128 KiB (MAX_ARG_STRLEN)
|
||||||
|
|
||||||
def __init__(self, cfg, terminal):
|
def __init__(self, cfg, terminal, channel=None, log=None, label="warpgate"):
|
||||||
self.cfg, self.terminal = cfg, terminal
|
self.cfg, self.terminal = cfg, terminal
|
||||||
t = cfg.terminals[terminal]
|
t = cfg.terminals[terminal]
|
||||||
self.os = t["os"]
|
self.os = t["os"]
|
||||||
self.target = t.get("target", terminal)
|
self.target = t.get("target", terminal)
|
||||||
|
self.channel = channel or WarpgateChannel(cfg, terminal, log, label)
|
||||||
self.repo = t["repo"]
|
self.repo = t["repo"]
|
||||||
|
|
||||||
def default_run_root(self):
|
def default_run_root(self):
|
||||||
@@ -230,14 +374,19 @@ class PosixWarpgateTransport:
|
|||||||
def quote(self, s):
|
def quote(self, s):
|
||||||
return sh_quote(s)
|
return sh_quote(s)
|
||||||
|
|
||||||
# ---------------------------------------------------------------- relay primitive
|
# ---------------------------------------------------------------- channel primitive
|
||||||
def sh(self, script, timeout=150, marker=None):
|
def sh(self, script, timeout=150, marker=None):
|
||||||
"""Run a bash script on the target. With `marker`, its absence in stdout is an error."""
|
"""Run a bash script on the target. With `marker`, its absence in stdout is an error."""
|
||||||
rc, out, err = relay(self.cfg, self.target, ["bash", "-c", script], timeout)
|
rc, out, err = self.channel.sh(script, timeout, getattr(self, "_op", None))
|
||||||
if (marker and marker not in out) or (not marker and rc != 0 and not out.strip()):
|
if (marker and marker not in out) or (not marker and rc != 0 and not out.strip()):
|
||||||
raise TransportError("relay %s exit %s: %s" % (self.target, rc, (out + err).strip()[-300:]))
|
raise TransportError("%s exit %s: %s" % (self.channel.name, rc, (out + err).strip()[-300:]))
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
@operation("ping")
|
||||||
|
def ping(self):
|
||||||
|
out = self.sh("echo \"PONG=$(uname -sr) $(uname -n)\"", timeout=60, marker="PONG=")
|
||||||
|
return re.search(r"PONG=[^\n]*", out).group(0)
|
||||||
|
|
||||||
# ---------------------------------------------------------------- file ops
|
# ---------------------------------------------------------------- file ops
|
||||||
def mkdir_script(self, path, clean=False):
|
def mkdir_script(self, path, clean=False):
|
||||||
q = self.quote(path)
|
q = self.quote(path)
|
||||||
@@ -248,6 +397,7 @@ class PosixWarpgateTransport:
|
|||||||
s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q
|
s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q
|
||||||
return s
|
return s
|
||||||
|
|
||||||
|
@operation("mkdir")
|
||||||
def mkdir(self, path, clean=False):
|
def mkdir(self, path, clean=False):
|
||||||
self.sh(self.mkdir_script(path, clean) + "echo MKDIR=ok\n", marker="MKDIR=ok")
|
self.sh(self.mkdir_script(path, clean) + "echo MKDIR=ok\n", marker="MKDIR=ok")
|
||||||
|
|
||||||
@@ -261,10 +411,11 @@ class PosixWarpgateTransport:
|
|||||||
return ("printf %%s '%s' | base64 -d > %s && mv -f %s %s || exit 1\n"
|
return ("printf %%s '%s' | base64 -d > %s && mv -f %s %s || exit 1\n"
|
||||||
"echo \"WROTE=$(wc -c < %s | tr -d ' ')\"\n" % (b64, part, part, q, q))
|
"echo \"WROTE=$(wc -c < %s | tr -d ' ')\"\n" % (b64, part, part, q, q))
|
||||||
|
|
||||||
|
@operation("write")
|
||||||
def write_text(self, path, text):
|
def write_text(self, path, text):
|
||||||
data = self.b64(text)
|
data = self.b64(text)
|
||||||
size = len(text.encode("utf-8"))
|
size = len(text.encode("utf-8"))
|
||||||
if len(data) <= self.CHUNK:
|
if len(data) <= self.CHUNK or self.channel.stdin: # stdin carries the script: no argv limit
|
||||||
out = self.sh(self.put_script(path, data), marker="WROTE=")
|
out = self.sh(self.put_script(path, data), marker="WROTE=")
|
||||||
else: # append chunks to <path>.b64, decode on the last call
|
else: # append chunks to <path>.b64, decode on the last call
|
||||||
q, qb = self.quote(path), self.quote(path + ".b64")
|
q, qb = self.quote(path), self.quote(path + ".b64")
|
||||||
@@ -279,6 +430,7 @@ class PosixWarpgateTransport:
|
|||||||
if re.findall(r"WROTE=(\d+)", out) != [str(size)]:
|
if re.findall(r"WROTE=(\d+)", out) != [str(size)]:
|
||||||
raise TransportError("upload of %s: size mismatch (%s)" % (path, out.strip()[-200:]))
|
raise TransportError("upload of %s: size mismatch (%s)" % (path, out.strip()[-200:]))
|
||||||
|
|
||||||
|
@operation("read")
|
||||||
def read_text(self, path):
|
def read_text(self, path):
|
||||||
q = self.quote(path)
|
q = self.quote(path)
|
||||||
out = self.sh("if [ -f %s ]; then printf B64=; base64 < %s | tr -d '\\n'; echo; else echo MISSING; fi" % (q, q),
|
out = self.sh("if [ -f %s ]; then printf B64=; base64 < %s | tr -d '\\n'; echo; else echo MISSING; fi" % (q, q),
|
||||||
@@ -289,6 +441,7 @@ class PosixWarpgateTransport:
|
|||||||
raise TransportError("read %s: unexpected relay output" % path)
|
raise TransportError("read %s: unexpected relay output" % path)
|
||||||
return _b64_payload(out, "B64")
|
return _b64_payload(out, "B64")
|
||||||
|
|
||||||
|
@operation("probe")
|
||||||
def probe(self, run_dir, names):
|
def probe(self, run_dir, names):
|
||||||
d = self.quote(run_dir)
|
d = self.quote(run_dir)
|
||||||
out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo"
|
out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo"
|
||||||
@@ -302,13 +455,33 @@ class PosixWarpgateTransport:
|
|||||||
raise TransportError("relay %s: unexpected output" % self.target)
|
raise TransportError("relay %s: unexpected output" % self.target)
|
||||||
return text
|
return text
|
||||||
|
|
||||||
|
@operation("grep")
|
||||||
def grep_first(self, path, needle):
|
def grep_first(self, path, needle):
|
||||||
line = self._b64_cmd("grep -F -m1 -e %s %s" % (self.quote(needle), self.quote(path)))
|
line = self._b64_cmd("grep -F -m1 -e %s %s" % (self.quote(needle), self.quote(path)))
|
||||||
return line.rstrip("\n") or None
|
return line.rstrip("\n") or None
|
||||||
|
|
||||||
|
@operation("tail")
|
||||||
def tail(self, path, n):
|
def tail(self, path, n):
|
||||||
return self._b64_cmd("tail -n %d %s" % (int(n), self.quote(path))).rstrip("\n")
|
return self._b64_cmd("tail -n %d %s" % (int(n), self.quote(path))).rstrip("\n")
|
||||||
|
|
||||||
|
def fetch_script(self, run_dir, ops):
|
||||||
|
s = "d=%s\n" % self.quote(run_dir)
|
||||||
|
for i, o in enumerate(ops):
|
||||||
|
f = "\"$d\"/%s" % self.quote(o[1])
|
||||||
|
if o[0] == "read":
|
||||||
|
s += ("if [ -f %s ]; then printf F%d=; base64 < %s | tr -d '\\n'; echo; else echo M%d; fi\n"
|
||||||
|
% (f, i, f, i))
|
||||||
|
else:
|
||||||
|
cmd = ("grep -F -m1 -e %s %s" % (self.quote(o[2]), f) if o[0] == "grep"
|
||||||
|
else "tail -n %d %s" % (int(o[2]), f))
|
||||||
|
s += "printf F%d=; { %s; } 2>/dev/null | base64 | tr -d '\\n'; echo\n" % (i, cmd)
|
||||||
|
return s + "echo FETCH=ok\n"
|
||||||
|
|
||||||
|
@operation("fetch")
|
||||||
|
def fetch(self, run_dir, ops):
|
||||||
|
"""Several reads of one run directory in one call (collect)."""
|
||||||
|
return parse_fetch(self.sh(self.fetch_script(run_dir, ops), timeout=200, marker="FETCH=ok"), ops, run_dir)
|
||||||
|
|
||||||
# ---------------------------------------------------------------- processes
|
# ---------------------------------------------------------------- processes
|
||||||
def runner(self, run_dir, cwd, command, stdout_name):
|
def runner(self, run_dir, cwd, command, stdout_name):
|
||||||
return "run.sh", POSIX_RUNNER.format(cwd=self.quote(cwd), d=self.quote(run_dir), command=command,
|
return "run.sh", POSIX_RUNNER.format(cwd=self.quote(cwd), d=self.quote(run_dir), command=command,
|
||||||
@@ -323,13 +496,15 @@ class PosixWarpgateTransport:
|
|||||||
"echo $! > launch.pid\n"
|
"echo $! > launch.pid\n"
|
||||||
"echo LAUNCHED=$!\n" % (d, DETACH[self.os] % self.quote(script_name)))
|
"echo LAUNCHED=$!\n" % (d, DETACH[self.os] % self.quote(script_name)))
|
||||||
|
|
||||||
|
@operation("launch")
|
||||||
def launch(self, task, script_path, run_dir):
|
def launch(self, task, script_path, run_dir):
|
||||||
self.sh(self.launch_script(run_dir, posixpath.basename(script_path)), marker="LAUNCHED=")
|
self.sh(self.launch_script(run_dir, posixpath.basename(script_path)), marker="LAUNCHED=")
|
||||||
|
|
||||||
|
@operation("start")
|
||||||
def start_run(self, task, run_dir, files, script_name):
|
def start_run(self, task, run_dir, files, script_name):
|
||||||
"""mkdir + clean, write `files` {name: text}, launch `script_name` -- one relay call unless a file is
|
"""mkdir + clean, write `files` {name: text}, launch `script_name` -- one call; on the relay a file larger
|
||||||
larger than CHUNK (those go up first through write_text)."""
|
than CHUNK goes up first through write_text."""
|
||||||
big = {n: t for n, t in files.items() if len(self.b64(t)) > self.CHUNK}
|
big = {} if self.channel.stdin else {n: t for n, t in files.items() if len(self.b64(t)) > self.CHUNK}
|
||||||
script = self.mkdir_script(run_dir, clean=True)
|
script = self.mkdir_script(run_dir, clean=True)
|
||||||
if big:
|
if big:
|
||||||
self.mkdir(run_dir, clean=True)
|
self.mkdir(run_dir, clean=True)
|
||||||
@@ -359,23 +534,26 @@ class PosixWarpgateTransport:
|
|||||||
"kill -KILL -- \"-$p\" 2>/dev/null; pkill -KILL -s \"$p\" 2>/dev/null\n"
|
"kill -KILL -- \"-$p\" 2>/dev/null; pkill -KILL -s \"$p\" 2>/dev/null\n"
|
||||||
"sleep 1; echo \"KILLED=ok left=$(pgrep -s \"$p\" 2>/dev/null | wc -l | tr -d ' ')\"\n" % d)
|
"sleep 1; echo \"KILLED=ok left=$(pgrep -s \"$p\" 2>/dev/null | wc -l | tr -d ' ')\"\n" % d)
|
||||||
|
|
||||||
|
@operation("kill")
|
||||||
def kill(self, task, run_dir):
|
def kill(self, task, run_dir):
|
||||||
self.sh(self.kill_script(run_dir), marker="KILLED=")
|
self.sh(self.kill_script(run_dir), marker="KILLED=")
|
||||||
|
|
||||||
def forget(self, task):
|
def forget(self, task):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@operation("busy")
|
||||||
def process_count(self, name):
|
def process_count(self, name):
|
||||||
try:
|
try:
|
||||||
out = self.sh("echo COUNT=$(pgrep -x %s | wc -l | tr -d ' ')" % self.quote(name), timeout=60)
|
out = self.sh("echo COUNT=$(pgrep -x %s | wc -l | tr -d ' ')" % self.quote(name), timeout=60)
|
||||||
|
except Unreached:
|
||||||
|
raise # a direct-ssh terminal retries this on its fallback
|
||||||
except TransportError:
|
except TransportError:
|
||||||
return None
|
return None
|
||||||
m = re.search(r"COUNT=(\d+)", out)
|
m = re.search(r"COUNT=(\d+)", out)
|
||||||
return int(m.group(1)) if m else None
|
return int(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
def WarpgateSshTransport(cfg, terminal):
|
def WarpgateSshTransport(cfg, terminal, channel=None, log=None, label="warpgate"):
|
||||||
"""Factory: the transport class for a warpgate-ssh terminal follows its `os`."""
|
"""Factory: the transport class for a remote terminal follows its `os`; the channel defaults to Warpgate."""
|
||||||
if cfg.terminals[terminal]["os"] == "windows":
|
cls = WindowsWarpgateTransport if cfg.terminals[terminal]["os"] == "windows" else PosixWarpgateTransport
|
||||||
return WindowsWarpgateTransport(cfg, terminal)
|
return cls(cfg, terminal, channel=channel, log=log, label=label)
|
||||||
return PosixWarpgateTransport(cfg, terminal)
|
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ agentd:
|
|||||||
max_rounds_per_thread: 8 # rebuild the executor thread (with a handoff summary) after this many rounds
|
max_rounds_per_thread: 8 # rebuild the executor thread (with a handoff summary) after this many rounds
|
||||||
comment_max_chars: 6000
|
comment_max_chars: 6000
|
||||||
cli_terminal: oregon # identity used by `agentd new/comment/state/assign`
|
cli_terminal: oregon # identity used by `agentd new/comment/state/assign`
|
||||||
|
# ssh_key: AGENTD_SSH_KEY # direct-ssh: Vault key / env var of the dedicated private key (default)
|
||||||
|
# ssh_dir: /run/plane-agentd/ssh # direct-ssh: ControlMaster sockets + generated known_hosts (default state_dir/ssh)
|
||||||
|
|
||||||
# Plane webhook receiver (`agentd webhook-serve`, docs/runbook.md "Webhook"); leave the section out to run on cron only.
|
# Plane webhook receiver (`agentd webhook-serve`, docs/runbook.md "Webhook"); leave the section out to run on cron only.
|
||||||
webhook:
|
webhook:
|
||||||
@@ -27,7 +29,10 @@ notify: {kind: none} # notification hook: only `none` (no-op) e
|
|||||||
|
|
||||||
terminals:
|
terminals:
|
||||||
win10:
|
win10:
|
||||||
transport: warpgate-ssh
|
transport: warpgate-ssh # or direct-ssh over Tailscale (design §9, docs/deploy-lxc.md "直连 SSH"):
|
||||||
|
# transport: direct-ssh
|
||||||
|
# ssh: {host: 100.123.209.19, user: administrator, port: 22, host_key: "ssh-ed25519 AAAA..."} # pinned host key
|
||||||
|
# fallback: warpgate-ssh # used only when the direct path fails before the command ran
|
||||||
target: win10
|
target: win10
|
||||||
os: windows
|
os: windows
|
||||||
launch: schtasks
|
launch: schtasks
|
||||||
|
|||||||
+19
-7
@@ -1,5 +1,5 @@
|
|||||||
# plane-agentd live configuration on the LXC agentd-01 (/opt/plane-agentd, runs as root; docs/deploy-lxc.md).
|
# plane-agentd live configuration on the LXC agentd-01 (/opt/plane-agentd, runs as root; docs/deploy-lxc.md).
|
||||||
# No secrets: token_key values are Vault key names; the tokens are injected by
|
# No secrets: token_key / ssh_key values are Vault key names; the values are injected by
|
||||||
# `agent-access.py exec --secret-env KEY=KEY` (cron/agentd-tick-lxc.sh). Rollback to oregon: config/agentd.oregon.yaml.
|
# `agent-access.py exec --secret-env KEY=KEY` (cron/agentd-tick-lxc.sh). Rollback to oregon: config/agentd.oregon.yaml.
|
||||||
plane: {base_url: https://plane.kappiee.com, workspace: kappiee}
|
plane: {base_url: https://plane.kappiee.com, workspace: kappiee}
|
||||||
vault: {client: ~/.local/share/server-access-client/agent-access.py, config: https://nas.tail9531b.ts.net:8443/identity/agent.json}
|
vault: {client: ~/.local/share/server-access-client/agent-access.py, config: https://nas.tail9531b.ts.net:8443/identity/agent.json}
|
||||||
@@ -12,11 +12,15 @@ agentd:
|
|||||||
max_rounds_per_thread: 8
|
max_rounds_per_thread: 8
|
||||||
comment_max_chars: 6000
|
comment_max_chars: 6000
|
||||||
cli_terminal: oregon
|
cli_terminal: oregon
|
||||||
|
# direct-ssh (PAD-12, docs/deploy-lxc.md "直连 SSH"): the dedicated key comes from Vault AGENTD_SSH_KEY via
|
||||||
|
# --secret-env; ControlMaster sockets and the known_hosts generated from the pinned host keys live in ssh_dir
|
||||||
|
ssh_key: AGENTD_SSH_KEY
|
||||||
|
ssh_dir: /run/plane-agentd/ssh
|
||||||
|
|
||||||
# Plane webhook receiver (`agentd webhook-serve`, systemd plane-agentd-webhook.service; docs/runbook.md "Webhook").
|
# Plane webhook receiver (`agentd webhook-serve`, systemd plane-agentd-webhook.service; docs/runbook.md "Webhook").
|
||||||
# One workspace-level Plane webhook (work items + comments) -> POST http://10.77.10.249:8790/plane -> one tick.
|
# One workspace-level Plane webhook (work items + comments) -> POST http://10.77.10.249:8790/plane -> one tick.
|
||||||
webhook:
|
webhook:
|
||||||
bind: 10.77.10.249 # LAN only (Plane VM 10.77.10.68 is on the same /24)
|
bind: 10.77.10.249 # LAN only (Plane VM 506 10.77.10.69 is on the same /24)
|
||||||
port: 8790
|
port: 8790
|
||||||
path: /plane
|
path: /plane
|
||||||
secret_key: PLANE_WEBHOOK_SECRET # Vault key name; the value is injected by agent-access.py exec --secret-env
|
secret_key: PLANE_WEBHOOK_SECRET # Vault key name; the value is injected by agent-access.py exec --secret-env
|
||||||
@@ -30,8 +34,13 @@ notify: {kind: none} # notification hook, no-op (no ntfy servic
|
|||||||
|
|
||||||
terminals:
|
terminals:
|
||||||
win10:
|
win10:
|
||||||
transport: warpgate-ssh
|
# direct over Tailscale (PAD-12); Warpgate only when the direct path fails before a command ran.
|
||||||
target: win10
|
# Rollback: transport: warpgate-ssh (ssh/fallback may stay, they are ignored then)
|
||||||
|
transport: direct-ssh
|
||||||
|
ssh: {host: 100.123.209.19, user: administrator, port: 22,
|
||||||
|
host_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzLX7QNHDFYwTgY4wNl0ihZryIyKgpp9HPpPDWbrTX3"}
|
||||||
|
fallback: warpgate-ssh
|
||||||
|
target: win10 # Warpgate target of the fallback
|
||||||
os: windows
|
os: windows
|
||||||
launch: schtasks
|
launch: schtasks
|
||||||
repo: 'C:\Users\Administrator\Documents\GwentAssistantProject\gwent'
|
repo: 'C:\Users\Administrator\Documents\GwentAssistantProject\gwent'
|
||||||
@@ -44,9 +53,12 @@ terminals:
|
|||||||
capabilities: [android, game]
|
capabilities: [android, game]
|
||||||
executors: [codex]
|
executors: [codex]
|
||||||
oregon:
|
oregon:
|
||||||
# the dispatcher (LXC, no agent CLIs) reaches oregon through Warpgate like any other host (design §9)
|
# the dispatcher (LXC, no agent CLIs) reaches oregon directly over Tailscale, Warpgate as fallback (design §9)
|
||||||
transport: warpgate-ssh
|
transport: direct-ssh
|
||||||
target: oregon # agent-access.py target name
|
ssh: {host: 100.127.58.78, user: ubuntu, port: 22,
|
||||||
|
host_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICUoI6W7npX+h/kHcXX3iy2XAWbNusrOjg28Xed75wWr"}
|
||||||
|
fallback: warpgate-ssh
|
||||||
|
target: oregon # agent-access.py target name (fallback)
|
||||||
os: linux
|
os: linux
|
||||||
launch: nohup
|
launch: nohup
|
||||||
repo: /home/ubuntu/Documents/wjj/project/gwent
|
repo: /home/ubuntu/Documents/wjj/project/gwent
|
||||||
|
|||||||
@@ -44,4 +44,5 @@ worker 跑在 Claude 所在的 Linux 机(`gwent-design/tools/board-worker-plan
|
|||||||
## 通用规则
|
## 通用规则
|
||||||
- 消息要短、要有结论;路径用仓库相对路径;大图只贴缩略拼图(≤1600px)。
|
- 消息要短、要有结论;路径用仓库相对路径;大图只贴缩略拼图(≤1600px)。
|
||||||
- 不在评论里贴 token、密码、完整日志。
|
- 不在评论里贴 token、密码、完整日志。
|
||||||
|
- 不在 win10 的 checkout 上执行任何 git 写操作(commit / checkout / reset / pull / stash);产物留在 live 或 stage,入库由 Claude 侧完成。
|
||||||
- 有疑问优先按任务书里的默认值继续并在评论里说明,而不是停下等人。
|
- 有疑问优先按任务书里的默认值继续并在评论里说明,而不是停下等人。
|
||||||
|
|||||||
+16
-2
@@ -33,7 +33,9 @@
|
|||||||
"interval": {"type": "integer", "minimum": 10},
|
"interval": {"type": "integer", "minimum": 10},
|
||||||
"max_rounds_per_thread": {"type": "integer", "minimum": 1},
|
"max_rounds_per_thread": {"type": "integer", "minimum": 1},
|
||||||
"comment_max_chars": {"type": "integer", "minimum": 500},
|
"comment_max_chars": {"type": "integer", "minimum": 500},
|
||||||
"cli_terminal": {"type": "string"}
|
"cli_terminal": {"type": "string"},
|
||||||
|
"ssh_dir": {"type": "string"},
|
||||||
|
"ssh_key": {"type": "string"}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"webhook": {
|
"webhook": {
|
||||||
@@ -71,8 +73,20 @@
|
|||||||
"required": ["transport", "os", "launch", "repo", "plane_user", "token_key", "executors"],
|
"required": ["transport", "os", "launch", "repo", "plane_user", "token_key", "executors"],
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
"transport": {"enum": ["local", "warpgate-ssh"]},
|
"transport": {"enum": ["local", "warpgate-ssh", "direct-ssh"]},
|
||||||
"target": {"type": "string"},
|
"target": {"type": "string"},
|
||||||
|
"ssh": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["host", "user", "host_key"],
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"host": {"type": "string"},
|
||||||
|
"user": {"type": "string"},
|
||||||
|
"port": {"type": "integer", "minimum": 1},
|
||||||
|
"host_key": {"type": "string"}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"fallback": {"enum": ["warpgate-ssh"]},
|
||||||
"os": {"enum": ["linux", "darwin", "windows"]},
|
"os": {"enum": ["linux", "darwin", "windows"]},
|
||||||
"launch": {"enum": ["subprocess", "nohup", "schtasks"]},
|
"launch": {"enum": ["subprocess", "nohup", "schtasks"]},
|
||||||
"repo": {"type": "string"},
|
"repo": {"type": "string"},
|
||||||
|
|||||||
+31
-7
@@ -1,13 +1,37 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# LXC agentd-01 cron entry (*/2, root): one agentd tick under server-access exec (Plane tokens only in the
|
# LXC agentd-01 cron entry (*/2, root): one agentd tick under server-access exec (Plane tokens only in the
|
||||||
# process env), with a lock. The outer flock skips a tick while the previous one still runs; agentd also holds
|
# process env), with a lock. The flock skips a tick while the previous one still runs; agentd also holds
|
||||||
# its own state lock. Logs: /var/log/plane-agentd/{agentd.log,cron.log}; state: /var/lib/plane-agentd.
|
# its own state lock. Logs: /var/log/plane-agentd/{agentd.log,cron.log}; state: /var/lib/plane-agentd.
|
||||||
# Also the webhook receiver's tick command (agentd/webhook.py): exit 75 = lock held (flock -E 75, or agentd's own
|
# Also the webhook receiver's tick command (agentd/webhook.py): exit 75 = lock held (flock -E 75, or agentd's own
|
||||||
# state lock), the receiver then retries.
|
# state lock), the receiver then retries.
|
||||||
exec flock -n -E 75 /var/lib/plane-agentd/cron.lock \
|
# AGENTD_SSH_KEY = the direct-ssh key (PAD-12). agent-access.py has no optional secrets: when Vault cannot hand it
|
||||||
timeout 900 python3 -I /root/.local/share/server-access-client/agent-access.py \
|
# out (credential_unavailable / credential_invalid, nothing ran) the tick is repeated once without it, so the
|
||||||
--config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
|
# direct-ssh terminals use their warpgate fallback instead of every tick failing. The tick's stderr is appended to
|
||||||
|
# cron.log after its stdout (it has to be inspected for that error first).
|
||||||
|
LOCK=/var/lib/plane-agentd/cron.lock
|
||||||
|
LOG=/var/log/plane-agentd/cron.log
|
||||||
|
CLIENT=(python3 -I /root/.local/share/server-access-client/agent-access.py --config https://nas.tail9531b.ts.net:8443/identity/agent.json)
|
||||||
|
|
||||||
|
exec 9>>"$LOCK"
|
||||||
|
flock -n -E 75 9 || exit $?
|
||||||
|
|
||||||
|
tick() { # "$@" = extra exec arguments; fd 9 (the lock) is not handed to the tick's children
|
||||||
|
timeout 900 "${CLIENT[@]}" exec \
|
||||||
--secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \
|
--secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \
|
||||||
--secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN --timeout 850 -- \
|
--secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN \
|
||||||
python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml \
|
"$@" --timeout 850 -- \
|
||||||
tick --once >> /var/log/plane-agentd/cron.log 2>&1
|
python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once 9>&-
|
||||||
|
}
|
||||||
|
|
||||||
|
err=$(mktemp) || exit 1
|
||||||
|
trap 'rm -f "$err"' EXIT
|
||||||
|
tick --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY >>"$LOG" 2>"$err"
|
||||||
|
rc=$?
|
||||||
|
cat "$err" >>"$LOG"
|
||||||
|
case "$rc:$(cat "$err")" in
|
||||||
|
"1:access-hub: credential_unavailable" | "1:access-hub: credential_invalid")
|
||||||
|
echo "$(date -Is) AGENTD_SSH_KEY not available from Vault: tick repeated without it (direct-ssh terminals use their warpgate fallback)" >>"$LOG"
|
||||||
|
tick >>"$LOG" 2>&1
|
||||||
|
rc=$? ;;
|
||||||
|
esac
|
||||||
|
exit $rc
|
||||||
|
|||||||
+78
-5
@@ -1,6 +1,6 @@
|
|||||||
# 在精简 LXC 上部署 agentd(design §9 阶段 3)
|
# 在精简 LXC 上部署 agentd(design §9 阶段 3)
|
||||||
|
|
||||||
调度器跑在一台**不装任何 agent CLI** 的 Debian 12 LXC(`agentd-01`)上;所有终端(win10、oregon、mac…)都经 Warpgate SSH 远程启动(`transport: warpgate-ssh`)。LXC 只做调度、回帖、通知。
|
调度器跑在一台**不装任何 agent CLI** 的 Debian 12 LXC(`agentd-01`)上;所有终端(win10、oregon、mac…)都是远程启动:经 Warpgate SSH(`transport: warpgate-ssh`),或经 Tailscale 直连并以 Warpgate 兜底(`transport: direct-ssh`,PAD-12,§9)。LXC 只做调度、回帖、通知。
|
||||||
|
|
||||||
实际部署(2026-10-21,PAD-11):`agentd-01` = PVE CT 600,Debian 12,tailnet `100.69.78.124`,Warpgate target `agentd-01`,登录 **root**(Vault skill `ssh-agentd-01`;没有另建部署用户,agentd 以 root 运行)。下面是实际做过的步骤;从 oregon 发命令都经 `agent-access.py … run --target agentd-01 --timeout N -- bash -c '…'`。
|
实际部署(2026-10-21,PAD-11):`agentd-01` = PVE CT 600,Debian 12,tailnet `100.69.78.124`,Warpgate target `agentd-01`,登录 **root**(Vault skill `ssh-agentd-01`;没有另建部署用户,agentd 以 root 运行)。下面是实际做过的步骤;从 oregon 发命令都经 `agent-access.py … run --target agentd-01 --timeout N -- bash -c '…'`。
|
||||||
|
|
||||||
@@ -9,17 +9,25 @@ LXC 交付时已有 `python3`(3.11)、`python3-yaml`、`openssh-client`、`g
|
|||||||
```bash
|
```bash
|
||||||
apt-get update && apt-get install -y python3 python3-yaml openssh-client git cron ca-certificates
|
apt-get update && apt-get install -y python3 python3-yaml openssh-client git cron ca-certificates
|
||||||
```
|
```
|
||||||
`jsonschema` 可选,没有就用内置子集校验器。
|
`jsonschema` 可选,没有就用内置子集校验器。`direct-ssh` 要用的 `ssh`、`ssh-agent`、`ssh-add`、`ssh-keygen` 都在 `openssh-client` 包里(Debian 12:OpenSSH 9.2)。
|
||||||
|
|
||||||
## 2. 网络
|
## 2. 网络
|
||||||
| 去向 | 用途 |
|
| 去向 | 用途 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `nas.tail9531b.ts.net:8443`(tailnet) | Agent Vault 身份配置 `identity/agent.json`(含 vault token 与 Warpgate 主机公钥)→ LXC 已在 tailnet |
|
| `nas.tail9531b.ts.net:8443`(tailnet) | Agent Vault 身份配置 `identity/agent.json`(含 vault token 与 Warpgate 主机公钥)→ LXC 已在 tailnet |
|
||||||
| Vault(身份配置里的地址) | 取 Plane token、Warpgate 密码 |
|
| Vault(身份配置里的地址) | 取 Plane token、Warpgate 密码 |
|
||||||
| Warpgate SSH `115.28.210.101:2222` | 所有终端的中继(LXC → win10 / oregon) |
|
| Warpgate SSH `115.28.210.101:2222` | `warpgate-ssh` 终端的中继,以及 `direct-ssh` 的兜底(LXC → win10 / oregon) |
|
||||||
|
| tailnet:win10 `100.123.209.19:22`、oregon `100.127.58.78:22` | `direct-ssh` 直连(PAD-12,§9) |
|
||||||
| `https://plane.kappiee.com` | Plane API |
|
| `https://plane.kappiee.com` | Plane API |
|
||||||
| `https://git.kappiee.com` | `git pull`(公开仓库,无需凭据) |
|
| `https://git.kappiee.com` | `git pull`(公开仓库,无需凭据) |
|
||||||
|
|
||||||
|
## 2.1 到 NAS 的直连路径(OPNsense 静态端口 NAT,2026-10-21)
|
||||||
|
PVE 站点是双层 NAT(OPNsense → 192.168.1.1),默认的随机源端口会让 tailscale 打洞失败、只能走 DERP(此前取 vault 身份端点 30/30 超时)。修法:
|
||||||
|
- **OPNsense 出站 NAT 规则**(用户在 GUI 里添加,因为 opnsense 中继账号 kappiee 无权限,我们只读):描述 `Tailscale agentd-01 static-port`,接口 WAN,源 `10.77.10.249/32`,目标 any,协议 UDP,勾选"静态端口"(`staticnatport 1`)。从 kpe-edge-01 的 `Tailscale edge static-port`(10.77.10.110/32)规则克隆而来。加规则后要清掉旧状态(Firewall → Diagnostics → States,按源 IP)。
|
||||||
|
- **LXC tailscaled 用 UDP 41642**:`/etc/default/tailscaled` 里 `PORT="41642"`(备份 `.bak-20261021144838`),必须与 edge 的 41641 不同,否则两台机器的静态端口映射会在同一 NAS 端点上撞元组。
|
||||||
|
- **验证**(LXC 上):`tailscale ping 100.89.188.28` 应得 `pong from nas … via 223.102.82.206:1549`(不是 `via DERP`);`tailscale netcheck` 应 `UDP: true`、`MappingVariesByDestIP: false`;身份端点循环(注意 `agent.json` 含 bearer token,只看状态码和耗时):`for i in $(seq 30); do curl -s -o /dev/null -w '%{http_code} %{time_total}\n' --max-time 15 https://nas.tail9531b.ts.net:8443/identity/agent.json; done`。2026-10-21 实测 30/30 成功、中位数 0.089 s,ping 5 ms。
|
||||||
|
- **回滚**:在 OPNsense GUI 删掉该规则(Firewall → NAT → Outbound)并清状态;LXC 上还原 `/etc/default/tailscaled.bak-20261021144838` 后 `systemctl restart tailscaled`。回滚后路径退回 DERP,调度器应同时切回 oregon(runbook)。
|
||||||
|
|
||||||
## 3. Agent Vault 客户端
|
## 3. Agent Vault 客户端
|
||||||
只复制程序文件(客户端没有落盘的 token 缓存;oregon 目录里的 `agent-service-check.py` 是运维工具,不需要):
|
只复制程序文件(客户端没有落盘的 token 缓存;oregon 目录里的 `agent-service-check.py` 是运维工具,不需要):
|
||||||
```bash
|
```bash
|
||||||
@@ -41,7 +49,7 @@ mkdir -p /var/lib/plane-agentd /var/log/plane-agentd && chmod 700 /var/lib/plane
|
|||||||
python3 -m unittest && python3 cli/agentd check-config
|
python3 -m unittest && python3 cli/agentd check-config
|
||||||
```
|
```
|
||||||
不需要本地配置文件,全部在仓库里(部署可复现):
|
不需要本地配置文件,全部在仓库里(部署可复现):
|
||||||
- `config/agentd.yaml` = agentd-01 的实时配置:`state_dir: /var/lib/plane-agentd`(状态、心跳、锁),`log_dir: /var/log/plane-agentd`(`agentd.log`),`helpers_dir: tools/win10`,`vault.client: ~/.local/share/server-access-client/agent-access.py`(root 下即 `/root/...`)。终端 win10(warpgate-ssh/windows/schtasks)和 oregon(warpgate-ssh/linux/nohup,`run_root: /home/ubuntu/.gwent-board/agentd-runs`,`bins.claude`);没有 local 终端。
|
- `config/agentd.yaml` = agentd-01 的实时配置:`state_dir: /var/lib/plane-agentd`(状态、心跳、锁),`log_dir: /var/log/plane-agentd`(`agentd.log`),`helpers_dir: tools/win10`,`vault.client: ~/.local/share/server-access-client/agent-access.py`(root 下即 `/root/...`)。终端 win10(direct-ssh + 兜底 warpgate-ssh/windows/schtasks)和 oregon(direct-ssh + 兜底 warpgate-ssh/linux/nohup,`run_root: /home/ubuntu/.gwent-board/agentd-runs`,`bins.claude`);没有 local 终端。`agentd.ssh_key: AGENTD_SSH_KEY`,`agentd.ssh_dir: /run/plane-agentd/ssh`(ControlMaster 套接字和生成的 known_hosts,agentd 自己建,权限 0700)。
|
||||||
- `tools/win10/win10-ps.sh`、`win10-put.sh`:win10 helper(vault 客户端目录取 `AGENT_ACCESS_HOME`,默认 `~/.local/share/server-access-client`;身份取 `AGENT_ACCESS_CONFIG`)。相对 `helpers_dir` 按仓库根解析。
|
- `tools/win10/win10-ps.sh`、`win10-put.sh`:win10 helper(vault 客户端目录取 `AGENT_ACCESS_HOME`,默认 `~/.local/share/server-access-client`;身份取 `AGENT_ACCESS_CONFIG`)。相对 `helpers_dir` 按仓库根解析。
|
||||||
- `config/protocols/gwent-board-protocol.md`:看板协议(拼进每轮 prompt,LXC 本机读取)。来源是 gwent 仓库 `design/board-protocol.md`,**协议更新时复制过来提交**,然后 LXC `git pull`。
|
- `config/protocols/gwent-board-protocol.md`:看板协议(拼进每轮 prompt,LXC 本机读取)。来源是 gwent 仓库 `design/board-protocol.md`,**协议更新时复制过来提交**,然后 LXC `git pull`。
|
||||||
- `config/agentd.oregon.yaml`:oregon 作调度器时的配置(回滚用,见 runbook)。
|
- `config/agentd.oregon.yaml`:oregon 作调度器时的配置(回滚用,见 runbook)。
|
||||||
@@ -60,7 +68,7 @@ python3 -m unittest && python3 cli/agentd check-config
|
|||||||
```
|
```
|
||||||
*/2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh
|
*/2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh
|
||||||
```
|
```
|
||||||
`cron/agentd-tick-lxc.sh` = `flock -n /var/lib/plane-agentd/cron.lock timeout 900 python3 -I /root/.local/share/server-access-client/agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once >> /var/log/plane-agentd/cron.log 2>&1`。
|
`cron/agentd-tick-lxc.sh` = 拿 `/var/lib/plane-agentd/cron.lock`(`flock -n -E 75`)后执行 `timeout 900 python3 -I /root/.local/share/server-access-client/agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once`,输出追加到 `/var/log/plane-agentd/cron.log`;Vault 取不到 `AGENTD_SSH_KEY` 时不带它重跑一次(§9)。
|
||||||
注意:经中继装 crontab 时别在 `set -e` 下写 `(crontab -l | grep -v …; echo …) | crontab -`——没有旧 crontab 时 grep 返回 1,子 shell 提前退出,装进去的是空 crontab。直接 `printf '%s\n' '<行>' | crontab -`。
|
注意:经中继装 crontab 时别在 `set -e` 下写 `(crontab -l | grep -v …; echo …) | crontab -`——没有旧 crontab 时 grep 返回 1,子 shell 提前退出,装进去的是空 crontab。直接 `printf '%s\n' '<行>' | crontab -`。
|
||||||
5. 验证:`/var/lib/plane-agentd/heartbeat.json` 的 `at` 每 2 分钟更新。
|
5. 验证:`/var/lib/plane-agentd/heartbeat.json` 的 `at` 每 2 分钟更新。
|
||||||
6. 冒烟经 LXC 调度器:在 oregon 上 `cli/agentd smoke GWENT oregon --watch`(`--watch` = 只建项并等待,不在本机 tick;oregon 上 `cli/agentd` 默认读 `config/agentd.yaml`,CLI 只需要 Plane token)。
|
6. 冒烟经 LXC 调度器:在 oregon 上 `cli/agentd smoke GWENT oregon --watch`(`--watch` = 只建项并等待,不在本机 tick;oregon 上 `cli/agentd` 默认读 `config/agentd.yaml`,CLI 只需要 Plane token)。
|
||||||
@@ -100,3 +108,68 @@ python3 -m unittest && python3 cli/agentd check-config
|
|||||||
远程 shell 是目标账户的登录 shell(bash/zsh 均可,agentd 发的是 `bash -c '<script>'`);终端需有 `bash base64 nohup pgrep pkill`(Linux 另需 `setsid`,macOS 需 `perl`)。
|
远程 shell 是目标账户的登录 shell(bash/zsh 均可,agentd 发的是 `bash -c '<script>'`);终端需有 `bash base64 nohup pgrep pkill`(Linux 另需 `setsid`,macOS 需 `perl`)。
|
||||||
4. Plane:建用户 `mac`,token 存 Vault 键 `PLANE_MAC_API_TOKEN`;项目 `routing.by_assignee` 加 `mac: mac`,`default_executor` 加 `mac: claude-code`;`cron/agentd-tick-lxc.sh` 加 `--secret-env PLANE_MAC_API_TOKEN=PLANE_MAC_API_TOKEN`。
|
4. Plane:建用户 `mac`,token 存 Vault 键 `PLANE_MAC_API_TOKEN`;项目 `routing.by_assignee` 加 `mac: mac`,`default_executor` 加 `mac: claude-code`;`cron/agentd-tick-lxc.sh` 加 `--secret-env PLANE_MAC_API_TOKEN=PLANE_MAC_API_TOKEN`。
|
||||||
5. 提交、推送、LXC `git pull`;`cli/agentd check-config`,然后 `cli/agentd smoke GWENT mac --watch`。
|
5. 提交、推送、LXC `git pull`;`cli/agentd check-config`,然后 `cli/agentd smoke GWENT mac --watch`。
|
||||||
|
|
||||||
|
## 8. Webhook 接收端(PAD-7)
|
||||||
|
1. 网络:Plane VM(PVE VM 506 plane-02,10.77.10.69,prime-cli 安装;没有 VM 505)与 LXC 10.77.10.249 同在 10.77.10.0/24,接收端只听 `10.77.10.249:8790`(LXC 内无防火墙)。**Plane 侧前提**:Plane 默认拒绝内网 webhook 地址("The webhook URL is not allowed. Provide a publicly reachable HTTP or HTTPS URL."),要在 Plane 的 api(创建时检查)与 worker(每次投递检查)容器环境里设 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(逗号分隔 IP/CIDR)并重建相关容器(design §10)。Plane 3.3.3 没有 compose override 文件:prime-cli 由 `/opt/plane/plane.env` 渲染 `/var/plane/docker-compose.yml`,两处都已有空的 `WEBHOOK_ALLOWED_IPS`(compose 里每个服务一行 `WEBHOOK_ALLOWED_IPS: ""`)。做法:备份两个文件为 `.bak-pad7-<时间戳>` → 把 `plane.env` 的 `WEBHOOK_ALLOWED_IPS=` 与渲染 compose 里的空值改为 `10.77.10.249/32` → `docker compose -p plane -f /var/plane/docker-compose.yml up -d --no-deps api worker webhook-consumer outbox-poller automation-consumer beat-worker`(先 `--dry-run`,确认只重建这 6 个;商业版的 v2 投递走 webhook-consumer/worker,创建时校验在 api;不要用 prime-cli,它要 TTY。compose 里 `WEBHOOK_ALLOWED_HOSTS: ${WEB_URL}` 不带 env 文件时为空,prime-cli 起的容器也是空值,属现状)。**状态(2026-10-22):已应用**(备份 `*.bak-pad7-20261022-011453`,6 个容器 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`),webhook `c039c471-e78a-49c9-b009-5cf77a389ad6` 已建并启用(v2;签名与接收端一致,见 design §10),接收端 active/enabled。纯 http 本身是允许的(http/https 都行)。
|
||||||
|
2. 密钥:Plane 建 webhook 时生成、只显示一次 → `tools/plane-webhook.py create http://10.77.10.249:8790/plane` 直接写进 Vault `PLANE_WEBHOOK_SECRET`(建出来是停用状态;在 oregon 上以 kappiee 管理员会话运行,见 runbook "Webhook")。核对长度:`agent-access.py … exec --secret-env S=PLANE_WEBHOOK_SECRET -- python3 -c 'import os;print(len(os.environ["S"]))'`。
|
||||||
|
3. 代码与服务(LXC):
|
||||||
|
```bash
|
||||||
|
cd /opt/plane-agentd && flock /var/lib/plane-agentd/cron.lock git pull
|
||||||
|
cp systemd/plane-agentd-webhook.service /etc/systemd/system/ && systemctl daemon-reload
|
||||||
|
systemctl enable --now plane-agentd-webhook
|
||||||
|
curl -s http://10.77.10.249:8790/healthz # {"ok": true, ...}
|
||||||
|
tail /var/log/plane-agentd/webhook.log # listening on http://10.77.10.249:8790/plane (tenants: GWENT)
|
||||||
|
```
|
||||||
|
单元文件复用 cron 脚本的 vault 客户端调用方式:`python3 -I /root/.local/share/server-access-client/agent-access.py --config <identity> exec --secret-env PLANE_WEBHOOK_SECRET=PLANE_WEBHOOK_SECRET --timeout 315360000 -- python3 /opt/plane-agentd/cli/agentd … webhook-serve --bind 10.77.10.249 --port 8790`,`Restart=always`。tick 由接收端经 `systemd-run --scope … cron/agentd-tick-lxc.sh` 拉起(同一把 `cron.lock`,`flock -n -E 75`)。
|
||||||
|
4. 启用 Plane 侧:`tools/plane-webhook.py patch <id> is_active=true`,然后 `systemctl restart plane-agentd-webhook`(若第 2 步之后密钥变过)。
|
||||||
|
5. 验证:随便改一个 GWENT 工作项 → `webhook.log` 出现 `decision=tick` 和 `tick start`/`tick done`;错签名请求得 401 且没有 tick;`journalctl -u cron` 里 cron 仍每 2 分钟一次;`systemctl restart plane-agentd-webhook` 后 `/healthz` 恢复。
|
||||||
|
|
||||||
|
## 9. 直连 SSH(PAD-12)
|
||||||
|
LXC 经 Tailscale 直连终端的 sshd,不经过 Warpgate(design §9"终端通道")。每个终端单独配置,Warpgate 留作兜底。前提:终端在 tailnet 上,LXC(`100.69.78.124`)能连到它的 22 端口。实测:win10 `100.123.209.19`(Windows OpenSSH,账号 administrator,直连 RTT 27 ms),oregon `100.127.58.78`(账号 ubuntu,RTT 233 ms)。
|
||||||
|
|
||||||
|
**上线顺序**:`cron/agentd-tick-lxc.sh` 带着 `--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY`。agent-access.py 没有可选密钥,取不到键时报 `credential_unavailable`,整个 exec 失败;cron 脚本认出这种情况后会去掉这个参数再跑一次(cron.log 记一行 `AGENTD_SSH_KEY not available from Vault …`),所以先 pull 也不会停派单,只是直连终端一直走兜底。仍然建议第 1–3 步做完再 pull,免得每个 tick 多取一次 Vault。
|
||||||
|
|
||||||
|
1. **生成专用密钥**(在 LXC 上;私钥只进 Vault,不留在磁盘上):
|
||||||
|
```bash
|
||||||
|
umask 077; d=$(mktemp -d -p /dev/shm) # tmpfs:私钥不落到磁盘(磁盘上 shred 在日志式/写时复制文件系统上不可靠)
|
||||||
|
ssh-keygen -q -t ed25519 -N '' -C plane-agentd@agentd-01 -f "$d/agentd"
|
||||||
|
cat "$d/agentd.pub" # 公钥,下面两步要用
|
||||||
|
```
|
||||||
|
私钥 `$d/agentd` 原样(多行文本)写进 Vault 键 **`AGENTD_SSH_KEY`**:在 Vault UI 里粘贴,或者用与 `tools/plane-webhook.py` 的 `vault_store()` 相同的接口(`POST /v1/credentials`,agent 身份),读文件直接写,不打印。值里的换行被转义成 `\n` 也可以,agentd 会还原。写完删掉:`rm -rf "$d"`(在 tmpfs 上,删掉即可)。核对时只看长度:
|
||||||
|
`agent-access.py --config <identity> exec --secret-env K=AGENTD_SSH_KEY --timeout 60 -- python3 -c 'import os;print(len(os.environ["K"]))'`(ed25519 私钥约 400 字符)。
|
||||||
|
2. **oregon**(经 Warpgate 操作):在 `~ubuntu/.ssh/authorized_keys` 追加一行(目录 700、文件 600):
|
||||||
|
```
|
||||||
|
restrict,from="100.69.78.124" ssh-ed25519 AAAA…(上一步的公钥) plane-agentd@agentd-01
|
||||||
|
```
|
||||||
|
`restrict` 关掉所有转发、pty 和 `~/.ssh/rc`(agentd 用 `-T`,不需要 pty),不必再单独写 `no-port-forwarding` 等;`from=` 只放行 LXC 的 tailnet 地址。sshd 不认识 `restrict`(OpenSSH 7.2 之前)时会把整行判为无效:认证失败、终端走兜底,不会放宽权限;第 5 步的 ping 能看出来(`sshd -V` 或 Windows 上 `sshd.exe -V` 看版本)。可选的再一层:Tailscale ACL 只允许 LXC(`100.69.78.124` 或它的 tag)访问 win10/oregon 的 `:22`,这样密钥泄露了也没用。
|
||||||
|
3. **win10**(经 Warpgate,`tools/win10/win10-ps.sh`):Administrator 属于 Administrators 组,所以 sshd 读的是 `C:\ProgramData\ssh\administrators_authorized_keys`,不是用户目录下的文件。用 ASCII 追加同样的一行(别覆盖已有的行),再把 ACL 收紧到只有 Administrators 和 SYSTEM(用 SID,不受系统语言影响),否则 sshd 会忽略这个文件:
|
||||||
|
```powershell
|
||||||
|
$f = 'C:\ProgramData\ssh\administrators_authorized_keys'
|
||||||
|
$line = 'restrict,from="100.69.78.124" ssh-ed25519 AAAA… plane-agentd@agentd-01'
|
||||||
|
if ((Test-Path $f) -and (Get-Content -Raw $f) -and -not (Get-Content -Raw $f).EndsWith("`n")) { $line = "`r`n" + $line } # 原文件末尾没换行时,别把新行粘到最后一把钥匙后面
|
||||||
|
Add-Content -Path $f -Encoding ascii -Value $line
|
||||||
|
icacls $f /inheritance:r /grant '*S-1-5-32-544:F' /grant '*S-1-5-18:F'
|
||||||
|
```
|
||||||
|
顺便确认 sshd 的默认 shell 是 cmd.exe(`HKLM:\SOFTWARE\OpenSSH` 下没有 `DefaultShell`,或它指向 cmd.exe)。agentd 发的远程命令是 `powershell.exe -NoProfile -NonInteractive -EncodedCommand <引导>`,真正的脚本经 stdin 传过去。外层如果被改成 PowerShell,stdin 可能转不到内层;第 5 步的 ping 能直接发现这个问题。
|
||||||
|
4. **钉住的主机公钥**已写在 `config/agentd.yaml` 的 `terminals.<t>.ssh.host_key`,经两个通道核对过:Warpgate 读终端上的 `ssh_host_ed25519_key.pub`,与 tailnet 上 `ssh-keyscan -t ed25519` 的结果一致。新加终端照此办理:
|
||||||
|
```bash
|
||||||
|
ssh-keyscan -t ed25519 <tailnet ip> # LXC 上,直连
|
||||||
|
agent-access.py … run --target <t> -- cat /etc/ssh/ssh_host_ed25519_key.pub # 第二通道(Windows:type C:\ProgramData\ssh\ssh_host_ed25519_key.pub)
|
||||||
|
```
|
||||||
|
两边一致才能写进配置。agentd 每次调用都用它生成 known_hosts(`<ssh_dir>/known_hosts.<终端>`,别名 `agentd-<终端>`),全局和用户的 known_hosts 都不读。
|
||||||
|
5. **上线与验证**(在 LXC 上;拿着 cron 锁做,期间 cron 和 webhook 的 tick 会跳过,webhook 会自动重试):
|
||||||
|
```bash
|
||||||
|
cd /root/.local/share/server-access-client
|
||||||
|
flock /var/lib/plane-agentd/cron.lock bash -c '
|
||||||
|
git -C /opt/plane-agentd pull &&
|
||||||
|
for t in win10 oregon; do
|
||||||
|
python3 -I agent-access.py --config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
|
||||||
|
--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY --timeout 120 -- \
|
||||||
|
python3 /opt/plane-agentd/cli/agentd ping $t --transport direct-ssh -n 3
|
||||||
|
done'
|
||||||
|
```
|
||||||
|
预期每个终端打印 3 行 `term=<t> via=direct op=ping ms=… rc=0` 和 `ping n: … ms PONG=…`。第 1 次要握手(win10 约 0.5–1.5 s,oregon 约 2–3 s),第 2、3 次复用连接,应明显更快。win10 若是 `timeout`,多半是 stdin 没到内层 PowerShell,见第 3 步。`--transport warpgate-ssh` 可顺带确认兜底仍然通(约 9 s)。
|
||||||
|
`cli/agentd check-config` 会多出 `direct-ssh …` 行,`cron/exec secret args` 末尾多了 `--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY`。
|
||||||
|
6. **观察**:下一个 tick 后看 `/var/log/plane-agentd/remote.log`,应该只有 `via=direct` 行;出现 `via=fallback` 说明直连失败了,同时 `agentd.log` 会有一行原因。在 oregon 上 `cli/agentd smoke GWENT win10 --watch`,量"建项 → `[worker] 第 1 轮开始`",目标 ≤ 15 s(原来 91 s)。
|
||||||
|
7. **回滚**:把终端的 `transport` 改回 `warpgate-ssh`(提交、推送、LXC 上 `git pull`;`ssh`/`fallback` 可以留着)。cron 脚本里的 `--secret-env AGENTD_SSH_KEY=…` 可以保留:agentd 总会把它从子进程环境里去掉。Vault 键被删掉也不会停 tick(脚本会不带它重跑),但每个 tick 会多取一次 Vault,届时最好也删掉这一行。紧急情况下可在 LXC 上 `git -C /opt/plane-agentd checkout 83bc0f2 -- config/agentd.yaml`(旧配置,全部走 Warpgate,新代码照常能用)。要彻底撤销直连授权:删掉两台终端 authorized_keys 里那一行,再在 LXC 上关掉已经建好的 ControlMaster 连接(删 authorized_keys 不会断开它们,紧挨着的 webhook tick 能让它们活过 60 s):`for s in /run/plane-agentd/ssh/*-*; do [ -S "$s" ] && ssh -O exit -o ControlPath="$s" x; done`。
|
||||||
|
8. **轮换密钥**:照第 1 步生成新钥,先把新公钥加到两台终端,再更新 Vault `AGENTD_SSH_KEY`,然后删掉旧公钥的行,再照第 7 步关掉已有的 ControlMaster 连接(否则它们空闲 60 s 后才退出)。之后新连接就用新钥。改了某个终端钉住的 `host_key` 不需要这一步:套接字名里带着 pin 的指纹,新 pin 自动用新连接。
|
||||||
|
|||||||
+34
-14
@@ -3,22 +3,22 @@
|
|||||||
Plane 看板驱动的通用 agent 调度服务:任何人/agent 在 Plane 建工作项并指派给一个"终端",agentd 在该终端起对应的 CLI agent(Codex、Claude Code、脚本…)执行,把结果贴回工作项并流转状态。调度是确定性代码,模型只出现在执行器和总指挥里。
|
Plane 看板驱动的通用 agent 调度服务:任何人/agent 在 Plane 建工作项并指派给一个"终端",agentd 在该终端起对应的 CLI agent(Codex、Claude Code、脚本…)执行,把结果贴回工作项并流转状态。调度是确定性代码,模型只出现在执行器和总指挥里。
|
||||||
|
|
||||||
## 1. 目标与非目标
|
## 1. 目标与非目标
|
||||||
- 目标:多项目隔离(一份配置一段)、多终端(本机/经 Warpgate 的远程机)、多 CLI(执行器适配器)、多轮(线程持久、进程不常驻)、失败可解释可重试、凭据只经 Vault 注入内存、任何会话可接手。
|
- 目标:多项目隔离(一份配置一段)、多终端(本机/经 Warpgate 或 Tailscale 直连的远程机)、多 CLI(执行器适配器)、多轮(线程持久、进程不常驻)、失败可解释可重试、凭据只经 Vault 注入内存、任何会话可接手。
|
||||||
- 非目标:不是工作流引擎(无 DAG/分支,顺序交给 Plane "blocked by"——**TODO:阶段 1 尚未实现**,调度不读 blocked-by 关系,被阻塞的 Todo 项照样会被拾起);不替代 Plane/Temporal 的界面;不做账号系统;不存密钥。
|
- 非目标:不是工作流引擎(无 DAG/分支,顺序交给 Plane "blocked by"——**TODO:阶段 1 尚未实现**,调度不读 blocked-by 关系,被阻塞的 Todo 项照样会被拾起);不替代 Plane/Temporal 的界面;不做账号系统;不存密钥。
|
||||||
|
|
||||||
## 2. 架构
|
## 2. 架构
|
||||||
```
|
```
|
||||||
Plane ──webhook v2(HMAC)/ 轮询兜底──► agentd(接收端 + 调度核心 + 执行器适配器)
|
Plane ──webhook(HMAC)即时触发 / cron 轮询兜底──► agentd(接收端 + 调度核心 + 执行器适配器)
|
||||||
│ 按 assignee→终端、label→执行器/模型,串行组,重试,超时,心跳
|
│ 按 assignee→终端、label→执行器/模型,串行组,重试,超时,心跳
|
||||||
┌───────────────┼───────────────┐
|
┌───────────────┼───────────────┐
|
||||||
codex(远程/本机) claude-code(本机/远程) shell
|
codex(远程/本机) claude-code(本机/远程) shell
|
||||||
└───────────────┴───────────────┘
|
└───────────────┴───────────────┘
|
||||||
执行器只产出"最后消息 + 产物路径";agentd 统一回帖、改状态、通知(ntfy)
|
执行器只产出"最后消息 + 产物路径";agentd 统一回帖、改状态、通知(notify 钩子,暂为空操作)
|
||||||
```
|
```
|
||||||
阶段 3 之后调度核心搬到 Temporal(workflow = 一个工作项的生命周期,activity = 起执行器/等结果/回帖),用 Temporal UI 看运行与重试。
|
阶段 3 之后调度核心搬到 Temporal(workflow = 一个工作项的生命周期,activity = 起执行器/等结果/回帖),用 Temporal UI 看运行与重试。
|
||||||
|
|
||||||
## 3. 概念与契约
|
## 3. 概念与契约
|
||||||
- **终端(terminal)**:怎么到达一台机器 — `transport`(local | warpgate-ssh)、`os`、`launch`(subprocess | schtasks | nohup)、`repo`、`plane_user`(回帖身份,对应 Vault 键)、`concurrency`、`capabilities`、可用 `executors`。
|
- **终端(terminal)**:怎么到达一台机器 — `transport`(local | warpgate-ssh | direct-ssh,§9)、`os`、`launch`(subprocess | schtasks | nohup)、`repo`、`plane_user`(回帖身份,对应 Vault 键)、`concurrency`、`capabilities`、可用 `executors`。
|
||||||
- **执行器(executor)**:某个 CLI 的适配器 — 起一轮的命令模板、续线程命令模板、`resume: true|false`、可用模型、默认推理强度、沙箱参数、最后消息取法、线程 ID 取法。
|
- **执行器(executor)**:某个 CLI 的适配器 — 起一轮的命令模板、续线程命令模板、`resume: true|false`、可用模型、默认推理强度、沙箱参数、最后消息取法、线程 ID 取法。
|
||||||
- **路由(routing)**:`by_assignee`(Plane 用户 → 终端)、`by_label`(`exec:*`、`model:*`、`effort:*`、`needs-*` 串行组、`cap:*` 能力、`budget:*` 时限)、每终端默认执行器。
|
- **路由(routing)**:`by_assignee`(Plane 用户 → 终端)、`by_label`(`exec:*`、`model:*`、`effort:*`、`needs-*` 串行组、`cap:*` 能力、`budget:*` 时限)、每终端默认执行器。
|
||||||
- **工作项契约**:正文 = 任务书;评论 = 消息;执行器最后消息首行 ∈ `[review] | [question] | [blocked] | [progress]`;agentd 回帖格式固定(轮次、run id、退出码、日志路径)。
|
- **工作项契约**:正文 = 任务书;评论 = 消息;执行器最后消息首行 ∈ `[review] | [question] | [blocked] | [progress]`;agentd 回帖格式固定(轮次、run id、退出码、日志路径)。
|
||||||
@@ -34,7 +34,7 @@ Plane ──webhook v2(HMAC)/ 轮询兜底──► agentd(接收端 + 调
|
|||||||
| 环境类(焦点丢失/弹窗/断网) | 当 blocked 处理,暂停同串行组的派发 | Waiting |
|
| 环境类(焦点丢失/弹窗/断网) | 当 blocked 处理,暂停同串行组的派发 | Waiting |
|
||||||
| 运行中工作项被移出 In Progress | 结束任务进程,回帖"运行已取消:工作项状态被改为 X",释放终端槽位/串行组,保留线程 | 不改 |
|
| 运行中工作项被移出 In Progress | 结束任务进程,回帖"运行已取消:工作项状态被改为 X",释放终端槽位/串行组,保留线程 | 不改 |
|
||||||
| 连续 `max_progress_rounds`(默认 12)轮 `[progress]` | 回帖说明,不再续跑 | Waiting → 应答者 |
|
| 连续 `max_progress_rounds`(默认 12)轮 `[progress]` | 回帖说明,不再续跑 | Waiting → 应答者 |
|
||||||
| agentd 自身 | cron 拉起;心跳 10 min 无更新 → ntfy 报警 | — |
|
| agentd 自身 | cron 拉起(webhook 只是提前触发);心跳 10 min 无更新 → 报警(notify 钩子,未实现) | — |
|
||||||
回到 Todo 的工作项无条件重新拾起;逻辑性失败永不自动重试。
|
回到 Todo 的工作项无条件重新拾起;逻辑性失败永不自动重试。
|
||||||
|
|
||||||
阶段 1 的具体取舍(均有单测):
|
阶段 1 的具体取舍(均有单测):
|
||||||
@@ -58,7 +58,6 @@ executors:
|
|||||||
projects:
|
projects:
|
||||||
GWENT:
|
GWENT:
|
||||||
id: 79cb5a60-aa4f-4138-9864-1dd70426e665
|
id: 79cb5a60-aa4f-4138-9864-1dd70426e665
|
||||||
webhook_secret_key: PLANE_WEBHOOK_GWENT
|
|
||||||
states: {todo: Todo, running: In Progress, waiting: Waiting, review: Review, done: Done}
|
states: {todo: Todo, running: In Progress, waiting: Waiting, review: Review, done: Done}
|
||||||
reviewer: oregon
|
reviewer: oregon
|
||||||
routing:
|
routing:
|
||||||
@@ -67,16 +66,20 @@ projects:
|
|||||||
default_executor: {win10: codex, oregon: claude-code}
|
default_executor: {win10: codex, oregon: claude-code}
|
||||||
retry: {transient_max: 3, backoff_minutes: [2, 5, 10]}
|
retry: {transient_max: 3, backoff_minutes: [2, 5, 10]}
|
||||||
timeout_hours: 4
|
timeout_hours: 4
|
||||||
notify: {ntfy_topic: gwent, on: [waiting, review, failed]}
|
webhook: {bind: 10.77.10.249, port: 8790, path: /plane, secret_key: PLANE_WEBHOOK_SECRET,
|
||||||
|
tick_command: [systemd-run, --scope, --quiet, --collect, /opt/plane-agentd/cron/agentd-tick-lxc.sh],
|
||||||
|
debounce_seconds: 3, retry_seconds: 15, retry_window_seconds: 120}
|
||||||
|
notify: {kind: none} # 钩子:核心每次改状态调用 Agentd.notify(),none = 空操作
|
||||||
```
|
```
|
||||||
|
|
||||||
## 6. 仓库布局
|
## 6. 仓库布局
|
||||||
```
|
```
|
||||||
agentd/ 调度核心、plane 客户端、vault 注入、状态存储(json/sqlite)、心跳
|
agentd/ 调度核心、plane 客户端、vault 注入、状态存储(json/sqlite)、心跳
|
||||||
agentd/executors/ codex.py claude_code.py shell.py(适配器契约见 docs/executor-contract.md)
|
agentd/executors/ codex.py claude_code.py shell.py(适配器契约见 docs/executor-contract.md)
|
||||||
agentd/transports/ local.py warpgate_ssh.py(远程 Linux/macOS:bash -c 经中继;Windows:helper 脚本 + schtasks)
|
agentd/transports/ local.py warpgate_ssh.py(远程 Linux/macOS:bash -c;Windows:PowerShell + schtasks;Warpgate 通道)direct_ssh.py(Tailscale 直连通道,PAD-12)
|
||||||
agentd/web/ 配置页面(阶段 2):读写 YAML + schema 校验 + 连通/冒烟测试 + 最近运行
|
agentd/web/ 配置页面(阶段 2):读写 YAML + schema 校验 + 连通/冒烟测试 + 最近运行
|
||||||
cli/ agentd new|show|comment|state|assign|smoke
|
cli/ agentd tick|new|show|comment|state|assign|smoke|webhook-serve|check-config
|
||||||
|
systemd/ plane-agentd-webhook.service(LXC 上的 webhook 接收端)
|
||||||
config/ agentd.yaml(示例 + 实际)、schema.json
|
config/ agentd.yaml(示例 + 实际)、schema.json
|
||||||
docs/ design.md executor-contract.md runbook.md
|
docs/ design.md executor-contract.md runbook.md
|
||||||
```
|
```
|
||||||
@@ -84,17 +87,34 @@ docs/ design.md executor-contract.md runbook.md
|
|||||||
## 7. 阶段
|
## 7. 阶段
|
||||||
1. **核心**:配置驱动调度(轮询)、codex-remote(win10) + claude-local(oregon) 两个执行器、线程续接与重建、失败说明与重试、心跳、CLI;gwent 项目作为首个租户切换过来(替换现有 board-worker-plane)。
|
1. **核心**:配置驱动调度(轮询)、codex-remote(win10) + claude-local(oregon) 两个执行器、线程续接与重建、失败说明与重试、心跳、CLI;gwent 项目作为首个租户切换过来(替换现有 board-worker-plane)。
|
||||||
2. **配置页面**:单页,三个标签(终端/执行器/路由)+ 状态条;保存即校验、提交 Git、热加载;不显示密钥。
|
2. **配置页面**:单页,三个标签(终端/执行器/路由)+ 状态条;保存即校验、提交 Git、热加载;不显示密钥。
|
||||||
3. **Webhook + ntfy**:Plane webhook v2 签名校验、即时触发;ntfy 通知 Waiting/Review/失败;轮询保留兜底。前提:Plane VM 进 tailnet 或接收端放在可达位置。
|
3. **Webhook + 通知**:Plane webhook 签名校验、即时触发,轮询保留兜底——**PAD-7 已实现**(§10)。通知 Waiting/Review/失败:只留了空操作钩子 `notify: {kind: none}`(没有 ntfy 服务)。
|
||||||
4. **Temporal**:调度核心迁为 workflow/activity,获得重试/超时/历史/UI;agentd 变为 worker + webhook 入口。
|
4. **Temporal**:调度核心迁为 workflow/activity,获得重试/超时/历史/UI;agentd 变为 worker + webhook 入口。
|
||||||
5. **Gitea 联动**:执行器产物以分支/PR 提交并回帖链接;可选 Gitea webhook 回写 Plane。
|
5. **Gitea 联动**:执行器产物以分支/PR 提交并回帖链接;可选 Gitea webhook 回写 Plane。
|
||||||
|
|
||||||
## 8. 安全
|
## 8. 安全
|
||||||
配置里只有 Vault 键名;token 由 `agent-access.py exec --secret-env` 注入 agentd 进程;执行器不持有 Plane 凭据、不能直接改工作项;webhook 按项目签名校验;页面仅 tailnet 可达,登录走既有网关。
|
配置里只有 Vault 键名;token 由 `agent-access.py exec --secret-env` 注入 agentd 进程;执行器不持有 Plane 凭据、不能直接改工作项;webhook 是工作区级的一个密钥(Vault 键 `PLANE_WEBHOOK_SECRET`,只经 `--secret-env` 进接收端进程,不传给子进程),HMAC 校验失败 401,只认配置里的项目;接收端只听内网地址;页面仅 tailnet 可达,登录走既有网关。
|
||||||
|
|
||||||
## 9. 部署位置
|
## 9. 部署位置
|
||||||
- Plane VM(plane-01)只跑 Plane:它是设备化节点(固定版本、每日备份停写路径),不放 agentd。
|
- 网络(2026-10-21):LXC 到 NAS 的 tailnet 路径依赖 OPNsense 的静态端口出站 NAT 规则 + LXC tailscaled 固定 UDP 41642,已直连(详见 docs/deploy-lxc.md §2.1);曾因 DERP 导致调度器暂回 oregon,已于同日切回 LXC。
|
||||||
|
- Plane VM(Plane 3.3.3,PVE VM 506 plane-02)只跑 Plane:它是设备化节点(固定版本、每日备份停写路径),不放 agentd。
|
||||||
- 阶段 1–2:agentd 跑在 oregon(已有 Vault 客户端、Warpgate 中继脚本、Claude Code、现有 worker 状态),cron 拉起;配置页面同进程,只监听 tailnet 地址。
|
- 阶段 1–2:agentd 跑在 oregon(已有 Vault 客户端、Warpgate 中继脚本、Claude Code、现有 worker 状态),cron 拉起;配置页面同进程,只监听 tailnet 地址。
|
||||||
- 阶段 3 起(**2026-10-21 PAD-11 已切换**):迁到 PVE 上的精简 LXC `agentd-01`(CT 600,root,仓库 `/opt/plane-agentd`,状态 `/var/lib/plane-agentd`,日志 `/var/log/plane-agentd`;1 vCPU / 1–2 GB,tailnet + 内网 10.77.10.0/24,与 Plane/Temporal 同网段,webhook 走内网)。**LXC 上不装任何 agent CLI**,只有 Python 3.10+、PyYAML、Agent Vault 客户端、ssh 客户端;所有终端(win10、oregon、mac…)一律远程启动(`transport: warpgate-ssh`),LXC 只负责调度、回帖、通知。对外访问经 Warpgate HTTP target,复用既有登录。
|
- 阶段 3 起(**2026-10-21 PAD-11 已切换**):迁到 PVE 上的精简 LXC `agentd-01`(CT 600,root,仓库 `/opt/plane-agentd`,状态 `/var/lib/plane-agentd`,日志 `/var/log/plane-agentd`;1 vCPU / 1–2 GB,tailnet + 内网 10.77.10.0/24,与 Plane/Temporal 同网段,webhook 走内网)。**LXC 上不装任何 agent CLI**,只有 Python 3.10+、PyYAML、Agent Vault 客户端、ssh 客户端;所有终端(win10、oregon、mac…)一律远程启动(`transport: warpgate-ssh` 或 `direct-ssh`,见下文"终端通道"),LXC 只负责调度、回帖、通知。对外访问经 Warpgate HTTP target,复用既有登录。
|
||||||
- 远程启动的通用实现(P1-6,已实现):Linux/macOS 经 Warpgate SSH(`agent-access.py run --target <终端 target> -- bash -c …`)上传 `prompt.md`/`run.sh` 并一次调用内启动,`setsid nohup` 脱离会话(macOS 用 perl `POSIX::setsid`),轮询 `exit.txt`,从终端上的输出文件取回最后消息/线程 ID,kill 按进程组 + 会话;中继目标名由终端配置给出。oregon 已切到这条路径(调度器暂仍在 oregon,启动经 Warpgate 回到本机)。部署步骤:`docs/deploy-lxc.md`。
|
- 远程启动的通用实现(P1-6,已实现):Linux/macOS 经 Warpgate SSH(`agent-access.py run --target <终端 target> -- bash -c …`)上传 `prompt.md`/`run.sh` 并一次调用内启动,`setsid nohup` 脱离会话(macOS 用 perl `POSIX::setsid`),轮询 `exit.txt`,从终端上的输出文件取回最后消息/线程 ID,kill 按进程组 + 会话;中继目标名由终端配置给出。oregon 已切到这条路径(调度器暂仍在 oregon,启动经 Warpgate 回到本机)。部署步骤:`docs/deploy-lxc.md`。
|
||||||
- 防重复启动(PAD-11):启动调用之前先把计划中的运行(run id、终端、handle、记账字段)作为 `launching` 标记落盘;启动调用失败/超时或 agentd 中途退出时,下一 tick 探测运行目录(`exit.txt`/`launch.pid`/`pid.txt`):有 → 接管;无 → 同一 run id 换新目录 `<run_id>-r2` 重启(不清空旧目录);探测不通 → 保留标记并占住终端槽位。transport 拒绝清空含 pid 文件的运行目录。某终端中继失败时本 tick 余下操作跳过该终端,其他终端照常。
|
- 防重复启动(PAD-11):启动调用之前先把计划中的运行(run id、终端、handle、记账字段)作为 `launching` 标记落盘;启动调用失败/超时或 agentd 中途退出时,下一 tick 探测运行目录(`exit.txt`/`launch.pid`/`pid.txt`):有 → 接管;无 → 同一 run id 换新目录 `<run_id>-r2` 重启(不清空旧目录);探测不通 → 保留标记并占住终端槽位。transport 拒绝清空含 pid 文件的运行目录。某终端中继失败时本 tick 余下操作跳过该终端,其他终端照常。
|
||||||
- Windows 仍用 helper 脚本 `win10-ps.sh` / `win10-put.sh`(仓库 `tools/win10/`,写死 `--target win10`)+ 计划任务。**TODO(统一)**:Windows 也改走 `relay()`(argv = `powershell.exe -EncodedCommand …`,文件同样 base64 分块),删掉对 helper 脚本与 `helpers_dir` 的依赖,LXC 上就不必再复制这两个脚本。
|
- Windows 走 Warpgate 时仍用 helper 脚本 `win10-ps.sh` / `win10-put.sh`(仓库 `tools/win10/`,写死 `--target win10`)+ 计划任务:`agent-access.py run` 把 ssh 的 stdin 接到 /dev/null(`run_remote` 里 `stdin=subprocess.DEVNULL`),cmd.exe 命令行又限 8191 字符,文件只能按 2400 字符分块上传。走 `direct-ssh` 时不用 helper 脚本。收尾读取在 Warpgate 上也已合成一次调用(脚本足够短,放得进 `-EncodedCommand`)。
|
||||||
|
- **终端通道(PAD-12,2026-10-22)**:LXC 到每个远程终端走哪条路,按终端配置 `transport`:
|
||||||
|
- `warpgate-ssh`(原有,缺省):`agent-access.py run` 经 Warpgate SSH 网关(`115.28.210.101:2222`)。每次调用都要取 Vault 凭据、与网关握手、密码认证、网关再连终端,LXC 上实测中位数约 9 s(客户端 + vault 0.8 s、网关 banner/kex 1.8 s、密码认证 2.2 s、二跳 + 命令 2–5 s)。客户端写死 `ControlMaster=no`,连接不能复用,数据只能放 argv。一次 win10 启动约 10 次串行调用(busy 检查、mkdir/清空、prompt.md 分块上传 + 重置 + 解码、run.ps1、schtasks):GW-26-1 用了 91 s;收尾约 7 次,约 60 s。
|
||||||
|
- `direct-ssh`(新):LXC 经 Tailscale 直连终端的 sshd(`ssh: {host, user, port, host_key}`)。认证用一把专用 ed25519 密钥:Vault 键 `AGENTD_SSH_KEY` 经 `--secret-env` 进 tick 进程,载入本进程私有的 ssh-agent(0700 临时目录里的套接字,`ssh-add -` 从 stdin 读)后立即从环境变量删掉,子进程拿不到;不落盘,不进 argv、日志、Plane 评论或异常信息。主机公钥钉死在配置里,`StrictHostKeyChecking=yes`,known_hosts 由钉住的公钥生成,只含这一把。`ControlMaster=auto`,`ControlPersist=60`,套接字放在只有 root 可进的 `agentd.ssh_dir`(`/run/plane-agentd/ssh`),名字是 `<host_key 指纹 8 位>-%C`:pin 改了就换新连接,不会沿用按旧 pin 校验过的主连接。密钥在 CLI 启动时就从环境变量里拿走(第一次直连之前起的子进程也继承不到)。脚本走 stdin:Windows 是一段固定的 PowerShell 引导(`-EncodedCommand`),从 stdin 读真正脚本的 base64;Linux 是固定的 bash 加载器。所以每个操作恰好一次调用,不受 argv 长度限制。win10 启动(建目录、清空并遵守 HASPID 拒绝、写 UTF-8 无 BOM 的 prompt.md/run.ps1、注册并启动计划任务)一次;收尾一次(exit、最后消息、线程 ID、stderr 尾、事件尾)加删计划任务一次。oregon 启动一次,收尾一次。busy 检查和 poll 仍各一次。实测直连路径:win10 RTT 27 ms(握手到认证 0.24 s),oregon RTT 233 ms(握手 2.2 s)。目标:派单延迟 ≤ 15 s。
|
||||||
|
- `fallback: warpgate-ssh`(可选,需要 `target`):直连**在远程命令运行之前**失败时(ssh 退出 255、没有任何 stdout,且 stderr 里有 ssh 自己在行首打印的连接超时/拒绝/无路由/解析失败/认证失败/agent 错误,或者本进程没拿到密钥。远程加载器把脚本自己的 255 改报 254,所以 255 只可能来自 ssh;有 stdout 就说明命令已经跑过),这个操作改走 Warpgate 重做,本 tick 里该终端余下的操作也都走 Warpgate,`agentd.log` 记一行。**主机公钥不符(包括终端不提供 ssh-ed25519 主机密钥)绝不回退**:按安全事件处理,`agentd.log` 记 `SECURITY …` 行,该终端本 tick 标为 down。命令可能已经运行过的失败(超时、会话中途断开、认不出的 255)不会换路径重试。启动这种非幂等操作仍靠 `launching` 标记加下一 tick 的探测/接管(`resolve_launch`),不存在双启动的路径。直连的每个操作都只有一次远程调用,所以"整个操作换路径重做"不会重复任何副作用。
|
||||||
|
- **取舍**:直连绕过了 Warpgate 的会话录像和集中授权。访问控制改由终端自己负责:authorized_keys 只认这把专用密钥,并限定来源 `from="100.69.78.124"`(LXC 的 tailnet 地址)。用户已逐终端批准(win10、oregon,2026-10-22)。回滚:把该终端的 `transport` 改回 `warpgate-ssh`(`ssh`/`fallback` 可以留着,此时被忽略)。
|
||||||
|
- 观测:每次远程调用在 `<log_dir>/remote.log` 记一行(`term= via=direct|warpgate|fallback op= ms= rc=`)。部署后用 `agentd ping <终端> [--transport direct-ssh|warpgate-ssh]` 验证。步骤见 `docs/deploy-lxc.md` §9。
|
||||||
|
|
||||||
|
## 10. Webhook 触发(PAD-7)
|
||||||
|
- **形态**:`agentd webhook-serve`(`agentd/webhook.py`,只用标准库)常驻在 LXC 上(systemd `plane-agentd-webhook`),Plane 工作区级 webhook(事件 `issue` + `issue_comment`)投到 `http://10.77.10.249:8790/plane`。它不自己调度,只是**提前跑一次 cron 的 tick 脚本**:调度仍然只有一份代码、一把锁、一个状态文件;cron 每 2 分钟照跑,webhook 丢了/接收端挂了只是退回到轮询延迟。
|
||||||
|
- **校验**:`X-Plane-Signature` = HMAC-SHA256(密钥, 请求体) 的十六进制(Plane 签 `json.dumps(payload)` 并原样发送);常量时间比较,先比原始字节、再比重新序列化的结果。缺失/不符 → 401,不触发。
|
||||||
|
- **路由**:`data.project` ∈ 配置的项目 → 200 并触发;否则(别的项目、删除事件没有项目)→ 202 忽略。不按事件类型或操作者过滤:agentd 自己回帖/改状态也会产生投递,结果是每次有动作的 tick 之后多一次 tick(空闲 tick 不写 Plane,不会循环)。
|
||||||
|
- **合并与补跑**:第一条投递后等 3 s,期间的投递合并成一次 tick;tick 运行中又来的投递 → 结束后再跑一次(运行中的 tick 可能在变化之前读过 Plane)。锁被占(cron/手工 tick;脚本 `flock -n -E 75`,`cli tick` 撞上 agentd 状态锁也退出 75)→ 每 15 s 重试,最长 2 min,之后交给 cron。
|
||||||
|
- **进程隔离**:tick 用 `systemd-run --scope` 跑在独立 cgroup 里,重启/停止接收端不会杀掉正在跑的 tick(tick 可能正在启动远程运行)。
|
||||||
|
- **延迟构成**:投递 → tick 开始 ≈ 3 s 合并窗口;tick 本身 = vault 取 token(1–2 s)+ 读 Plane + 先 poll 每个在途运行(每个中继调用约 10 s)+ 启动(中继约 12 s;win10 上传更久)。`[worker] 第 k 轮开始` 评论在启动调用返回之后才发,所以"建项 → 开始评论"受中继成本限制,而不是触发方式。PAD-12 之后,直连终端每次调用不再是约 10 s,而是亚秒到约 2 s(§9 终端通道)。
|
||||||
|
- **Plane 的 SSRF 防护**:Plane(≥1.4 / 商业版 3.x)拒绝解析到私有/CGNAT 地址的 webhook URL(创建时和每次投递时都查),除非 Plane 的环境里有 `WEBHOOK_ALLOWED_IPS`(CIDR 列表)或 `WEBHOOK_ALLOWED_HOSTS`。内网接收端需要在 Plane 的 api 与 worker 容器环境里设 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(改 `/opt/plane/plane.env` 与渲染的 `/var/plane/docker-compose.yml`,重建 api/worker,见 deploy-lxc §8)。2026-10-22 已应用(同时重建了 webhook-consumer、outbox-poller、automation-consumer、beat-worker,见 deploy-lxc §8),webhook 已启用,cron 保留为兜底。Plane 建出的是 v2 webhook(`plane/webhook/bgtasks/webhook_task.py` 的 `send_webhook`,负载 `{version: v2, event, entity_type, data, …}`),签名同为 HMAC-SHA256(secret, json.dumps(payload)),`data.project` 路由照常,接收端无需改动。
|
||||||
|
- **心跳里的跳过原因**:每个 tick 把没派出去的 Todo 项和原因写进 `heartbeat.json` 的 `skipped`(`concurrency 1/1 (GW-15)`、`serialize_group game held by GW-4`、`busy_process codex (1 running, not started by agentd)`、`relay down`、`no route for assignee kappiee`、`start failed: …`),同一项的原因变化时才写一行日志。
|
||||||
|
|||||||
@@ -26,11 +26,11 @@ agentd 不常驻:一轮运行比起它的那次 tick 活得久,所以执行
|
|||||||
- `kill(handle)` 既用于超时,也用于取消(工作项在运行中被移出 In Progress);必须能在运行未结束时调用。取消后 core 会尝试 `collect(handle)` 取线程 ID,所以 `collect` 在未结束的运行上也不能抛出致命错误(抛异常会被忽略)。
|
- `kill(handle)` 既用于超时,也用于取消(工作项在运行中被移出 In Progress);必须能在运行未结束时调用。取消后 core 会尝试 `collect(handle)` 取线程 ID,所以 `collect` 在未结束的运行上也不能抛出致命错误(抛异常会被忽略)。
|
||||||
|
|
||||||
## RunnerExecutor(现有三个适配器的共同形态)
|
## RunnerExecutor(现有三个适配器的共同形态)
|
||||||
1. 在终端上建运行目录 `<run_root>/<dir_name>`(清空旧文件;目录里已有 `pid.txt`/`launch.pid` 时拒绝并报 `HASPID`),写 `prompt.md`。transport 有 `start_run` 时(远程 POSIX),建目录、写 `prompt.md` 与 runner、启动合并成一次中继调用。
|
1. 在终端上建运行目录 `<run_root>/<dir_name>`(清空旧文件;目录里已有 `pid.txt`/`launch.pid` 时拒绝并报 `HASPID`),写 `prompt.md`。transport 有 `start_run` 时(远程终端),建目录、写 `prompt.md` 与 runner、启动由 transport 一起做:直连(direct-ssh)和远程 POSIX 是一次调用,Windows 经 Warpgate 时仍逐步上传。
|
||||||
2. 用配置里的命令模板生成命令:`run`(新线程)或 `resume`(续线程)。占位符:`{repo} {model} {effort} {tier} {out} {thread} {perm} {script} {run_dir}`;未知占位符原样保留。`{tier}` 展开为 `tier_arg`(如 `-c 'service_tier="fast"'`),无 tier 时为空。tier 来源:`tier:<x>` 标签或 `routing.by_label` 的 `tier`,否则执行器的 `tier_defaults[<model>]`;`tier:none` 表示不加档位参数。模板首词若在终端 `bins` 里,替换为绝对路径(Windows 前加 `&`)。参数请用单引号,bash 和 PowerShell 都认。
|
2. 用配置里的命令模板生成命令:`run`(新线程)或 `resume`(续线程)。占位符:`{repo} {model} {effort} {tier} {out} {thread} {perm} {script} {run_dir}`;未知占位符原样保留。`{tier}` 展开为 `tier_arg`(如 `-c 'service_tier="fast"'`),无 tier 时为空。tier 来源:`tier:<x>` 标签或 `routing.by_label` 的 `tier`,否则执行器的 `tier_defaults[<model>]`;`tier:none` 表示不加档位参数。模板首词若在终端 `bins` 里,替换为绝对路径(Windows 前加 `&`)。参数请用单引号,bash 和 PowerShell 都认。
|
||||||
3. transport 生成 runner 脚本(Linux `run.sh`,Windows `run.ps1`):进入 repo,写 `pid.txt`/`started.txt`,把 `prompt.md` 从 stdin 喂给命令,stdout → `stdout_name`,stderr → `stderr.txt`,结束写 `exit.txt`(`exit=N at …`)。
|
3. transport 生成 runner 脚本(Linux `run.sh`,Windows `run.ps1`):进入 repo,写 `pid.txt`/`started.txt`,把 `prompt.md` 从 stdin 喂给命令,stdout → `stdout_name`,stderr → `stderr.txt`,结束写 `exit.txt`(`exit=N at …`)。
|
||||||
4. transport 分离启动(本机 `setsid -f bash run.sh`;远程 Linux `setsid nohup bash run.sh &`,macOS 用 `perl POSIX::setsid`,`$!` 记入 `launch.pid`;Windows 一次性计划任务 `agentd-<run_id>`)。
|
4. transport 分离启动(本机 `setsid -f bash run.sh`;远程 Linux `setsid nohup bash run.sh &`,macOS 用 `perl POSIX::setsid`,`$!` 记入 `launch.pid`;Windows 一次性计划任务 `agentd-<run_id>`)。
|
||||||
5. `poll` = `exit.txt` 是否存在;`collect` 由各适配器从终端上保存的文件解析(远程时经中继读回,不依赖调度器本机有任何 CLI):
|
5. `poll` = `exit.txt` 是否存在;`collect` 由各适配器从终端上保存的文件解析(远程时读回,不依赖调度器本机有任何 CLI)。适配器把要读的东西列成 `COLLECT`(`("read", 文件)`、`("grep", 文件, 子串)`、`("tail", 文件, n)`),经 `transport.fetch` 一次调用取回;transport 没有 `fetch` 时逐个读:
|
||||||
|
|
||||||
| 适配器 | stdout | 最后消息 | 线程 ID |
|
| 适配器 | stdout | 最后消息 | 线程 ID |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
@@ -41,7 +41,7 @@ agentd 不常驻:一轮运行比起它的那次 tick 活得久,所以执行
|
|||||||
配置里的 `thread_from` / `last_message` 目前只是说明性字段,解析逻辑在适配器里。
|
配置里的 `thread_from` / `last_message` 目前只是说明性字段,解析逻辑在适配器里。
|
||||||
|
|
||||||
## Transport 接口(`agentd/transports/`)
|
## Transport 接口(`agentd/transports/`)
|
||||||
`os`、`join`、`quote`、`default_run_root`、`mkdir(path, clean)`(clean 遇到 pid 文件必须拒绝)、`probe(run_dir, names)`(存在的文件名列表,一次调用)、`write_text`、`read_text`(不存在返回 None)、`grep_first(path, needle)`、`tail(path, n)`、`runner(run_dir, cwd, command, stdout_name)`、`launch(task, script, run_dir)`、`kill(task, run_dir)`、`forget(task)`、`process_count(name)`(None = 不可达);可选 `start_run(task, run_dir, files, script_name)`(一次完成建目录 + 写文件 + 启动)。
|
`os`、`join`、`quote`、`default_run_root`、`mkdir(path, clean)`(clean 遇到 pid 文件必须拒绝)、`probe(run_dir, names)`(存在的文件名列表,一次调用)、`write_text`、`read_text`(不存在返回 None)、`grep_first(path, needle)`、`tail(path, n)`、`runner(run_dir, cwd, command, stdout_name)`、`launch(task, script, run_dir)`、`kill(task, run_dir)`、`forget(task)`、`process_count(name)`(None = 不可达);可选 `start_run(task, run_dir, files, script_name)`(建目录 + 写文件 + 启动)和 `fetch(run_dir, ops)`(一次读回多个结果,`collect` 用)。远程 transport 的脚本由一个"通道"送达:`warpgate-ssh`(`agent-access.py run`,只能走 argv)或 `direct-ssh`(Tailscale 直连 OpenSSH,脚本走 stdin,每个操作一次调用;可配 `fallback: warpgate-ssh`,见 design §9)。直连在命令运行前失败时抛 `Unreached`(`RoutedTransport` 据此换到兜底通道),主机公钥不符时抛 `HostKeyError`(不回退);`process_count` 不能吞掉 `Unreached`。
|
||||||
- `local`:Linux 本机文件 + `setsid -f`;kill = 对 `pid.txt` 的进程组发 SIGTERM。用于调度器与执行器同机的部署和测试。
|
- `local`:Linux 本机文件 + `setsid -f`;kill = 对 `pid.txt` 的进程组发 SIGTERM。用于调度器与执行器同机的部署和测试。
|
||||||
- `warpgate-ssh` + `os: linux|darwin`(`PosixWarpgateTransport`):每个操作是一次 `agent-access.py --config <vault.config> run --target <target> -- bash -c '<script>'`(`relay()`,目标名取终端 `target`,缺省为终端名;客户端把 argv `shlex.join` 后交给目标账户的登录 shell,stdin 为 /dev/null)。文件:base64 放进 `printf %s '<b64>' | base64 -d`,超过 60000 字符分块追加再解码,回显字节数校验;读文件/`grep_first`/`tail` 的结果以 `B64=` 标记回传。runner 与本机相同(`run.sh`),`setsid nohup` 脱离 SSH 会话(pid = pgid = sid);kill = 对该进程组 `kill -TERM -- -<pid>` + `pkill -TERM -s <pid>`,最多等 5 s 后 KILL,`exit.txt` 已存在时不发信号(防 pid 复用);`process_count` = `pgrep -x`。`repo`/`run_root` 必须是远程绝对路径。每次调用约 10–14 s,所以启动只用一次调用。
|
- `warpgate-ssh` + `os: linux|darwin`(`PosixWarpgateTransport`):每个操作是一次 `agent-access.py --config <vault.config> run --target <target> -- bash -c '<script>'`(`relay()`,目标名取终端 `target`,缺省为终端名;客户端把 argv `shlex.join` 后交给目标账户的登录 shell,stdin 为 /dev/null)。文件:base64 放进 `printf %s '<b64>' | base64 -d`,超过 60000 字符分块追加再解码,回显字节数校验;读文件/`grep_first`/`tail` 的结果以 `B64=` 标记回传。runner 与本机相同(`run.sh`),`setsid nohup` 脱离 SSH 会话(pid = pgid = sid);kill = 对该进程组 `kill -TERM -- -<pid>` + `pkill -TERM -s <pid>`,最多等 5 s 后 KILL,`exit.txt` 已存在时不发信号(防 pid 复用);`process_count` = `pgrep -x`。`repo`/`run_root` 必须是远程绝对路径。每次调用约 10–14 s,所以启动只用一次调用。
|
||||||
- `warpgate-ssh` + `os: windows`(`WindowsWarpgateTransport`):经 `win10-ps.sh`(PowerShell `-EncodedCommand`)和 `win10-put.sh`(分块上传 + SHA-256 校验);文件读取用 base64 标记回传;kill = `taskkill /T /F /PID <pid.txt>` + `schtasks /End /Delete`。脚本目录由 `agentd.helpers_dir`(或终端 `helpers_dir`)配置;脚本目前写死 `--target win10`(TODO:改走同一个 `relay()`,argv 为 `powershell.exe -NoProfile -NonInteractive -EncodedCommand <b64>`,去掉 helper 脚本)。路径都放进 PowerShell 单引号字面量(`'` 双写);`schtasks /TR` 里脚本路径用 `\"…\"` 包裹,依赖 Windows PowerShell 5.1 的旧式原生参数传递(PowerShell 7.3+ 的 Standard 模式下需改写)。runner 进不了 repo 时写 `exit=97` 结束。
|
- `warpgate-ssh` + `os: windows`(`WindowsWarpgateTransport`):经 `win10-ps.sh`(PowerShell `-EncodedCommand`)和 `win10-put.sh`(分块上传 + SHA-256 校验);文件读取用 base64 标记回传;kill = `taskkill /T /F /PID <pid.txt>` + `schtasks /End /Delete`。脚本目录由 `agentd.helpers_dir`(或终端 `helpers_dir`)配置;脚本目前写死 `--target win10`(TODO:改走同一个 `relay()`,argv 为 `powershell.exe -NoProfile -NonInteractive -EncodedCommand <b64>`,去掉 helper 脚本)。路径都放进 PowerShell 单引号字面量(`'` 双写);`schtasks /TR` 里脚本路径用 `\"…\"` 包裹,依赖 Windows PowerShell 5.1 的旧式原生参数传递(PowerShell 7.3+ 的 Standard 模式下需改写)。runner 进不了 repo 时写 `exit=97` 结束。
|
||||||
|
|||||||
+79
-4
@@ -19,13 +19,15 @@ cd /root/.local/share/server-access-client && python3 -I agent-access.py \
|
|||||||
--config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
|
--config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
|
||||||
--secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \
|
--secret-env PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN \
|
||||||
--secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN \
|
--secret-env PLANE_WINDOW10_API_TOKEN=PLANE_WINDOW10_API_TOKEN \
|
||||||
|
--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY \
|
||||||
--timeout 900 -- python3 /opt/plane-agentd/cli/agentd tick --once
|
--timeout 900 -- python3 /opt/plane-agentd/cli/agentd tick --once
|
||||||
```
|
```
|
||||||
|
`AGENTD_SSH_KEY` 是 direct-ssh 终端的专用密钥(PAD-12,design §9)。手工运行时不加这个参数不会出错,只是直连终端整次 tick 都走 Warpgate 兜底(慢),`agentd.log` 里会有一行 `direct-ssh unavailable in this process`。但加了参数而 Vault 里没有这个键,agent-access.py 会以 `access-hub: credential_unavailable` 直接失败,agentd 根本不运行(它没有可选密钥);cron 脚本遇到这种情况会去掉这个参数再跑一次,手工运行则要自己去掉。
|
||||||
crontab(LXC root,每 2 分钟):
|
crontab(LXC root,每 2 分钟):
|
||||||
```
|
```
|
||||||
*/2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh
|
*/2 * * * * /opt/plane-agentd/cron/agentd-tick-lxc.sh
|
||||||
```
|
```
|
||||||
`cron/agentd-tick-lxc.sh` = `flock -n /var/lib/plane-agentd/cron.lock timeout 900 python3 -I agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once >> /var/log/plane-agentd/cron.log 2>&1`。空闲 tick 不打印任何东西(cron 日志为空是正常的),活着与否看 `heartbeat.json` 的 `at`;`skip: another tick holds …` 表示与手工 tick 撞锁,无害。
|
`cron/agentd-tick-lxc.sh` = 拿 `/var/lib/plane-agentd/cron.lock`(`flock -n -E 75`,撞锁退出 75)后执行 `timeout 900 python3 -I agent-access.py --config <identity> exec --secret-env PLANE_OREGON_API_TOKEN=… --secret-env PLANE_WINDOW10_API_TOKEN=… --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY --timeout 850 -- python3 /opt/plane-agentd/cli/agentd --config /opt/plane-agentd/config/agentd.yaml tick --once`,输出追加到 `/var/log/plane-agentd/cron.log`(stderr 排在 stdout 之后)。如果这次 exec 只报了 `access-hub: credential_unavailable`/`credential_invalid`(退出 1,agentd 没运行),脚本在 cron.log 记一行 `AGENTD_SSH_KEY not available from Vault: tick repeated without it …`,去掉 `AGENTD_SSH_KEY` 再跑一次:直连终端走兜底,派单不中断。空闲 tick 不打印任何东西(cron 日志为空是正常的),活着与否看 `heartbeat.json` 的 `at`;`skip: another tick holds …` 表示与手工 tick 撞锁,无害。
|
||||||
实际配置 `config/agentd.yaml`(已提交,不含密钥):`state_dir: /var/lib/plane-agentd`、`log_dir: /var/log/plane-agentd`,所以下表的 `state/…` 在 LXC 上是 `/var/lib/plane-agentd/…`,`agentd.log` 在 `/var/log/plane-agentd/`。改配置/代码 = 在 oregon 提交并推送,然后 LXC 上 `cd /opt/plane-agentd && git pull`(下一个 tick 生效)。
|
实际配置 `config/agentd.yaml`(已提交,不含密钥):`state_dir: /var/lib/plane-agentd`、`log_dir: /var/log/plane-agentd`,所以下表的 `state/…` 在 LXC 上是 `/var/lib/plane-agentd/…`,`agentd.log` 在 `/var/log/plane-agentd/`。改配置/代码 = 在 oregon 提交并推送,然后 LXC 上 `cd /opt/plane-agentd && git pull`(下一个 tick 生效)。
|
||||||
oregon 上的 `cli/agentd` 只当 CLI 用(new/show/comment/state/assign,`smoke … --watch`);**不要在 oregon 上跑 `tick` 或不带 `--watch` 的 `smoke`**(那是第二个调度器)。
|
oregon 上的 `cli/agentd` 只当 CLI 用(new/show/comment/state/assign,`smoke … --watch`);**不要在 oregon 上跑 `tick` 或不带 `--watch` 的 `smoke`**(那是第二个调度器)。
|
||||||
暂停一个终端(不再派新运行、也不探测,在途运行照常收尾):把它的 `concurrency` 设为 0。
|
暂停一个终端(不再派新运行、也不探测,在途运行照常收尾):把它的 `concurrency` 设为 0。
|
||||||
@@ -48,16 +50,40 @@ cli/agentd state GWENT 12 Todo # Plane 状态名或配置键(todo/review
|
|||||||
cli/agentd assign GWENT 12 window10
|
cli/agentd assign GWENT 12 window10
|
||||||
cli/agentd smoke GWENT oregon --watch # 建一个要求回复 "[review] ok" 的工作项,等排程中的调度器跑完,PASS 则移到 Done
|
cli/agentd smoke GWENT oregon --watch # 建一个要求回复 "[review] ok" 的工作项,等排程中的调度器跑完,PASS 则移到 Done
|
||||||
cli/agentd smoke GWENT win10 --labels model:sol --watch # --labels 给冒烟项加标签(路由到模型/档位)
|
cli/agentd smoke GWENT win10 --labels model:sol --watch # --labels 给冒烟项加标签(路由到模型/档位)
|
||||||
|
cli/agentd ping win10 [--transport direct-ssh|warpgate-ssh] [-n 3] # 在终端上跑一条空命令并计时,每次调用一行
|
||||||
|
# (direct-ssh 需要 --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY,不需要 Plane token)
|
||||||
# 不带 --watch = 本机自己 tick(只在调度器所在机器上、且它的 cron 停着时用)
|
# 不带 --watch = 本机自己 tick(只在调度器所在机器上、且它的 cron 停着时用)
|
||||||
```
|
```
|
||||||
CLI 以 `--as <terminal>` / `agentd.cli_terminal` / 第一个有 token 的终端的身份操作。
|
CLI 以 `--as <terminal>` / `agentd.cli_terminal` / 第一个有 token 的终端的身份操作。
|
||||||
|
|
||||||
|
## Webhook(PAD-7:Plane 事件即时触发 tick)
|
||||||
|
cron 每 2 分钟一次的 tick 保留作兜底;另有常驻接收端,Plane 一有工作项/评论变化就立刻触发一次 tick。
|
||||||
|
- **服务**:LXC `plane-agentd-webhook.service`(单元文件在仓库 `systemd/`,`Restart=always`)= `agent-access.py … exec --secret-env PLANE_WEBHOOK_SECRET=PLANE_WEBHOOK_SECRET --timeout 315360000 -- python3 /opt/plane-agentd/cli/agentd webhook-serve --bind 10.77.10.249 --port 8790`。只听内网地址;密钥只在进程环境里,agentd 起的子进程(tick 脚本)环境里去掉了它。
|
||||||
|
- **端点**:`POST http://10.77.10.249:8790/plane`;`GET /healthz` → `{"ok": true, "pending", "queued", "running", "retrying_since", "ticks", "last": {rc, seconds, deliveries, at}}`。
|
||||||
|
- **Plane 侧**:工作区级 webhook(kappiee 工作区设置 → Webhooks,或 `tools/plane-webhook.py`),订阅"工作项"(`issue`)和"评论"(`issue_comment`)。Plane 会把所有项目的事件都发来,接收端按 `data.project` 只认配置里的项目。
|
||||||
|
- **签名**:头 `X-Plane-Signature` = 用 webhook 密钥对请求体做 HMAC-SHA256 的十六进制摘要(Plane 签的是 `json.dumps(payload)`,发出的正是这串字节;接收端先比原始请求体,再比重新序列化的结果,均为常量时间比较)。缺失/不对 → 401,什么都不触发。其他头:`X-Plane-Event`(issue / issue_comment)、`X-Plane-Delivery`(每次投递的 UUID)。请求体:`{"event","action"(created/updated/deleted),"webhook_id","workspace_id","workspace_slug","data":{…,"project":<uuid>},"activity":{field,new_value,old_value,actor…}}`;删除事件的 `data` 只有 `id`(没有项目 → 202 忽略,由 cron 兜底)。
|
||||||
|
- **触发逻辑**:配置项目的投递 → 200,3 s(`debounce_seconds`)内的投递合并成一次 tick;tick 运行期间又来的投递 → 它结束后再跑一次(正在跑的 tick 可能在变化之前就读过 Plane)。tick 命令就是 cron 脚本(`systemd-run --scope … cron/agentd-tick-lxc.sh`,在独立 scope 里跑,重启接收端不会杀掉正在跑的 tick)。脚本持 `flock -n -E 75`:锁被 cron/手工 tick 占着 → 退出码 75 → 每 15 s 重试,最长 2 min,之后交给下一个 cron tick。tick 本身的输出照旧进 `cron.log`/`agentd.log`。
|
||||||
|
- **日志** `/var/log/plane-agentd/webhook.log`:每次投递一行(`delivery=<X-Plane-Delivery> event=issue/updated project=GWENT item=<uuid> decision=tick|202 ignored|401 bad signature|401 missing signature`),每次 tick 两行(`tick start (n deliveries: …)`、`tick done rc=0 in 12.3s` / `tick busy …, retry in 15s` / `giving up`)。Plane 侧的投递记录:`tools/plane-webhook.py logs <webhook id>`。
|
||||||
|
- **查看**:`systemctl status plane-agentd-webhook`、`journalctl -u plane-agentd-webhook`、`curl -s http://10.77.10.249:8790/healthz`、`tail -f /var/log/plane-agentd/webhook.log`。
|
||||||
|
- **轮换密钥**(在 oregon 上,kappiee 管理员会话,密钥不出进程、直接写进 Vault):
|
||||||
|
```bash
|
||||||
|
cd ~/.local/share/server-access-client && python3 -I agent-access.py --config https://nas.tail9531b.ts.net:8443/identity/agent.json exec \
|
||||||
|
--secret-env PLANE_KAPPIEE_PASSWORD=PLANE_KAPPIEE_PASSWORD --timeout 90 -- \
|
||||||
|
python3 -I /home/ubuntu/Documents/wjj/project/plane-agentd/tools/plane-webhook.py regenerate <webhook id> # id: … plane-webhook.py list
|
||||||
|
```
|
||||||
|
然后 LXC 上 `systemctl restart plane-agentd-webhook`(接收端只在启动时取一次密钥;轮换到重启之间的投递会 401,cron 兜底)。核对:`agent-access.py … exec --secret-env S=PLANE_WEBHOOK_SECRET -- python3 -c 'import os;print(len(os.environ["S"]))'`。
|
||||||
|
- **停用**:只停接收端 `systemctl disable --now plane-agentd-webhook`(Plane 投递会失败并重试,多次失败后 Plane 自动停用该 webhook 并给管理员发邮件);连 Plane 侧一起停:`plane-webhook.py patch <id> is_active=false`。cron 不受影响。
|
||||||
|
- **通知**:配置里的 `notify: {kind: none}` 是钩子,核心每次改状态都调用 `Agentd.notify()`,目前什么都不做(没有 ntfy 服务)。
|
||||||
|
|
||||||
## 文件
|
## 文件
|
||||||
| 路径 | 内容 |
|
| 路径 | 内容 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `state/agentd-state.json` | 每项目每工作项:轮次、线程 ID 与所在终端/执行器、线程轮数、各轮标签、已见评论、重试计数/下次重试、在途运行(handle、deadline) |
|
| `state/agentd-state.json` | 每项目每工作项:轮次、线程 ID 与所在终端/执行器、线程轮数、各轮标签、已见评论、重试计数/下次重试、在途运行(handle、deadline) |
|
||||||
| `<log_dir>/agentd.log`(默认 `state/`) | 每次 tick 的动作与错误 |
|
| `<log_dir>/agentd.log`(默认 `state/`) | 每次 tick 的动作与错误 |
|
||||||
| `state/heartbeat.json` | 最近一次 tick 的时间、在途运行、启动未确认的运行(`launching`)、本 tick 跳过的不可达终端(`down`)、错误;超过 10 分钟不更新 = agentd 没在跑 |
|
| `state/heartbeat.json` | 最近一次 tick 的时间、在途运行、启动未确认的运行(`launching`)、本 tick 跳过的不可达终端(`down`)、没派出去的 Todo 项及原因(`skipped`:`[{project, item, terminal, reason}]`)、错误;超过 10 分钟不更新 = agentd 没在跑 |
|
||||||
|
| `<log_dir>/webhook.log` | webhook 接收端:每次投递的决定、每次触发的 tick(见上节) |
|
||||||
|
| `<log_dir>/remote.log` | 每次远程调用一行:`term=win10 via=direct\|warpgate\|fallback op=start\|fetch\|read\|busy\|probe\|kill\|forget… ms=… rc=…`,失败时末尾带 `unreached`/`hostkey`/`unknown`(PAD-12)。不回显到 cron.log;没有轮转,需要时手工截断 |
|
||||||
|
| `agentd.ssh_dir`(LXC `/run/plane-agentd/ssh`) | direct-ssh 的 ControlMaster 套接字(`<host_key 指纹 8 位>-<%C 哈希>`,空闲 60 s 后自动退出;手工关闭:`for s in /run/plane-agentd/ssh/*-*; do [ -S "$s" ] && ssh -O exit -o ControlPath="$s" x; done`)和由钉住公钥生成的 `known_hosts.<终端>`;root 0700,agentd 自己建。ssh-agent 在 `/tmp/agentd-agent-*`(每个 tick 进程一个,退出时删除) |
|
||||||
| `state/agentd.lock` | tick 互斥锁 |
|
| `state/agentd.lock` | tick 互斥锁 |
|
||||||
| `state/runs/<run_id>/` | `transport: local` 终端的运行目录:`prompt.md run.sh pid.txt started.txt exit.txt stderr.txt` + 执行器输出 |
|
| `state/runs/<run_id>/` | `transport: local` 终端的运行目录:`prompt.md run.sh pid.txt started.txt exit.txt stderr.txt` + 执行器输出 |
|
||||||
| oregon `/home/ubuntu/.gwent-board/agentd-runs/<run_id>/` | 远程 Linux 终端(`run_root`):同上,另有 `launch.pid`(启动时的 `$!`,与 `pid.txt` 相同);claude 输出 `out.json` |
|
| oregon `/home/ubuntu/.gwent-board/agentd-runs/<run_id>/` | 远程 Linux 终端(`run_root`):同上,另有 `launch.pid`(启动时的 `$!`,与 `pid.txt` 相同);claude 输出 `out.json` |
|
||||||
@@ -76,7 +102,10 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
|
|||||||
| 现象 | 原因 / 处理 |
|
| 现象 | 原因 / 处理 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `no Plane token in the environment` / tick 报 `no Plane token` | 没在 `agent-access.py exec --secret-env …` 下运行,或键名不对;`check-config` 看 MISSING |
|
| `no Plane token in the environment` / tick 报 `no Plane token` | 没在 `agent-access.py exec --secret-env …` 下运行,或键名不对;`check-config` 看 MISSING |
|
||||||
| `skip: another tick holds state/agentd.lock` | 上一 tick 还在跑(win10 上传慢);正常。持续出现则查是否有卡死的 agentd 进程 |
|
| `skip: another tick holds state/agentd.lock` | 上一 tick 还在跑(win10 上传慢);正常(`tick` 此时退出码 75)。持续出现则查是否有卡死的 agentd 进程 |
|
||||||
|
| Todo 项一直不动、心跳看着空闲 | 看 `heartbeat.json` 的 `skipped`:`concurrency n/m (GW-x)` 槽位被谁占着;`serialize_group … held by …`;`busy_process codex (… not started by agentd)` = win10 上有不是 agentd 起的(或上一轮还没退干净的)codex.exe;`relay down`;`no route for assignee …` = 指派的用户不是终端。原因第一次出现时 `agentd.log`/`cron.log` 有一行 `GW-n not dispatched (<终端>): <原因>` |
|
||||||
|
| `webhook.log` 里 `decision=401` | 签名不对:Plane 侧密钥轮换后接收端没重启,或不是 Plane 发的。`systemctl restart plane-agentd-webhook` |
|
||||||
|
| Plane 建 webhook 报 "The webhook URL is not allowed…" | Plane 的 SSRF 防护拒绝内网地址:Plane api/worker 环境需 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(design §10) |
|
||||||
| `[worker] 第 k 轮失败:类型=crash(临时性)…重试 n/N` | 网络/中继/429/5xx:按 `backoff_minutes` 自动重试,续原线程;超过 `transient_max` 转 Waiting |
|
| `[worker] 第 k 轮失败:类型=crash(临时性)…重试 n/N` | 网络/中继/429/5xx:按 `backoff_minutes` 自动重试,续原线程;超过 `transient_max` 转 Waiting |
|
||||||
| `类型=timeout` | 超过 `timeout_hours`(或 `budget:2h` 标签);进程树已结束,转 Waiting。看运行目录日志决定拆分任务或加预算 |
|
| `类型=timeout` | 超过 `timeout_hours`(或 `budget:2h` 标签);进程树已结束,转 Waiting。看运行目录日志决定拆分任务或加预算 |
|
||||||
| `类型=crash`(非临时)且是续线程 | 自动在新线程重跑一次(附交接摘要);再失败转 Waiting |
|
| `类型=crash`(非临时)且是续线程 | 自动在新线程重跑一次(附交接摘要);再失败转 Waiting |
|
||||||
@@ -88,7 +117,13 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
|
|||||||
| `run GW-n-k: nothing at …; relaunching in GW-n-k-r2` | 启动没有落地:同一 run id、新目录 `<run_id>-r2`(再失败 -r3…)重新启动,旧目录不清空。运行目录里已有 pid 文件时 transport 拒绝清空(`HASPID`),下一 tick 探测后接管 |
|
| `run GW-n-k: nothing at …; relaunching in GW-n-k-r2` | 启动没有落地:同一 run id、新目录 `<run_id>-r2`(再失败 -r3…)重新启动,旧目录不清空。运行目录里已有 pid 文件时 transport 拒绝清空(`HASPID`),下一 tick 探测后接管 |
|
||||||
| heartbeat `launching` 一直不消失 | 终端探测不通(中继坏),标记保留并占着该终端的槽位;中继恢复后自动接管或重启。人工处理:停 cron,在终端上看 `<run_root>/<run_id>/`,再从状态文件删掉该项的 `launching` |
|
| heartbeat `launching` 一直不消失 | 终端探测不通(中继坏),标记保留并占着该终端的槽位;中继恢复后自动接管或重启。人工处理:停 cron,在终端上看 `<run_root>/<run_id>/`,再从状态文件删掉该项的 `launching` |
|
||||||
| `upload of … size mismatch` / `staging … size mismatch` | 远程写文件不完整(磁盘满、`run_root` 不可写);看远程 `run_root` |
|
| `upload of … size mismatch` / `staging … size mismatch` | 远程写文件不完整(磁盘满、`run_root` 不可写);看远程 `run_root` |
|
||||||
| 远程 Linux 终端每步都慢 | 每次中继调用约 10–14 s(取 Vault 凭据 + SSH 握手)。一轮:启动 1 次(prompt 超过约 45 KB 时多几次分块上传)、每 tick poll 1 次、收尾 claude 约 3 次 |
|
| 远程终端每步都慢 | `warpgate-ssh`(或 `via=fallback`)每次调用约 9–14 s(取 Vault 凭据 + 网关握手 + 密码认证 + 二跳)。`direct-ssh` 首次握手 win10 约 0.5 s、oregon 约 2 s,之后 60 s 内复用连接。看 `remote.log` 的 `ms=`。一轮的调用次数:启动 1 次(Warpgate 上 prompt 超过约 45 KB 时多几次分块上传;win10 经 Warpgate 约 8–10 次中继调用),每 tick poll 1 次,收尾 1 次(win10 另加删计划任务 1 次) |
|
||||||
|
| `SECURITY <终端>: … host key verification FAILED …` | 终端出示的主机公钥和配置里钉住的不一致:可能是重装、sshd 重新生成了主机密钥,也可能是中间人。**不会回退到 Warpgate**,该终端本 tick 标为 down(`heartbeat.down`),每个 tick 都会再报一次。先经第二通道核对(Warpgate:`agent-access.py … run --target oregon -- cat /etc/ssh/ssh_host_ed25519_key.pub`,win10 `win10-ps.sh "Get-Content C:\ProgramData\ssh\ssh_host_ed25519_key.pub"`),确认是合法变更后再更新 `terminals.<t>.ssh.host_key`(deploy-lxc §9 第 4 步)。新的 pin 会换一个 ControlMaster 套接字名(`<pin 指纹>-%C`),不会复用按旧 pin 校验过的连接。终端根本不提供 ssh-ed25519 主机密钥(`no matching host key type`)也按这一行处理。等核对期间要继续派单:把 `transport` 改回 `warpgate-ssh` |
|
||||||
|
| `<终端>: direct-ssh failed before the command ran (…); using the warpgate-ssh fallback for the rest of this tick` | 直连在命令运行前就失败了(tailscale 断、sshd 停、authorized_keys 缺或 ACL 不对、密钥被拒)。这个操作已在 Warpgate 上重做,本 tick 余下的也走 Warpgate,下个 tick 再试直连。`remote.log` 里会出现 `via=fallback`。查原因:`cli/agentd ping <终端> --transport direct-ssh -n 1`,括号里是 ssh 的原话(如 `Permission denied (publickey)`、`connect to host … Connection timed out`) |
|
||||||
|
| `direct-ssh unavailable in this process: AGENTD_SSH_KEY is not set …` / `ssh-add failed …` | 进程没拿到密钥(手工运行没加 `--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY`,或 cron 脚本因为 Vault 取不到键而去掉了它),或 Vault 里的值不是无口令的 OpenSSH 私钥。直连终端整次都走兜底;没配 `fallback` 的终端本 tick down |
|
||||||
|
| cron.log:`AGENTD_SSH_KEY not available from Vault: tick repeated without it …` | Vault 键 `AGENTD_SSH_KEY` 被删、改名或不可读,agent-access.py 拒绝了整个 exec。tick 已不带密钥重跑,直连终端每个 tick 都走 Warpgate 兜底(慢,但不停派单)。修复:按 deploy-lxc §9 第 1 步恢复这个键;`agent-access.py … list` 能看到键名是否还在 |
|
||||||
|
| 手工 `exec … --secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY` 只打印 `access-hub: credential_unavailable` | 同上,Vault 里没有这个键(也可能是某个 Plane token 键)。手工运行时去掉 `AGENTD_SSH_KEY` 那个参数即可 |
|
||||||
|
| `ssh administrator@… timeout after 150s` / `ssh … exit 255: …`(无 fallback 行) | 直连的命令**可能已经运行**(超时、会话中途断开、认不出的 255),按规则不换路径重试:终端本 tick down;启动的话保留 `launching` 标记,下一 tick 探测后接管或在 `-r2` 重启。win10 上每次都超时:多半是 stdin 没到 PowerShell(sshd 默认 shell 被改成了 PowerShell),见 deploy-lxc §9 第 3 步 |
|
||||||
| 工作项在运行中被人改了状态 | 运行还没结束:下一 tick 取消(kill + `[worker] 运行已取消…` 评论),不改状态。已经结束(`exit.txt` 已写):结果照常回帖,但不再改状态/重试 |
|
| 工作项在运行中被人改了状态 | 运行还没结束:下一 tick 取消(kill + `[worker] 运行已取消…` 评论),不改状态。已经结束(`exit.txt` 已写):结果照常回帖,但不再改状态/重试 |
|
||||||
| Plane 回帖失败 | 在途记录保留,下一 tick 重新 collect 并回帖(可能重复一条评论) |
|
| Plane 回帖失败 | 在途记录保留,下一 tick 重新 collect 并回帖(可能重复一条评论) |
|
||||||
| 想手工终止一轮 | 把工作项移出 In Progress(如 Waiting)即可,下一 tick agentd 自己 kill 并释放槽位。中继不通、kill 一直失败时:在终端上手工结束(Linux 终端 `kill -TERM -- -$(cat pid.txt); pkill -TERM -s $(cat pid.txt)`,win10 `taskkill /T /F /PID <pid.txt>`),停 cron 后从 `state/agentd-state.json` 删掉该项的 `inflight` |
|
| 想手工终止一轮 | 把工作项移出 In Progress(如 Waiting)即可,下一 tick agentd 自己 kill 并释放槽位。中继不通、kill 一直失败时:在终端上手工结束(Linux 终端 `kill -TERM -- -$(cat pid.txt); pkill -TERM -s $(cat pid.txt)`,win10 `taskkill /T /F /PID <pid.txt>`),停 cron 后从 `state/agentd-state.json` 删掉该项的 `inflight` |
|
||||||
@@ -100,6 +135,35 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
|
|||||||
3. oregon 恢复 cron:`crontab ~/.gwent-board/crontab.backup-20261021-pad11`(即 `*/2 * * * * /home/ubuntu/Documents/wjj/project/plane-agentd/cron/agentd-tick.sh`)。`cron/agentd-tick.sh` 用 `config/agentd.oregon.yaml`(`state_dir: ~/.gwent-board/agentd`,日志 `~/.gwent-board/agentd/agentd.log`、`~/.gwent-board/agentd-cron.log`),与 LXC 配置只差路径。
|
3. oregon 恢复 cron:`crontab ~/.gwent-board/crontab.backup-20261021-pad11`(即 `*/2 * * * * /home/ubuntu/Documents/wjj/project/plane-agentd/cron/agentd-tick.sh`)。`cron/agentd-tick.sh` 用 `config/agentd.oregon.yaml`(`state_dir: ~/.gwent-board/agentd`,日志 `~/.gwent-board/agentd/agentd.log`、`~/.gwent-board/agentd-cron.log`),与 LXC 配置只差路径。
|
||||||
4. 看 oregon 的 `~/.gwent-board/agentd/heartbeat.json` 更新。再切回 LXC = 反过来做一遍(docs/deploy-lxc.md §5)。
|
4. 看 oregon 的 `~/.gwent-board/agentd/heartbeat.json` 更新。再切回 LXC = 反过来做一遍(docs/deploy-lxc.md §5)。
|
||||||
|
|
||||||
|
## 上线记录(2026-10-22,PAD-12:直连 SSH + 单次调用派单)
|
||||||
|
- 代码 5192e04:direct-ssh 传输(会话建立前失败才回落 Warpgate,主机公钥不符绝不回落)、主机公钥固定、Vault 密钥载入 ssh-agent、复用连接、win10 一次调用启动和一次调用收尾、`remote.log`、`agentd ping`。148 个单测通过;经正确性、安全两轮 opus 审查后修正。
|
||||||
|
- 密钥:在 LXC 的 `/dev/shm` 生成 ed25519,私钥只存在 Vault `AGENTD_SSH_KEY`(指纹 `SHA256:+6uooRFFB3tX4fQ+Sfl+3FmLm9KJpPYpSPzoTcCq/2M`),临时文件已删。公钥行 `restrict,from="100.69.78.124" ssh-ed25519 … plane-agentd@agentd-01`:
|
||||||
|
- oregon:`~ubuntu/.ssh/authorized_keys`,备份 `.bak-pad12-20261022-025955`;
|
||||||
|
- win10:`C:\ProgramData\ssh\administrators_authorized_keys`,备份 `.bak-pad12-20261022-110041`,ACL 未变(SYSTEM 和 Administrators 完全控制)。win10 的 sshd 是 OpenSSH_9.5p1,DefaultShell 未设(即 cmd.exe)。
|
||||||
|
- 两台的主机公钥经 ssh-keyscan 和经 Warpgate 读公钥文件两路核对一致后,固定写进配置。
|
||||||
|
- 部署 03:01(持 `cron.lock` 执行 git pull 和 ping)。
|
||||||
|
- ping:oregon 首次 3.9 s(含建立主连接),复用 0.77 s;win10 首次 0.52 s,复用 0.17 s。
|
||||||
|
- 部署时 win10 的第一次调用有一次 60 s 超时。随后 3 轮新建主连接的首次调用都是 0.52 s,复现不出,推测是 tailnet 线路空闲后刚恢复。这类超时不回落,该终端本 tick 算不可用、下个 tick 重试;在 `remote.log` 里继续观察。
|
||||||
|
- 冒烟(真实 webhook):
|
||||||
|
- GW-27 win10:建项 03:06:17.9 → 开工评论 03:06:28.4,**10.5 s**(之前 95 s)。其中 busy 0.22 s、start 0.32 s;收尾 read、fetch、forget 三次调用共 0.9 s。
|
||||||
|
- GW-28 oregon:建项 03:10:09.5 → 开工 03:10:23.6,**14.1 s**(start 4.1 s,含新建主连接);03:10:32.6 结果已贴回。
|
||||||
|
- 剩下的延迟在收尾:运行结束后要等下一次 tick(cron 2 分钟,或别的投递触发)才收。迁 Temporal(PAD-8)时用定时器轮询解决。
|
||||||
|
- 回滚:把该终端的 `transport` 改回 `warpgate-ssh` 再 pull。撤销访问:删掉两台 authorized_keys 里的那一行,再对 `/run/plane-agentd/ssh/` 下的主连接执行 `ssh -O exit`(见 deploy-lxc §9)。
|
||||||
|
|
||||||
|
## 上线记录(2026-10-22,PAD-7:Plane 侧 webhook 启用)
|
||||||
|
- 01:14 Plane VM 506:`plane.env` 与渲染 compose 设 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`(备份 `*.bak-pad7-20261022-011453`),`--dry-run` 确认后只重建 api、worker、webhook-consumer、outbox-poller、automation-consumer、beat-worker(01:15:43–01:16:33);01:18:26 实测 API 已返回 200(之前未测,停机上限约 2.5 分钟),api 容器 4 分钟内转 healthy。
|
||||||
|
- 01:19 `tools/plane-webhook.py create http://10.77.10.249:8790/plane` → 201(白名单生效),密钥写入 Vault `PLANE_WEBHOOK_SECRET`(替换临时值);01:22:30 `systemctl enable --now plane-agentd-webhook`(LXC 代码 1b4d965,单元文件与仓库一致);01:22:57 `patch c039c471-… is_active=true`。
|
||||||
|
- Plane 建出的是 **v2** webhook:负载 `{version: v2, event, entity_type, data, …}`,由 `plane/webhook/bgtasks/webhook_task.py` 的 `send_webhook` 发出,签名 `HMAC-SHA256(secret, json.dumps(payload))`,与接收端一致;`data.project` 路由到 GWENT 正常。
|
||||||
|
- 实测(真实投递):评论 GW-25 → 01:24:34 投递到达 → 01:24:37 tick → 4.6 s 完成;GW-25 改 Done 同样 3 s 后 tick。建项 GW-26(01:25:14)→ 01:25:15 投递 → 01:25:18 tick 开始 → 01:26:49 `[worker] 第 1 轮开始`(win10 codex/astra):**建项 → 开工 95 s,其中 tick 本身 91 s**(中继、上传任务书、计划任务启动);只靠 cron 时在这之前还要平均多等约 60 s。剩下的成本都在中继调用上(PAD-12)。
|
||||||
|
- cron `*/2` 保留为兜底(同一把 `cron.lock`);oregon crontab 仍为空,只有一个调度器。
|
||||||
|
|
||||||
|
## 上线记录(2026-10-21,PAD-7:webhook 接收端)
|
||||||
|
- 13:25 LXC 上 `plane-agentd-webhook` 启用(`6dc7e72`),cron 不变。**Plane 侧 webhook 未建成**:Plane 3.3.3 对 `http://10.77.10.249:8790/plane` 返回 400 "The webhook URL is not allowed. Provide a publicly reachable HTTP or HTTPS URL."(SSRF 防护拒绝内网地址,见 design §10);需要在 Plane 的 api/worker 环境加 `WEBHOOK_ALLOWED_IPS=10.77.10.249/32`,之后按 deploy-lxc §8 第 2–5 步建 webhook。在此之前 Vault 里的 `PLANE_WEBHOOK_SECRET` 是临时随机值(`plane_wh_interim_…`),`plane-webhook.py create` 会用 Plane 生成的密钥覆盖它。
|
||||||
|
- 验收用的是在 LXC 上按 Plane 格式签名的合成投递(`/root/lxc_send.sh good|bad|missing|other`):错签名、缺签名 → 401,不触发;别的项目 → 202;正确签名 → 200,3 s 后 tick 开始。tick 运行中又来一条 → 结束后补跑一次(13:27 两条投递:224 s 的 tick——收尾 GW-15、在 win10 启动 GW-5——之后接着 24 s 的补跑);那段时间 cron 的 tick 撞锁立即退出,cron 本身照常每 2 分钟触发。
|
||||||
|
- 冒烟 GW-17(oregon claude-code):建项 13:43:15.8 → 合成投递 13:43:41(经 oregon→LXC 中继发出,人为多了约 25 s)→ tick 开始 13:43:44 → `[worker] 第 1 轮开始` 13:44:56 → `[review] ok` 13:47:54,PASS 并移到 Done。**投递 → tick 开始 = 3 s;tick 开始 → 开始评论 = 72 s**(这一 tick 先 poll win10 上在途的 GW-5,再经中继启动 oregon;同期 cron 的启动 tick 也要 70–77 s)。真实 Plane 投递约 1 s 到达时,建项 → 开始评论预计约 75 s,而只靠 cron 是平均约 60 s 等待 + 同样的 tick 时长。≤10 s 的目标在"开始评论"这个口径上达不到:瓶颈是中继调用(每次约 10 s,vault + Warpgate SSH)和启动流程,不是触发方式。
|
||||||
|
- GW-16(第一次冒烟):webhook 触发的 tick 里 LXC→中继报 `access-hub: config_private_transport_failed`(LXC 经 tailnet 取 `nas…/identity/agent.json` 15 s 超时;13:38 的 cron tick 也撞上一次),启动失败;下一 cron tick 探测无运行目录 → `GW-16-1-r2` 重新启动,PASS(防重复启动机制按设计工作)。
|
||||||
|
- 重启:tick 运行中 `systemctl restart plane-agentd-webhook`,tick(在 `systemd-run --scope` 里)没被杀、照常结束(heartbeat 13:50:10,errors=[]);接收端约 19 s 后重新监听(取 vault 密钥 + 启动)。
|
||||||
|
|
||||||
## 切换记录(2026-10-21,PAD-11:调度器迁到 LXC agentd-01)
|
## 切换记录(2026-10-21,PAD-11:调度器迁到 LXC agentd-01)
|
||||||
- 代码:防重复启动(`launching` 标记 + 探测接管/换目录重启)与"中继不通的终端本 tick 跳过"随本次一起上线(4f45d36);LXC 用仓库里的 `config/agentd.yaml`,oregon 回滚用 `config/agentd.oregon.yaml`。
|
- 代码:防重复启动(`launching` 标记 + 探测接管/换目录重启)与"中继不通的终端本 tick 跳过"随本次一起上线(4f45d36);LXC 用仓库里的 `config/agentd.yaml`,oregon 回滚用 `config/agentd.oregon.yaml`。
|
||||||
- 切换前 LXC 手工 `tick --once`(oregon cron 仍在跑,Plane 上无 Todo 项):空闲、无错误;LXC 经 helper 探测 win10(`codex` 计数)与经中继探测 oregon 运行目录均正常。
|
- 切换前 LXC 手工 `tick --once`(oregon cron 仍在跑,Plane 上无 Todo 项):空闲、无错误;LXC 经 helper 探测 win10(`codex` 计数)与经中继探测 oregon 运行目录均正常。
|
||||||
@@ -120,3 +184,14 @@ Todo + 指派给终端用户 → agentd 选最小编号、终端有空位、串
|
|||||||
- 冒烟(经 Warpgate 远程启动):GW-10 第 1 轮 claude-code/sonnet PASS(`[review] ok`,启动 08:08:54 → 结束 08:10:17,session `fa3be0f7…`);评论"请回复 `[review] ok 2`"并移回 Todo 后第 2 轮续同一 session(`out.json` 的 `session_id` 不变),回 `[review] ok 2` 转 Review。启动只用 1 次中继调用(约 12 s);72 KB 的 prompt 走分块上传(约 75 s)。
|
- 冒烟(经 Warpgate 远程启动):GW-10 第 1 轮 claude-code/sonnet PASS(`[review] ok`,启动 08:08:54 → 结束 08:10:17,session `fa3be0f7…`);评论"请回复 `[review] ok 2`"并移回 Todo 后第 2 轮续同一 session(`out.json` 的 `session_id` 不变),回 `[review] ok 2` 转 Review。启动只用 1 次中继调用(约 12 s);72 KB 的 prompt 走分块上传(约 75 s)。
|
||||||
- kill:直接经中继对 oregon 起 `sleep 600 & sleep 500` 的运行再 kill,两个 sleep 都被结束(进程组 + 会话),`pid.txt` 与 `launch.pid` 相同(setsid 没有再 fork)。经 Plane 的取消冒烟(运行中移到 Waiting)因 Plane 当时 502(08:14 起)未做,待补。
|
- kill:直接经中继对 oregon 起 `sleep 600 & sleep 500` 的运行再 kill,两个 sleep 都被结束(进程组 + 会话),`pid.txt` 与 `launch.pid` 相同(setsid 没有再 fork)。经 Plane 的取消冒烟(运行中移到 Waiting)因 Plane 当时 502(08:14 起)未做,待补。
|
||||||
|
|
||||||
|
|
||||||
|
## 回滚记录(2026-10-21:调度器回到 oregon)
|
||||||
|
- 原因:LXC agentd-01 经 tailnet 到 NAS 的路径走 DERP 中继,取 vault 身份端点(`nas…/identity/agent.json`)TCP 连接 30/30 超时,tick 里的中继调用不稳定。网络修好后再切回 LXC(docs/deploy-lxc.md §5)。
|
||||||
|
- 做法:LXC 删 cron 行、停用 `plane-agentd-webhook`、等 `agentd.lock`/`cron.lock` 空闲;`agentd-state.json`(sha256 7201077b…eb87,两端一致)拷到 oregon `~/.gwent-board/agentd/`(旧文件备份为 `.bak-<时间戳>`);oregon 恢复 `*/2` 的 `cron/agentd-tick.sh`。LXC 上的文件、服务单元均未删除。`config/agentd.oregon.yaml` 无需改动(`check-config` 通过)。
|
||||||
|
- 验证:oregon 首个 tick 14:20:01 接管,在途的 GW-5-1(win10 Codex)已结束,结果回帖并进入 Review;heartbeat `errors=[]`、`inflight=[]`,下一 tick(14:22)同样。
|
||||||
|
- 重新切回 LXC 之前:先 `crontab -r`(oregon)并等 tick 结束,再按 deploy-lxc §5 把状态拷回 LXC,并重新 enable webhook 服务;任何时刻只能有一个调度器。
|
||||||
|
|
||||||
|
## 切换记录(2026-10-21,网络修复后重新切回 LXC)
|
||||||
|
- 网络:OPNsense 增加出站 NAT 静态端口规则 `Tailscale agentd-01 static-port`(10.77.10.249/32,UDP,WAN),LXC tailscaled 改听 UDP 41642;LXC 到 NAS 直连(`tailscale ping` 5 ms via 223.102.82.206:1549,身份端点 30/30 成功、中位 0.089 s)。详见 deploy-lxc §2.1。
|
||||||
|
- 切换:oregon crontab 清空(备份 `~/.gwent-board/crontab.backup-20261021-152423-recut`),状态文件(sha256 `54b747db…7155`,两端一致;LXC 旧文件备份 `agentd-state.json.bak-20261021-152504`)拷到 LXC,装 LXC crontab `*/2 … cron/agentd-tick-lxc.sh`。在途的 GW-18 带过去(LXC 上为 GW-18-2),之后的 run-1 结果已回帖。heartbeat errors=[]、down=[]、skipped=[],只有一个调度器。
|
||||||
|
- Webhook(PAD-7)仍未启用:`plane-agentd-webhook` inactive/disabled;Plane 的 `WEBHOOK_ALLOWED_IPS` 尚未设置(见 deploy-lxc §8 的具体改法,Plane 实为 VM 506 / 10.77.10.69)。PLANE_WEBHOOK_SECRET 仍是临时值。待办:应用该改动后做 §8 第 2–5 步;考虑轮换 Plane 的 AES_SECRET_KEY(排查时被打印进了一次会话记录)。
|
||||||
|
|||||||
+105
@@ -0,0 +1,105 @@
|
|||||||
|
# 使用说明:在 Plane 上派活给各机器上的 agent
|
||||||
|
|
||||||
|
核对依据:`agentd/core.py`、`agentd/webhook.py`、`config/agentd.yaml`、LXC 上的实际状态、Plane GWENT 项目的标签和成员(2026-10-22)。代码或配置改了要同步改这里。
|
||||||
|
|
||||||
|
## 1. 派单需要的字段
|
||||||
|
- **必填**:状态设为 `Todo`,指派人里至少有一个终端账号(`window10` 或 `oregon`)。标题加描述就是任务书,原样进入 prompt。标签可选,模块与调度无关。
|
||||||
|
- 有多个指派人时,按 Plane 返回的顺序,第一个能对上终端的账号生效(`core.py` 的 `route()`)。所以"kappiee + window10"会派到 win10。
|
||||||
|
- **只指派 `kappiee`、指派 `mac`(暂无终端)或不指派**:`route()` 返回空,这一项被静默跳过。Plane 上不会有任何评论,只在 `heartbeat.json` 的 `skipped` 里记一条 `no route for assignee kappiee`,`agentd.log` 里在原因第一次出现时记一行。
|
||||||
|
- 只有 `Todo` 状态的项会被拾起(`pick()`)。Backlog、Waiting、Review 永远不会自动执行。唯一例外是处于 In Progress、等待临时性重试的项。
|
||||||
|
|
||||||
|
## 2. 终端、执行器和默认值
|
||||||
|
|
||||||
|
| Plane 指派人 | 终端 | 执行器 | 默认模型 / effort / 档位 | 运行目录(日志和输出) |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `window10` | win10(Tailscale 直连 SSH,Warpgate 兜底;Windows 计划任务) | codex(`codex exec`,`workspace-write`) | `gpt-6.1-sol` / `medium` / `fast` | `C:\…\gwent\runtime\agentd\<run_id>\`(`prompt.md run.ps1 events.jsonl last.md stderr.txt exit.txt`) |
|
||||||
|
| `oregon` | oregon(Tailscale 直连 SSH,Warpgate 兜底;`setsid nohup`) | claude-code(`claude -p --permission-mode acceptEdits`) | `sonnet` / (不传 effort)/ — | `/home/ubuntu/.gwent-board/agentd-runs/<run_id>/`(`out.json` 等) |
|
||||||
|
| `kappiee` / `mac` / 无人 | 不派 | — | — | — |
|
||||||
|
|
||||||
|
- 两个终端的并发都是 1,按终端计数,跨项目共享。
|
||||||
|
- win10 设了 `busy_processes: [codex]`:agentd 自己在 win10 上没有在途运行、却发现有 `codex.exe` 在跑时,不派单(`busy_process codex (… not started by agentd)`)。
|
||||||
|
- oregon 的运行直接在 `/home/ubuntu/Documents/wjj/project/gwent` 里改文件。这正是指挥所用的 checkout。
|
||||||
|
|
||||||
|
## 3. 代码里实际生效的标签
|
||||||
|
Plane 里目前只建了 4 个标签:`model:astra`、`model:sol`、`effort:high`、`needs-game`。其余前缀的标签要先在 Plane 里建出来才能挂上。
|
||||||
|
|
||||||
|
| 标签 | 效果(`route()`) |
|
||||||
|
|---|---|
|
||||||
|
| `model:astra` | 改用 gpt-6-astra,effort high(来自 by_label 配置) |
|
||||||
|
| `model:sol` | 改用 gpt-6.1-sol,tier fast |
|
||||||
|
| `model:<x>`(其他任意) | 模型名原样改成 x,例如 oregon 上的 `model:opus`。模型名与执行器相关:`model:astra` 挂在 oregon 的项上会让 claude 失败 |
|
||||||
|
| `effort:<x>` | 只对 codex 生效。claude 的命令模板里没有 `{effort}`,所以被忽略 |
|
||||||
|
| `tier:<x>` / `tier:none` | 只对 codex 生效,改档位或去掉档位参数 |
|
||||||
|
| `needs-game` | 串行组 `game`:同组同时只跑一项;env 类失败会暂停整组,直到该项离开 Waiting。**`needs-*` 不是通用前缀**,只有配置里写了的才生效 |
|
||||||
|
| `exec:<executor>` | 要求同名执行器存在且终端支持。现在每个终端只有一个执行器,例如 `exec:codex` 挂在 oregon 上只会得到 `executor codex not available` 跳过,等于没用 |
|
||||||
|
| `cap:<x>` | 终端的 `capabilities` 里必须有 x。win10 有 `[android, game]`,oregon 没有,不满足就跳过 |
|
||||||
|
| `budget:2h` / `budget:90m` | 覆盖超时时间,默认 4 h |
|
||||||
|
|
||||||
|
多个标签按顺序处理,后面的覆盖前面的。
|
||||||
|
|
||||||
|
## 4. 一轮的完整流程
|
||||||
|
1. **触发**:Plane 的工作区级 webhook(`issue` 和 `issue_comment` 事件)约 1 s 内送到 `http://10.77.10.249:8790/plane`。
|
||||||
|
- 接收端用 HMAC 校验签名,只认 GWENT 项目,其他项目返回 202 忽略。
|
||||||
|
- 等 3 s,把这期间的投递合并成一次 tick。tick 跑的就是 cron 脚本。
|
||||||
|
- tick 运行中又来投递时,结束后再补跑一次。锁被占用(退出码 75)时每 15 s 重试,最多 2 min。
|
||||||
|
- 兜底:cron 每 2 分钟一次 tick,同一把 `cron.lock`。
|
||||||
|
- agentd 自己回帖、改状态也会产生投递,所以每个有动作的 tick 后面都会多一次空 tick(实测 11.1 s)。
|
||||||
|
2. **tick 的步骤**:先 poll 所有在途运行(每个一次远程调用:直连亚秒到约 2 s,走 Warpgate 时约 10 s),再按编号从小到大挑 Todo 项。依次检查:终端是否可达、并发槽位、串行组、busy 进程。
|
||||||
|
- 启动前先把 `launching` 标记落盘,防止重复启动。
|
||||||
|
- 启动后把项改为 In Progress 并指派给终端账号,以终端账号发评论 `[worker] 第 k 轮开始:codex / gpt-6-astra / high @ win10,新线程,run=GW-26-1。`
|
||||||
|
3. **实测耗时**(runbook 记录和 LXC 的 webhook.log):
|
||||||
|
- 建项到投递约 1 s,再加 3 s 合并窗口。
|
||||||
|
- **win10 启动的那次 tick 91.4 s**(GW-26:建项 01:25:14,开工评论 01:26:49)。
|
||||||
|
- oregon 启动的 tick 约 72 s。
|
||||||
|
- 空 tick 4.6–4.8 s。
|
||||||
|
- 以上是 Warpgate 时期的数字,瓶颈在中继调用,与触发方式无关。
|
||||||
|
- PAD-12 起两个终端都走 Tailscale 直连:win10 启动由约 10 次中继调用变成 busy 检查 + 1 次启动调用,收尾由约 7 次变成 2 次。目标是建项到开工评论 ≤ 15 s,上线后在这里补实测。直连不通时这一轮自动走 Warpgate,只是慢,派单行为不变(design §9)。
|
||||||
|
4. **结束时**:以终端账号回帖。
|
||||||
|
- 第一行是 `[codex] 第 k 轮结束(exit=…,run=…,终端 win10,日志 <运行目录>)`,下面是 agent 的最后消息原文(`<pre>` 包裹,超过 6000 字截断,并注明全文所在路径)。
|
||||||
|
- 再按最后消息首行的标签流转(`transition()`):
|
||||||
|
|
||||||
|
| 首行 | 状态 / 指派人 |
|
||||||
|
|---|---|
|
||||||
|
| `[review]` | Review,指派 `oregon`(reviewer) |
|
||||||
|
| `[question]` / `[blocked]` | Waiting,指派 `oregon`(answerer) |
|
||||||
|
| `[progress]` | 回到 Todo,留在原终端,下一 tick 续同一线程。连续 12 轮后转 Waiting |
|
||||||
|
| 没有标签但 exit=0 | 按 review 处理,并注明"未按协议标注首行" |
|
||||||
|
|
||||||
|
注意:Waiting 和 Review 都指派给 **oregon,不是 kappiee**。没有任何东西会自动处理这两个状态,由指挥(或你)介入。
|
||||||
|
5. **失败**:回帖 `[worker] 第 k 轮失败:类型=crash|timeout|env|logic,run,exit,重试 n/3。…` 并附上最后一帧输出。
|
||||||
|
- 临时性错误(stderr 尾部匹配超时、429、5xx、连接被拒等)自动重试,退避 2/5/10 min,续原线程,状态保持 In Progress。
|
||||||
|
- 续线程那一轮出现非临时性失败时,立即在新线程重跑一次,并附交接摘要。
|
||||||
|
- 其余情况转 Waiting,指派 oregon。超时会先 kill 进程树。逻辑类失败永不自动重试。
|
||||||
|
|
||||||
|
## 5. 回答问题和续跑
|
||||||
|
1. 在项上发评论回答,开头不要写 `[worker]`,这类评论永远会被跳过。
|
||||||
|
2. **把状态改回 Todo,并把指派人改回原终端**(例如 `window10`)。这一步是坑:如果指派人还是 oregon 就改 Todo,会派给 oregon 的 Claude。因为终端或执行器变了,会**重建线程**并附交接摘要。
|
||||||
|
3. 同一终端、同一执行器、线程续了不到 8 轮时,走 `codex exec resume` 或 `claude --resume`,只发送上次之后的新评论(按评论 ID 判断)。
|
||||||
|
4. 单发评论只会触发一次空 tick,**不会**让 agent 动起来,必须改回 Todo。运行期间发的评论,跑着的那一轮看不到,下一轮才带上。
|
||||||
|
|
||||||
|
## 6. 取消和停止
|
||||||
|
- **把运行中的项移出 In Progress**(改成 Waiting、Backlog、Cancelled 都可以)。下一 tick 会 kill 进程树,回帖 `[worker] 运行已取消:工作项状态被改为 X(run=…)`,释放槽位和串行组,保留线程 ID(`cancel()`)。
|
||||||
|
- 如果移到的是 **Todo**:同一 tick 里先取消,再立刻按新一轮重新派发(续原线程),相当于"带上新评论重启"。
|
||||||
|
- 运行已经结束时(`exit.txt` 已写)再移动:结果照常回帖,但不再改状态,也不重试。
|
||||||
|
- 暂停整个终端:把它的 `concurrency` 设为 0(需要改配置,不能在 Plane 里操作)。
|
||||||
|
|
||||||
|
## 7. 日志和状态的位置(LXC)
|
||||||
|
- `/var/log/plane-agentd/{agentd.log, cron.log, webhook.log}`
|
||||||
|
- `/var/lib/plane-agentd/{agentd-state.json, heartbeat.json}`:heartbeat 里有 `skipped` 及原因、`inflight`、`down`。
|
||||||
|
- 健康检查:`GET http://10.77.10.249:8790/healthz`
|
||||||
|
- 任务产出本身在终端的 repo 里,gwent 的惯例是 win10 的 `runtime/edits/GW-k/`。
|
||||||
|
|
||||||
|
|
||||||
|
## 8. 用手机试一下(冒烟测试,5 步)
|
||||||
|
1. GWENT 新建工作项,标题 `冒烟 oregon 往返`。描述:`只回复一行 [review] ok;不要读写或修改任何文件,不要运行命令。` oregon 的 Claude 是 acceptEdits,并且在指挥用的 checkout 里运行,所以这句必须写。
|
||||||
|
2. 指派人**只选 oregon**,不加标签(默认 sonnet),状态设为 **Todo**。
|
||||||
|
3. 约 1–1.5 分钟后:状态变成 In Progress,出现 oregon 发的评论 `[worker] 第 1 轮开始:claude-code / sonnet @ oregon,新线程,run=GW-n-1。`
|
||||||
|
4. 再过约 1–3 分钟(历次冒烟 1.5–3 min):出现评论 `[claude-code] 第 1 轮结束(exit=0 …)` 加 `[review] ok`,状态变为 Review、指派 oregon。
|
||||||
|
5. 可选,测续线程:评论 `请只回复 [review] ok 2`,状态改回 Todo(指派保持 oregon)。开始评论会写"续线程 xxxx",结束后回 `[review] ok 2`。最后手动改成 Done。
|
||||||
|
- 超过 3 分钟没动静:看 heartbeat 的 `skipped`,常见原因是 oregon 槽位被占,或指派人不对。
|
||||||
|
|
||||||
|
## 9. 通知和手机 App(2026-10-22 核实)
|
||||||
|
- Plane 官方手机 App 可以登录自建的 Commercial 版(选 "Self hosted sign in",填 `https://plane.kappiee.com`)。plane.kappiee.com 只对内网和 tailnet 开放,手机要开 Tailscale。
|
||||||
|
- **自建实例没有手机推送**:Plane 文档原文 "Push notifications are currently available only for Plane Cloud users."(docs.plane.so/devices/mobile)。App 里只能在收件箱看到通知。
|
||||||
|
- 收件箱只收你被指派、被 @、创建或订阅的项。Waiting 和 Review 现在都指派给 oregon,不会进你的收件箱。需要你处理的项,由指挥改成指派 kappiee 或 @你。
|
||||||
|
- 真正推到手机要走外部渠道:给 Plane 配 SMTP 发邮件,或用 ntfy 之类。计划放在最后(PAD-13)。agentd 里现在的 `notify: {kind: none}` 是空钩子。
|
||||||
@@ -0,0 +1,827 @@
|
|||||||
|
"""direct-ssh (PAD-12): config validation, channel selection and fallback rules, the ssh command line, the Windows
|
||||||
|
bootstrap / payload, single-call launch and collect. Nothing here opens a network connection: the POSIX direct
|
||||||
|
channel is replaced by a local `bash -c <loader>` fed on stdin, so the real loader and scripts run."""
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from contextlib import redirect_stdout
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from agentd import ROOT, cli, config
|
||||||
|
from agentd.config import Config
|
||||||
|
from agentd.executors import Run, make_executor
|
||||||
|
from agentd.transports import (CallLog, HostKeyError, RoutedTransport, TransportError, Unreached, make_transport,
|
||||||
|
parse_fetch)
|
||||||
|
from agentd.transports import direct_ssh
|
||||||
|
from agentd.transports.direct_ssh import (BOOTSTRAP, PS_COMMAND, SH_COMMAND, DirectChannel, KeyAgent, classify,
|
||||||
|
ps_payload)
|
||||||
|
from agentd.transports.warpgate_ssh import PosixWarpgateTransport, WarpgateSshTransport, WindowsWarpgateTransport
|
||||||
|
from tests.fakes import config_data
|
||||||
|
|
||||||
|
SCHEMA = json.loads((ROOT / "config" / "schema.json").read_text())
|
||||||
|
WIN_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzLX7QNHDFYwTgY4wNl0ihZryIyKgpp9HPpPDWbrTX3"
|
||||||
|
LIN_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICUoI6W7npX+h/kHcXX3iy2XAWbNusrOjg28Xed75wWr"
|
||||||
|
SECRET = "-----BEGIN OPENSSH PRIVATE KEY-----\nNOT-A-REAL-KEY-but-must-never-leak\n-----END OPENSSH PRIVATE KEY-----\n"
|
||||||
|
|
||||||
|
|
||||||
|
def direct_data(root="/srv/x"):
|
||||||
|
d = config_data()
|
||||||
|
d["vault"] = {"client": "/opt/vault/agent-access.py", "config": "https://vault.example/identity.json"}
|
||||||
|
d["agentd"]["ssh_dir"] = str(Path(root) / "ssh")
|
||||||
|
d["terminals"]["win10"].update(transport="direct-ssh", fallback="warpgate-ssh",
|
||||||
|
ssh={"host": "100.123.209.19", "user": "administrator", "port": 22, "host_key": WIN_KEY})
|
||||||
|
d["terminals"]["lin"] = {"transport": "direct-ssh", "target": "lin", "os": "linux", "launch": "nohup",
|
||||||
|
"repo": str(Path(root) / "repo"), "run_root": str(Path(root) / "runs"), "plane_user": "oregon",
|
||||||
|
"token_key": "TOK_LIN", "concurrency": 1, "executors": ["claude-code", "shell"],
|
||||||
|
"ssh": {"host": "100.127.58.78", "user": "ubuntu", "host_key": LIN_KEY},
|
||||||
|
"fallback": "warpgate-ssh"}
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
class FakeAgent:
|
||||||
|
sock, pub, error = "/tmp/agentd-agent-x/agent.sock", "/tmp/agentd-agent-x/agentd.pub", None
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigTest(unittest.TestCase):
|
||||||
|
def errs(self, d):
|
||||||
|
return config.validate(d)
|
||||||
|
|
||||||
|
def test_direct_config_is_valid_and_live_config_uses_it(self):
|
||||||
|
self.assertEqual(self.errs(direct_data()), [])
|
||||||
|
self.assertEqual(config.mini_validate(direct_data(), SCHEMA), [])
|
||||||
|
cfg = config.load(ROOT / "config" / "agentd.yaml")
|
||||||
|
for term in ("win10", "oregon"):
|
||||||
|
t = cfg.terminals[term]
|
||||||
|
self.assertEqual((t["transport"], t["fallback"], t["target"]), ("direct-ssh", "warpgate-ssh", term))
|
||||||
|
self.assertIsNone(config.host_key_error(t["ssh"]["host_key"]))
|
||||||
|
self.assertIn("--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY", cfg.secret_env_args())
|
||||||
|
config.load(ROOT / "config" / "agentd.oregon.yaml") # rollback config still valid
|
||||||
|
|
||||||
|
def test_bad_direct_settings_have_clear_errors(self):
|
||||||
|
cases = [
|
||||||
|
(lambda d: d["terminals"]["lin"].pop("ssh"), "direct-ssh needs ssh:"),
|
||||||
|
(lambda d: d["terminals"]["lin"]["ssh"].update(host_key="ssh-rsa AAAAB3Nza"), "must be 'ssh-ed25519 <base64>'"),
|
||||||
|
(lambda d: d["terminals"]["lin"]["ssh"].update(host_key=WIN_KEY[:-4]), "not an ssh-ed25519 public key"),
|
||||||
|
(lambda d: d["terminals"]["lin"]["ssh"].update(host_key="ssh-ed25519 !!!"), "not an ssh-ed25519 public key"),
|
||||||
|
(lambda d: d["terminals"]["lin"].pop("target"), "needs the Warpgate 'target'"),
|
||||||
|
(lambda d: d["terminals"]["lin"]["ssh"].update(host="-oProxyCommand=x"), "host and user must be non-empty"),
|
||||||
|
(lambda d: d["terminals"]["lin"]["ssh"].update(port=70000), "> 65535"),
|
||||||
|
(lambda d: d["terminals"]["lin"].update(run_root="~/runs"), "must be an absolute path"),
|
||||||
|
(lambda d: d["terminals"]["win10"].update(launch="nohup"), "direct-ssh on windows needs launch=schtasks"),
|
||||||
|
(lambda d: d["terminals"]["win10"].update(target="win11"), "(fallback) only reach target 'win10'"),
|
||||||
|
(lambda d: d["vault"].pop("config"), "fallback warpgate-ssh needs vault.config"),
|
||||||
|
(lambda d: d["terminals"]["oregon"].update(fallback="warpgate-ssh"), "only applies to transport direct-ssh"),
|
||||||
|
(lambda d: d["agentd"].update(ssh_dir="/" + "x" * 60), "too long for the ControlMaster socket"),
|
||||||
|
]
|
||||||
|
for mutate, fragment in cases:
|
||||||
|
d = direct_data()
|
||||||
|
mutate(d)
|
||||||
|
errs = self.errs(d)
|
||||||
|
self.assertTrue(any(fragment in e for e in errs), (fragment, errs))
|
||||||
|
|
||||||
|
def test_schema_rejects_unknown_ssh_key_and_fallback_value(self):
|
||||||
|
d = direct_data()
|
||||||
|
d["terminals"]["lin"]["ssh"]["password"] = "x"
|
||||||
|
self.assertTrue(any("password" in e for e in config.mini_validate(d, SCHEMA)))
|
||||||
|
d = direct_data()
|
||||||
|
d["terminals"]["lin"]["fallback"] = "local"
|
||||||
|
self.assertTrue(config.mini_validate(d, SCHEMA))
|
||||||
|
|
||||||
|
def test_key_env_is_always_scrubbed_and_only_requested_when_used(self):
|
||||||
|
cfg = Config(config_data()) # no direct-ssh terminal
|
||||||
|
self.assertNotIn("--secret-env AGENTD_SSH_KEY=AGENTD_SSH_KEY", cfg.secret_env_args())
|
||||||
|
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET}):
|
||||||
|
self.assertNotIn("AGENTD_SSH_KEY", cfg.scrubbed_env())
|
||||||
|
d = direct_data()
|
||||||
|
d["agentd"]["ssh_key"] = "OTHER_KEY"
|
||||||
|
self.assertEqual(Config(d).secret_env_args()[-1], "--secret-env OTHER_KEY=OTHER_KEY")
|
||||||
|
|
||||||
|
|
||||||
|
class SelectionTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.cfg = Config(direct_data())
|
||||||
|
|
||||||
|
def test_direct_terminal_gets_direct_primary_and_warpgate_fallback(self):
|
||||||
|
t = make_transport(self.cfg, "win10")
|
||||||
|
self.assertIsInstance(t, RoutedTransport)
|
||||||
|
self.assertIsInstance(t.primary, WindowsWarpgateTransport)
|
||||||
|
self.assertIsInstance(t.primary.channel, DirectChannel)
|
||||||
|
self.assertEqual((t.fallback.channel.via, t.fallback.channel.stdin, t.fallback.target), ("fallback", False, "win10"))
|
||||||
|
self.assertEqual(t.os, "windows")
|
||||||
|
self.assertTrue(hasattr(t, "start_run"))
|
||||||
|
|
||||||
|
def test_forced_channel_has_no_fallback(self):
|
||||||
|
self.assertIsNone(make_transport(self.cfg, "lin", force="direct-ssh").fallback)
|
||||||
|
t = make_transport(self.cfg, "lin", force="warpgate-ssh")
|
||||||
|
self.assertIsInstance(t, PosixWarpgateTransport)
|
||||||
|
self.assertEqual(t.channel.via, "warpgate")
|
||||||
|
|
||||||
|
def test_rollback_to_warpgate_keeps_ssh_section(self):
|
||||||
|
d = direct_data()
|
||||||
|
d["terminals"]["lin"]["transport"] = "warpgate-ssh" # ssh/fallback stay, ignored
|
||||||
|
self.assertEqual(config.validate(d), [])
|
||||||
|
self.assertIsInstance(make_transport(Config(d), "lin"), PosixWarpgateTransport)
|
||||||
|
|
||||||
|
|
||||||
|
class ClassifyTest(unittest.TestCase):
|
||||||
|
def test_classification(self):
|
||||||
|
self.assertEqual(classify(0, ""), "ok")
|
||||||
|
self.assertEqual(classify(1, "ssh: connect to host x port 22: Connection timed out"), "ok") # command ran
|
||||||
|
for err in ("ssh: connect to host 100.1.2.3 port 22: Connection timed out",
|
||||||
|
"ssh: connect to host 100.1.2.3 port 22: Connection refused",
|
||||||
|
"ssh: connect to host 100.1.2.3 port 22: No route to host",
|
||||||
|
"ssh: Could not resolve hostname x: Name or service not known",
|
||||||
|
"Connection timed out during banner exchange",
|
||||||
|
"kex_exchange_identification: read: Connection reset by peer",
|
||||||
|
"Connection closed by 100.1.2.3 port 22",
|
||||||
|
"ubuntu@100.1.2.3: Permission denied (publickey).",
|
||||||
|
"sign_and_send_pubkey: signing failed for ED25519 \"x\" from agent: agent refused operation"):
|
||||||
|
self.assertEqual(classify(255, err), "unreached", err)
|
||||||
|
for err in ("@@@@@@@@@@@\n@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @\nHost key verification failed.",
|
||||||
|
"No ED25519 host key is known for agentd-win10 and you have requested strict checking.\n"
|
||||||
|
"Host key verification failed."):
|
||||||
|
self.assertEqual(classify(255, err), "hostkey")
|
||||||
|
for err in ("Connection to 100.1.2.3 closed by remote host.", "Timeout, server 100.1.2.3 not responding.",
|
||||||
|
"client_loop: send disconnect: Connection reset by peer", ""):
|
||||||
|
self.assertEqual(classify(255, err), "unknown", err) # the command may have run: no fallback
|
||||||
|
|
||||||
|
|
||||||
|
class Prim:
|
||||||
|
"""Stand-in transport recording calls; `fail` = exception raised by every call."""
|
||||||
|
os = "linux"
|
||||||
|
|
||||||
|
def __init__(self, fail=None):
|
||||||
|
self.fail, self.calls = fail, []
|
||||||
|
|
||||||
|
def join(self, *p):
|
||||||
|
return "/".join(p)
|
||||||
|
|
||||||
|
def start_run(self, *a):
|
||||||
|
self.calls.append(("start_run", a))
|
||||||
|
if self.fail:
|
||||||
|
raise self.fail
|
||||||
|
return "ok"
|
||||||
|
|
||||||
|
def process_count(self, name):
|
||||||
|
self.calls.append(("process_count", name))
|
||||||
|
if self.fail:
|
||||||
|
raise self.fail
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
class RoutedTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.events = []
|
||||||
|
self.log = CallLog(event=self.events.append)
|
||||||
|
|
||||||
|
def test_unreached_falls_back_once_and_sticks(self):
|
||||||
|
p, f = Prim(Unreached("ssh: connect to host x: Connection timed out")), Prim()
|
||||||
|
t = RoutedTransport("lin", p, f, self.log)
|
||||||
|
self.assertEqual(t.start_run("task", "/r", {}, "run.sh"), "ok")
|
||||||
|
self.assertEqual(t.process_count("codex"), 0)
|
||||||
|
self.assertEqual(len(p.calls), 1) # direct not retried within the tick
|
||||||
|
self.assertEqual([c[0] for c in f.calls], ["start_run", "process_count"])
|
||||||
|
self.assertEqual(len(self.events), 1)
|
||||||
|
self.assertIn("fallback", self.events[0])
|
||||||
|
self.assertEqual(t.join("a", "b"), "a/b") # pure helpers come from the primary
|
||||||
|
|
||||||
|
def test_host_key_and_after_command_failures_never_fall_back(self):
|
||||||
|
for exc in (HostKeyError("host key verification FAILED"), TransportError("ssh timeout after 150s")):
|
||||||
|
p, f = Prim(exc), Prim()
|
||||||
|
t = RoutedTransport("lin", p, f, self.log)
|
||||||
|
with self.assertRaises(type(exc)):
|
||||||
|
t.start_run("task", "/r", {}, "run.sh")
|
||||||
|
self.assertEqual(f.calls, [])
|
||||||
|
|
||||||
|
def test_no_fallback_configured_raises(self):
|
||||||
|
with self.assertRaises(Unreached):
|
||||||
|
RoutedTransport("lin", Prim(Unreached("down")), None, self.log).start_run("t", "/r", {}, "run.sh")
|
||||||
|
|
||||||
|
def test_busy_check_unreached_is_not_swallowed(self):
|
||||||
|
cfg = Config(direct_data())
|
||||||
|
for cls, term in ((PosixWarpgateTransport, "lin"), (WindowsWarpgateTransport, "win10")):
|
||||||
|
ch = mock.Mock(stdin=True)
|
||||||
|
ch.sh.side_effect = ch.ps.side_effect = Unreached("down")
|
||||||
|
with self.assertRaises(Unreached):
|
||||||
|
cls(cfg, term, channel=ch).process_count("codex")
|
||||||
|
ch.sh.side_effect = ch.ps.side_effect = TransportError("exit 1")
|
||||||
|
self.assertIsNone(cls(cfg, term, channel=ch).process_count("codex"))
|
||||||
|
|
||||||
|
|
||||||
|
class MissingKeyTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
direct_ssh._AGENTS.clear()
|
||||||
|
direct_ssh._KEYS.clear()
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.cfg = Config(direct_data(self.tmp.name))
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
direct_ssh._AGENTS.clear()
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_missing_key_makes_direct_unavailable_and_uses_the_fallback(self):
|
||||||
|
events = []
|
||||||
|
log = CallLog(event=events.append)
|
||||||
|
env = {k: v for k, v in os.environ.items() if k != "AGENTD_SSH_KEY"}
|
||||||
|
with mock.patch.dict(os.environ, env, clear=True), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.subprocess.run") as run:
|
||||||
|
t = make_transport(self.cfg, "lin", log=log)
|
||||||
|
t.fallback = Prim()
|
||||||
|
self.assertEqual(t.process_count("claude"), 0)
|
||||||
|
self.assertEqual(t.process_count("claude"), 0)
|
||||||
|
run.assert_not_called() # no ssh attempted at all
|
||||||
|
self.assertTrue(any("AGENTD_SSH_KEY is not set" in e for e in events))
|
||||||
|
self.assertEqual(sum("unavailable in this process" in e for e in events), 1) # once per process
|
||||||
|
self.assertEqual(sum("using the warpgate-ssh fallback" in e for e in events), 1) # once per tick
|
||||||
|
|
||||||
|
def test_empty_key_is_popped_and_reported(self):
|
||||||
|
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": " "}):
|
||||||
|
a = KeyAgent("AGENTD_SSH_KEY").start()
|
||||||
|
self.assertNotIn("AGENTD_SSH_KEY", os.environ)
|
||||||
|
self.assertIn("not set", a.error)
|
||||||
|
|
||||||
|
@unittest.skipUnless(all(shutil.which(b) for b in ("ssh-agent", "ssh-add", "ssh-keygen", "ssh")), "needs OpenSSH")
|
||||||
|
def test_real_agent_loads_key_from_env_and_cleans_up(self):
|
||||||
|
keyfile = Path(self.tmp.name) / "k"
|
||||||
|
subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "agentd-test", "-f", str(keyfile)], check=True)
|
||||||
|
key = keyfile.read_text()
|
||||||
|
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": key.replace("\n", "\\n")}): # Vault-style escapes
|
||||||
|
a = KeyAgent("AGENTD_SSH_KEY").start()
|
||||||
|
self.assertNotIn("AGENTD_SSH_KEY", os.environ)
|
||||||
|
try:
|
||||||
|
self.assertIsNone(a.error)
|
||||||
|
self.assertEqual(oct(os.stat(a.dir).st_mode & 0o777), "0o700")
|
||||||
|
self.assertEqual(Path(a.pub).read_text().split()[:2], (keyfile.with_suffix(".pub")).read_text().split()[:2])
|
||||||
|
for f in Path(a.dir).iterdir(): # nothing on disk holds the private key
|
||||||
|
if f.is_file():
|
||||||
|
self.assertNotIn("PRIVATE KEY", f.read_text())
|
||||||
|
finally:
|
||||||
|
a.close()
|
||||||
|
self.assertFalse(os.path.exists(a.dir))
|
||||||
|
self.assertIsNotNone(a.proc.poll())
|
||||||
|
|
||||||
|
def test_bad_key_reports_without_leaking(self):
|
||||||
|
if not all(shutil.which(b) for b in ("ssh-agent", "ssh-add", "ssh")):
|
||||||
|
self.skipTest("needs OpenSSH")
|
||||||
|
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET}):
|
||||||
|
a = KeyAgent("AGENTD_SSH_KEY").start()
|
||||||
|
self.assertIn("ssh-add failed", a.error)
|
||||||
|
self.assertNotIn("NOT-A-REAL-KEY", a.error)
|
||||||
|
self.assertFalse(os.path.exists(a.dir)) # agent stopped, socket dir gone
|
||||||
|
|
||||||
|
|
||||||
|
class SshCommandTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.cfg = Config(direct_data(self.tmp.name))
|
||||||
|
self.events, self.lines = [], []
|
||||||
|
self.log = CallLog(event=self.events.append)
|
||||||
|
self.log.call = lambda *a, **k: self.lines.append(a)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def run_with(self, rc, err=b"", out=b"PONG=x\n"):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def fake_run(argv, input=None, stdout=None, stderr=None, env=None, timeout=None):
|
||||||
|
seen.update(argv=argv, input=input, env=env, timeout=timeout)
|
||||||
|
stdout.write(out)
|
||||||
|
stderr.write(err)
|
||||||
|
return mock.Mock(returncode=rc)
|
||||||
|
return seen, fake_run
|
||||||
|
|
||||||
|
def test_command_line_options_and_no_secret(self):
|
||||||
|
ch = DirectChannel(self.cfg, "lin", self.log)
|
||||||
|
seen, fake = self.run_with(0)
|
||||||
|
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET, "TOK_LIN": "plane-token", "SSH_AUTH_SOCK": "/x"}), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake):
|
||||||
|
rc, out, _ = ch.sh("echo hi", 30, "ping")
|
||||||
|
argv = seen["argv"]
|
||||||
|
opts = {argv[i + 1] for i, a in enumerate(argv) if a == "-o"}
|
||||||
|
ssh_dir = Path(self.tmp.name) / "ssh"
|
||||||
|
for o in ("BatchMode=yes", "IdentitiesOnly=yes", "IdentityAgent=" + FakeAgent.sock, "IdentityFile=" + FakeAgent.pub,
|
||||||
|
"StrictHostKeyChecking=yes", "UserKnownHostsFile=%s/known_hosts.lin" % ssh_dir,
|
||||||
|
"GlobalKnownHostsFile=/dev/null", "HostKeyAlias=agentd-lin", "HostKeyAlgorithms=ssh-ed25519",
|
||||||
|
"ConnectTimeout=8", "ServerAliveInterval=15", "ServerAliveCountMax=3", "ForwardAgent=no",
|
||||||
|
"ForwardX11=no", "LogLevel=ERROR", "ControlMaster=auto", "ControlPersist=60",
|
||||||
|
"ControlPath=%s/%s-%%C" % (ssh_dir, ch.pin), "PasswordAuthentication=no"):
|
||||||
|
self.assertIn(o, opts)
|
||||||
|
self.assertEqual(argv[:3], ["ssh", "-F", "/dev/null"])
|
||||||
|
self.assertEqual(argv[-3:], ["--", "100.127.58.78", SH_COMMAND])
|
||||||
|
self.assertEqual(argv[argv.index("-l") + 1], "ubuntu")
|
||||||
|
self.assertEqual(seen["input"], b"echo hi") # the script travels on stdin
|
||||||
|
self.assertFalse(any("NOT-A-REAL-KEY" in a or "plane-token" in a for a in argv))
|
||||||
|
self.assertNotIn("AGENTD_SSH_KEY", seen["env"])
|
||||||
|
self.assertNotIn("TOK_LIN", seen["env"])
|
||||||
|
self.assertNotIn("SSH_AUTH_SOCK", seen["env"])
|
||||||
|
self.assertEqual((Path(ssh_dir) / "known_hosts.lin").read_text(), "agentd-lin %s\n" % LIN_KEY)
|
||||||
|
self.assertEqual(oct(os.stat(ssh_dir).st_mode & 0o777), "0o700")
|
||||||
|
self.assertEqual((rc, out), (0, "PONG=x\n"))
|
||||||
|
self.assertEqual(self.lines[-1][:3], ("lin", "direct", "ping"))
|
||||||
|
self.assertEqual(self.lines[-1][4], 0)
|
||||||
|
|
||||||
|
def test_host_key_failure_is_loud_sticky_and_not_unreached(self):
|
||||||
|
ch = DirectChannel(self.cfg, "lin", self.log)
|
||||||
|
seen, fake = self.run_with(255, b"No ED25519 host key is known for agentd-lin and you have requested strict "
|
||||||
|
b"checking.\r\nHost key verification failed.\r\n", b"")
|
||||||
|
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.subprocess.run", side_effect=fake) as run:
|
||||||
|
with self.assertRaises(HostKeyError) as cm:
|
||||||
|
ch.sh("true", 30)
|
||||||
|
self.assertNotIsInstance(cm.exception, Unreached)
|
||||||
|
with self.assertRaises(HostKeyError):
|
||||||
|
ch.sh("true", 30)
|
||||||
|
self.assertEqual(run.call_count, 1) # no second attempt in this tick
|
||||||
|
self.assertTrue(self.events and self.events[0].startswith("SECURITY lin:"))
|
||||||
|
|
||||||
|
def test_connect_failure_is_unreached_and_timeout_is_not(self):
|
||||||
|
ch = DirectChannel(self.cfg, "lin", self.log)
|
||||||
|
_, fake = self.run_with(255, b"ssh: connect to host 100.127.58.78 port 22: Connection timed out\n", b"")
|
||||||
|
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake):
|
||||||
|
with self.assertRaises(Unreached):
|
||||||
|
ch.sh("true", 30)
|
||||||
|
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.subprocess.run",
|
||||||
|
side_effect=subprocess.TimeoutExpired("ssh", 30)):
|
||||||
|
with self.assertRaises(TransportError) as cm:
|
||||||
|
ch.sh("true", 30)
|
||||||
|
self.assertNotIsInstance(cm.exception, Unreached) # may have run: resolve_launch probes instead
|
||||||
|
|
||||||
|
def test_windows_ps_uses_bootstrap_and_stdin(self):
|
||||||
|
ch = DirectChannel(self.cfg, "win10", self.log)
|
||||||
|
seen, fake = self.run_with(0, out=b"X=ok\r\n")
|
||||||
|
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake):
|
||||||
|
self.assertEqual(ch.ps("'X=ok'", 30), "X=ok\n")
|
||||||
|
self.assertEqual(seen["argv"][-2:], ["100.123.209.19", PS_COMMAND])
|
||||||
|
self.assertEqual(base64.b64decode(seen["input"]).decode("utf-8"), "'X=ok'")
|
||||||
|
|
||||||
|
|
||||||
|
class BootstrapTest(unittest.TestCase):
|
||||||
|
def test_encoded_command_round_trip_and_length(self):
|
||||||
|
value = PS_COMMAND.split()[-1]
|
||||||
|
self.assertEqual(base64.b64decode(value).decode("utf-16-le"), BOOTSTRAP)
|
||||||
|
self.assertLess(len(PS_COMMAND), 2000) # cmd.exe limit is 8191
|
||||||
|
self.assertIn("[Console]::In.ReadToEnd()", BOOTSTRAP)
|
||||||
|
script = "'é 中文 ''q'''\n"
|
||||||
|
self.assertTrue(ps_payload(script).isascii())
|
||||||
|
self.assertEqual(base64.b64decode(ps_payload(script)).decode("utf-8"), script)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeWinChannel:
|
||||||
|
"""Direct-like Windows channel: answers the start / fetch scripts the way PowerShell would."""
|
||||||
|
stdin, name, via = True, "ssh fake", "direct"
|
||||||
|
|
||||||
|
def __init__(self, files=None, haspid=False):
|
||||||
|
self.calls, self.files, self.haspid = [], files or {}, haspid
|
||||||
|
|
||||||
|
def ps(self, script, timeout, op=None):
|
||||||
|
self.calls.append((op, script))
|
||||||
|
if op == "start":
|
||||||
|
if self.haspid:
|
||||||
|
return "HASPID\n"
|
||||||
|
out = ["WROTE=" + hashlib.sha256(base64.b64decode(b)).hexdigest()
|
||||||
|
for b in __import__("re").findall(r"FromBase64String\('([A-Za-z0-9+/=]*)'\)", script)]
|
||||||
|
return "\n".join(out + ["LAUNCHED=0"]) + "\n"
|
||||||
|
if op == "fetch":
|
||||||
|
ops = [l.split()[:4] for l in script.splitlines() if l.split(" ")[0] in ("AdRead", "AdGrep", "AdTail")]
|
||||||
|
out = []
|
||||||
|
for fn, i, name, *rest in ops:
|
||||||
|
text = self.files.get(name.strip("'"))
|
||||||
|
if fn == "AdRead":
|
||||||
|
out.append("M%s" % i if text is None else "F%s=%s" % (i, base64.b64encode(text.encode()).decode()))
|
||||||
|
else:
|
||||||
|
if fn == "AdGrep":
|
||||||
|
text = next((l for l in (text or "").splitlines() if "thread.started" in l), "")
|
||||||
|
else:
|
||||||
|
text = "\n".join((text or "").splitlines()[-int(rest[0]):])
|
||||||
|
out.append("F%s=%s" % (i, base64.b64encode(text.encode()).decode()))
|
||||||
|
return "\n".join(out) + "\n"
|
||||||
|
return "FORGOT=ok\n"
|
||||||
|
|
||||||
|
|
||||||
|
class WindowsDirectTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.cfg = config.load(ROOT / "config" / "agentd.yaml")
|
||||||
|
|
||||||
|
def transport(self, ch):
|
||||||
|
return WarpgateSshTransport(self.cfg, "win10", channel=ch)
|
||||||
|
|
||||||
|
def test_launch_is_one_call_with_exact_utf8_files(self):
|
||||||
|
ch = FakeWinChannel()
|
||||||
|
t = self.transport(ch)
|
||||||
|
ex = make_executor(self.cfg, "win10", "codex", t)
|
||||||
|
prompt = "中文 'quoted' \"dq\" $env:X `tick`\r\n" * 2000 # ~70 KB: far beyond any argv limit
|
||||||
|
h = ex.start(Run("GW-26-1", t.repo, prompt, "gpt-6-astra", "high"))
|
||||||
|
self.assertEqual([c[0] for c in ch.calls], ["start"])
|
||||||
|
script = ch.calls[0][1]
|
||||||
|
blobs = __import__("re").findall(r"FromBase64String\('([A-Za-z0-9+/=]*)'\)", script)
|
||||||
|
self.assertEqual(base64.b64decode(blobs[0]), prompt.encode("utf-8")) # UTF-8, no BOM, bytes exact
|
||||||
|
self.assertFalse(base64.b64decode(blobs[1]).startswith(b"\xef\xbb\xbf"))
|
||||||
|
self.assertIn("Set-Location -LiteralPath", base64.b64decode(blobs[1]).decode("utf-8"))
|
||||||
|
self.assertLess(script.index("'HASPID'; return"), script.index("WriteAllBytes"))
|
||||||
|
self.assertLess(script.index("WriteAllBytes"), script.index("schtasks /Run /TN 'agentd-GW-26-1'"))
|
||||||
|
self.assertTrue(h["run_dir"].endswith("\\runtime\\agentd\\GW-26-1"))
|
||||||
|
|
||||||
|
def test_launch_refuses_a_started_directory(self):
|
||||||
|
t = self.transport(FakeWinChannel(haspid=True))
|
||||||
|
with self.assertRaises(TransportError) as cm:
|
||||||
|
t.start_run("agentd-GW-1-1", "C:\\r\\GW-1-1", {"prompt.md": "p", "run.ps1": "r"}, "run.ps1")
|
||||||
|
self.assertIn("HASPID", str(cm.exception))
|
||||||
|
|
||||||
|
def test_collect_is_one_call_plus_forget(self):
|
||||||
|
files = {"exit.txt": "\ufeffexit=0 at 2026-10-22T10:00:00", "last.md": "[review] done\n",
|
||||||
|
"events.jsonl": '{"type":"x"}\n{"type":"thread.started","thread_id":"th-42"}\n{"type":"turn.completed"}',
|
||||||
|
"stderr.txt": "warn 1\nwarn 2"}
|
||||||
|
ch = FakeWinChannel(files)
|
||||||
|
ex = make_executor(self.cfg, "win10", "codex", self.transport(ch))
|
||||||
|
h = {"run_dir": "C:\\r\\GW-1-1", "task": "agentd-GW-1-1", "thread_id": None}
|
||||||
|
res = ex.collect(h)
|
||||||
|
ex.forget(h)
|
||||||
|
self.assertEqual([c[0] for c in ch.calls], ["fetch", "forget"])
|
||||||
|
self.assertEqual((res.last_message, res.exit_code, res.thread_id), ("[review] done", 0, "th-42"))
|
||||||
|
self.assertEqual(res.tail, 'warn 1\nwarn 2\n{"type":"turn.completed"}')
|
||||||
|
|
||||||
|
def test_warpgate_fetch_is_one_call_when_it_fits(self):
|
||||||
|
cfg = config.load(ROOT / "config" / "agentd.example.yaml")
|
||||||
|
t = WindowsWarpgateTransport(cfg, "win10")
|
||||||
|
self.assertFalse(t.channel.stdin)
|
||||||
|
ops = make_executor(cfg, "win10", "codex", t).COLLECT
|
||||||
|
script = t.fetch_script(t.join(t.default_run_root(), "GW-26-1-r2"), ops)
|
||||||
|
self.assertTrue(t.channel.fits(script))
|
||||||
|
calls = []
|
||||||
|
t.ps = lambda s, timeout=150: (calls.append(s), "\n".join("M%d" % i for i in range(len(ops))))[1]
|
||||||
|
self.assertEqual(t.fetch("C:\\r", ops), [None, None, None, "", ""])
|
||||||
|
self.assertEqual(len(calls), 1)
|
||||||
|
self.assertFalse(t.channel.fits("x" * 3000))
|
||||||
|
|
||||||
|
|
||||||
|
class LocalDirect:
|
||||||
|
"""POSIX direct channel stand-in: runs the real loader (SH_COMMAND) through a local login-style shell with the
|
||||||
|
script on stdin, exactly what sshd does with the remote command."""
|
||||||
|
stdin, name, via = True, "ssh local", "direct"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def sh(self, script, timeout, op=None):
|
||||||
|
self.calls.append(op)
|
||||||
|
p = subprocess.run(["sh", "-c", SH_COMMAND], input=script.encode("utf-8"), # as sshd: $SHELL -c "<command>"
|
||||||
|
capture_output=True, timeout=timeout)
|
||||||
|
return p.returncode, p.stdout.decode("utf-8", "replace"), p.stderr.decode("utf-8", "replace")
|
||||||
|
|
||||||
|
|
||||||
|
class PosixDirectTest(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
root = Path(self.tmp.name)
|
||||||
|
(root / "repo").mkdir()
|
||||||
|
self.cfg = Config(direct_data(root))
|
||||||
|
self.ch = LocalDirect()
|
||||||
|
self.t = PosixWarpgateTransport(self.cfg, "lin", channel=self.ch)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_loader_quoting_matches_sh_quote(self):
|
||||||
|
self.assertEqual(SH_COMMAND, "bash -c 'exec 3<&0 </dev/null; s=$(cat <&3); exec 3<&-; (eval \"$s\"); r=$?; "
|
||||||
|
"[ $r = 255 ] && r=254; exit $r'")
|
||||||
|
|
||||||
|
def test_loader_never_exits_255(self):
|
||||||
|
self.assertEqual(self.ch.sh("echo x >&2; exit 255", 10)[0], 254) # 255 = ssh's own "no session" code
|
||||||
|
self.assertEqual(self.ch.sh("exit 3", 10)[0], 3)
|
||||||
|
self.assertEqual(self.ch.sh("true", 10)[0], 0)
|
||||||
|
|
||||||
|
def test_big_prompt_launch_and_collect_are_one_call_each(self):
|
||||||
|
ex = make_executor(self.cfg, "lin", "shell", self.t)
|
||||||
|
prompt = ("ünïcode 'q' \"d\" $(nope) `x`\n" * 6000) # ~200 KB > CHUNK and > MAX_ARG_STRLEN
|
||||||
|
h = ex.start(Run("GW-7-1", self.cfg.terminals["lin"]["repo"], prompt, script="wc -c"))
|
||||||
|
self.assertEqual(self.ch.calls, ["start"])
|
||||||
|
self.assertEqual(Path(h["run_dir"], "prompt.md").read_text(encoding="utf-8"), prompt)
|
||||||
|
for _ in range(100):
|
||||||
|
if ex.poll(h) == "finished":
|
||||||
|
break
|
||||||
|
__import__("time").sleep(0.1)
|
||||||
|
n = len(self.ch.calls)
|
||||||
|
res = ex.collect(h)
|
||||||
|
self.assertEqual(self.ch.calls[n:], ["fetch"])
|
||||||
|
self.assertEqual((res.last_message, res.exit_code), (str(len(prompt.encode("utf-8"))), 0))
|
||||||
|
with self.assertRaises(TransportError): # pid.txt present: never wiped, never relaunched there
|
||||||
|
ex.start(Run("GW-7-1", self.cfg.terminals["lin"]["repo"], "again", script="true"))
|
||||||
|
self.assertEqual(Path(h["run_dir"], "prompt.md").read_text(encoding="utf-8"), prompt)
|
||||||
|
|
||||||
|
def test_fetch_missing_files_and_stdin_is_not_the_scripts(self):
|
||||||
|
d = Path(self.tmp.name) / "runs" / "x"
|
||||||
|
d.mkdir(parents=True)
|
||||||
|
(d / "a.txt").write_text("l1\nl2 needle\nl3\n")
|
||||||
|
ops = [("read", "a.txt"), ("read", "none"), ("grep", "a.txt", "needle"), ("grep", "none", "x"),
|
||||||
|
("tail", "a.txt", 2), ("tail", "none", 3)]
|
||||||
|
self.assertEqual(self.t.fetch(str(d), ops), ["l1\nl2 needle\nl3\n", None, "l2 needle", None, "l2 needle\nl3", ""])
|
||||||
|
rc, out, _ = self.ch.sh("cat; echo done", 10) # the script's own stdin is /dev/null
|
||||||
|
self.assertEqual((rc, out), (0, "done\n"))
|
||||||
|
|
||||||
|
def test_parse_fetch_rejects_garbage(self):
|
||||||
|
with self.assertRaises(TransportError):
|
||||||
|
parse_fetch("nothing here", [("read", "x")])
|
||||||
|
|
||||||
|
|
||||||
|
class CallLogTest(unittest.TestCase):
|
||||||
|
def test_line_format_and_once(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
events = []
|
||||||
|
log = CallLog(Path(tmp) / "sub" / "remote.log", events.append)
|
||||||
|
log.call("win10", "direct", "start", 812.4, 0)
|
||||||
|
log.call("win10", "fallback", "busy", 9100, 255, "unreached")
|
||||||
|
lines = (Path(tmp) / "sub" / "remote.log").read_text().splitlines()
|
||||||
|
self.assertTrue(lines[0].endswith("term=win10 via=direct op=start ms=812 rc=0"))
|
||||||
|
self.assertTrue(lines[1].endswith("via=fallback op=busy ms=9100 rc=255 unreached"))
|
||||||
|
log.event("a", once="k")
|
||||||
|
log.event("a", once="k")
|
||||||
|
self.assertEqual(events, ["a"])
|
||||||
|
|
||||||
|
|
||||||
|
class PingCliTest(unittest.TestCase):
|
||||||
|
def test_ping_prints_timing_and_forces_channel(self):
|
||||||
|
cfg = Config(direct_data())
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def fake_make(cfg_, term, log=None, force=None):
|
||||||
|
seen.update(term=term, force=force)
|
||||||
|
return mock.Mock(ping=mock.Mock(return_value="PONG=Linux 6.8 oregon"))
|
||||||
|
buf = io.StringIO()
|
||||||
|
with mock.patch("agentd.cli.make_transport", fake_make), redirect_stdout(buf), self.assertRaises(SystemExit) as cm:
|
||||||
|
cli.cmd_ping(cfg, cli.parser().parse_args(["ping", "lin", "--transport", "warpgate-ssh", "-n", "2"]))
|
||||||
|
self.assertEqual(cm.exception.code, 0)
|
||||||
|
self.assertEqual(seen, {"term": "lin", "force": "warpgate-ssh"})
|
||||||
|
self.assertEqual(buf.getvalue().count("PONG=Linux 6.8 oregon"), 2)
|
||||||
|
|
||||||
|
|
||||||
|
class ReviewFixesTest(unittest.TestCase):
|
||||||
|
"""Follow-ups of the PAD-12 review: classification limits, socket and known_hosts names, key handling."""
|
||||||
|
def setUp(self):
|
||||||
|
direct_ssh._KEYS.clear()
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.cfg = Config(direct_data(self.tmp.name))
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
direct_ssh._KEYS.clear()
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_output_or_remote_stderr_never_counts_as_unreached(self):
|
||||||
|
self.assertEqual(classify(255, "Connection reset by 100.1.2.3 port 22", "WROTE=ab\n"), "unknown")
|
||||||
|
self.assertEqual(classify(255, "git: ssh: connect to host github.com port 22: Connection timed out"), "unknown")
|
||||||
|
self.assertEqual(classify(255, "fatal: Could not read from remote repository"), "unknown")
|
||||||
|
|
||||||
|
def test_no_ed25519_offered_is_a_host_key_failure(self):
|
||||||
|
self.assertEqual(classify(255, "Unable to negotiate with 100.1.2.3 port 22: no matching host key type found. "
|
||||||
|
"Their offer: ssh-rsa"), "hostkey")
|
||||||
|
self.assertEqual(classify(255, "Unable to negotiate with 100.1.2.3 port 22: no matching cipher found. "
|
||||||
|
"Their offer: aes128-cbc"), "unreached")
|
||||||
|
|
||||||
|
def test_reset_after_output_is_not_retried_elsewhere(self):
|
||||||
|
ch = DirectChannel(self.cfg, "lin")
|
||||||
|
|
||||||
|
def fake_run(argv, input=None, stdout=None, stderr=None, env=None, timeout=None):
|
||||||
|
stdout.write(b"WROTE=1\n")
|
||||||
|
stderr.write(b"Connection reset by 100.127.58.78 port 22\r\n")
|
||||||
|
return mock.Mock(returncode=255)
|
||||||
|
with mock.patch("agentd.transports.direct_ssh.key_agent", return_value=FakeAgent), \
|
||||||
|
mock.patch("agentd.transports.direct_ssh.subprocess.run", fake_run):
|
||||||
|
with self.assertRaises(TransportError) as cm:
|
||||||
|
ch.sh("true", 30)
|
||||||
|
self.assertNotIsInstance(cm.exception, Unreached)
|
||||||
|
|
||||||
|
def test_control_path_follows_the_pinned_key(self):
|
||||||
|
a = DirectChannel(self.cfg, "lin")
|
||||||
|
d = direct_data(self.tmp.name)
|
||||||
|
d["terminals"]["lin"]["ssh"]["host_key"] = WIN_KEY
|
||||||
|
b = DirectChannel(Config(d), "lin")
|
||||||
|
path = lambda ch: [x for x in ch.argv(FakeAgent, "true") if x.startswith("ControlPath=")][0] # noqa: E731
|
||||||
|
self.assertNotEqual(path(a), path(b))
|
||||||
|
self.assertEqual(path(a), "ControlPath=%s/%s-%%C" % (self.cfg.ssh_dir, a.pin))
|
||||||
|
self.assertEqual(len(a.pin), 8)
|
||||||
|
long_dir = direct_data("/" + "x" * 40)
|
||||||
|
self.assertTrue(any("too long" in e for e in config.validate(long_dir))) # 41 + 67 > 104
|
||||||
|
|
||||||
|
def test_known_hosts_temp_files_are_per_terminal_and_process(self):
|
||||||
|
d = direct_ssh.control_dir(self.cfg)
|
||||||
|
tmps = []
|
||||||
|
real = os.replace
|
||||||
|
with mock.patch("agentd.transports.direct_ssh.os.replace",
|
||||||
|
side_effect=lambda a, b: (tmps.append(str(a)), real(a, b))):
|
||||||
|
DirectChannel(self.cfg, "win10").known_hosts(d)
|
||||||
|
DirectChannel(self.cfg, "lin").known_hosts(d)
|
||||||
|
self.assertEqual(len(set(tmps)), 2)
|
||||||
|
self.assertTrue(all(str(os.getpid()) in t for t in tmps))
|
||||||
|
self.assertEqual((d / "known_hosts.win10").read_text(), "agentd-win10 %s\n" % WIN_KEY)
|
||||||
|
self.assertEqual((d / "known_hosts.lin").read_text(), "agentd-lin %s\n" % LIN_KEY)
|
||||||
|
self.assertEqual(sorted(f.name for f in d.iterdir()), ["known_hosts.lin", "known_hosts.win10"])
|
||||||
|
|
||||||
|
def test_key_leaves_the_environment_at_cli_start(self):
|
||||||
|
buf = io.StringIO()
|
||||||
|
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET}), redirect_stdout(buf):
|
||||||
|
cli.main(["--config", str(ROOT / "config" / "agentd.yaml"), "check-config"])
|
||||||
|
self.assertNotIn("AGENTD_SSH_KEY", os.environ)
|
||||||
|
self.assertIn("direct-ssh key AGENTD_SSH_KEY: set", buf.getvalue())
|
||||||
|
self.assertNotIn("NOT-A-REAL-KEY", buf.getvalue())
|
||||||
|
self.assertTrue(direct_ssh.key_present("AGENTD_SSH_KEY"))
|
||||||
|
with mock.patch("agentd.transports.direct_ssh.shutil.which", return_value=None):
|
||||||
|
a = KeyAgent("AGENTD_SSH_KEY").start() # the agent takes the key over from the CLI
|
||||||
|
self.assertIn("not installed", a.error)
|
||||||
|
self.assertFalse(direct_ssh.key_present("AGENTD_SSH_KEY"))
|
||||||
|
|
||||||
|
def test_local_busy_check_gets_the_scrubbed_env(self):
|
||||||
|
cfg = Config(config_data())
|
||||||
|
from agentd.transports.local import LocalTransport
|
||||||
|
with mock.patch.dict(os.environ, {"AGENTD_SSH_KEY": SECRET, "TOK_OR": "plane-token"}), \
|
||||||
|
mock.patch("agentd.transports.local.subprocess.run", return_value=mock.Mock(stdout="2\n")) as run:
|
||||||
|
self.assertEqual(LocalTransport(cfg, "oregon").process_count("codex"), 2)
|
||||||
|
env = run.call_args.kwargs["env"]
|
||||||
|
self.assertNotIn("AGENTD_SSH_KEY", env)
|
||||||
|
self.assertNotIn("TOK_OR", env)
|
||||||
|
|
||||||
|
def test_task_names_are_quoted_in_kill_and_forget(self):
|
||||||
|
ch = mock.Mock(stdin=True)
|
||||||
|
ch.ps.return_value = "KILLED=ok"
|
||||||
|
t = WindowsWarpgateTransport(self.cfg, "win10", channel=ch)
|
||||||
|
t.kill("agentd-a'b", "C:\\runs\\x")
|
||||||
|
t.forget("agentd-a'b")
|
||||||
|
for c in ch.ps.call_args_list:
|
||||||
|
self.assertIn("'agentd-a''b'", c.args[0])
|
||||||
|
self.assertNotIn("'agentd-a'b'", c.args[0])
|
||||||
|
|
||||||
|
|
||||||
|
class FakeTerm:
|
||||||
|
"""Remote transport stand-in for core-level routing tests: pure helpers from the real Windows transport,
|
||||||
|
start_run / probe on a dict of run directories shared by the direct and fallback channels. `fail` = exceptions
|
||||||
|
for the next calls: Unreached and HostKeyError strike before the command ran, any other after it (the run
|
||||||
|
exists). A host-key failure sticks, like DirectChannel's."""
|
||||||
|
|
||||||
|
def __init__(self, cfg, dirs):
|
||||||
|
self.real = WindowsWarpgateTransport(cfg, "win10", channel=mock.Mock(stdin=True))
|
||||||
|
self.dirs, self.fail, self.calls, self.hostkey = dirs, [], [], None
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
return getattr(self.real, name)
|
||||||
|
|
||||||
|
def _next(self, name):
|
||||||
|
self.calls.append(name)
|
||||||
|
if self.hostkey:
|
||||||
|
raise self.hostkey
|
||||||
|
exc = self.fail.pop(0) if self.fail else None
|
||||||
|
if isinstance(exc, HostKeyError):
|
||||||
|
self.hostkey = exc
|
||||||
|
return exc
|
||||||
|
|
||||||
|
def start_run(self, task, run_dir, files, script_name):
|
||||||
|
exc = self._next("start_run")
|
||||||
|
if isinstance(exc, (Unreached, HostKeyError)):
|
||||||
|
raise exc
|
||||||
|
if run_dir in self.dirs:
|
||||||
|
raise TransportError("HASPID")
|
||||||
|
self.dirs[run_dir] = ["launch.pid"]
|
||||||
|
if exc:
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
def probe(self, run_dir, names):
|
||||||
|
exc = self._next("probe")
|
||||||
|
if exc:
|
||||||
|
raise exc
|
||||||
|
return list(self.dirs.get(run_dir, ()))
|
||||||
|
|
||||||
|
|
||||||
|
class CoreRoutingTest(unittest.TestCase):
|
||||||
|
"""Agentd.start -> codex executor -> RoutedTransport: the double-launch guarantee end to end."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
from tests.fakes import Harness
|
||||||
|
self.h = Harness(direct_data())
|
||||||
|
self.dirs, self.events = {}, []
|
||||||
|
self.direct, self.fallback = FakeTerm(self.h.cfg, self.dirs), FakeTerm(self.h.cfg, self.dirs)
|
||||||
|
log = CallLog(event=self.events.append)
|
||||||
|
self.h.d.executor_factory = lambda term, name: make_executor( # a fresh routed transport per tick
|
||||||
|
self.h.cfg, term, name, RoutedTransport(term, self.direct, self.fallback, log))
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.h.close()
|
||||||
|
|
||||||
|
def test_unreached_launch_moves_to_the_fallback_once(self):
|
||||||
|
iid = self.h.plane.add_issue(1)
|
||||||
|
self.direct.fail = [Unreached("ssh: connect to host 100.123.209.19 port 22: Connection timed out")]
|
||||||
|
self.h.tick()
|
||||||
|
rec = self.h.rec(iid)
|
||||||
|
self.assertNotIn("launching", rec)
|
||||||
|
self.assertEqual(rec["inflight"]["run_id"], "GW-1-1")
|
||||||
|
self.assertEqual((self.direct.calls, self.fallback.calls), (["start_run"], ["start_run"]))
|
||||||
|
self.assertEqual(len(self.dirs), 1)
|
||||||
|
self.assertEqual(self.h.plane.state_of(iid), "In Progress")
|
||||||
|
self.assertTrue(any("warpgate-ssh fallback" in e for e in self.events))
|
||||||
|
|
||||||
|
def test_failure_after_the_command_ran_is_probed_and_adopted_not_relaunched(self):
|
||||||
|
iid = self.h.plane.add_issue(1)
|
||||||
|
self.direct.fail = [TransportError("ssh 100.123.209.19 timeout after 150s")]
|
||||||
|
self.h.tick()
|
||||||
|
rec = self.h.rec(iid)
|
||||||
|
self.assertIn("launching", rec)
|
||||||
|
self.assertNotIn("inflight", rec)
|
||||||
|
self.assertEqual(self.fallback.calls, [])
|
||||||
|
self.h.tick() # probe finds launch.pid -> adopt
|
||||||
|
rec = self.h.rec(iid)
|
||||||
|
self.assertNotIn("launching", rec)
|
||||||
|
self.assertEqual(rec["inflight"]["handle"]["run_dir"], list(self.dirs)[0])
|
||||||
|
self.assertEqual(self.direct.calls, ["start_run", "probe"])
|
||||||
|
self.assertEqual((self.fallback.calls, len(self.dirs)), ([], 1))
|
||||||
|
|
||||||
|
def test_host_key_failure_keeps_the_marker_and_never_falls_back(self):
|
||||||
|
iid = self.h.plane.add_issue(1)
|
||||||
|
self.direct.fail = [HostKeyError("host key verification FAILED")]
|
||||||
|
self.h.tick()
|
||||||
|
self.assertIn("launching", self.h.rec(iid))
|
||||||
|
self.h.tick() # the probe fails the same way: marker kept, slot held
|
||||||
|
rec = self.h.rec(iid)
|
||||||
|
self.assertIn("launching", rec)
|
||||||
|
self.assertNotIn("inflight", rec)
|
||||||
|
self.assertEqual(self.direct.calls, ["start_run", "probe"])
|
||||||
|
self.assertEqual((self.fallback.calls, self.dirs), ([], {}))
|
||||||
|
self.assertEqual(self.h.plane.state_of(iid), "Todo")
|
||||||
|
|
||||||
|
|
||||||
|
FAKE_CLIENT = """import json, os, sys
|
||||||
|
argv = sys.argv[1:]
|
||||||
|
with open(os.environ["FAKE_CALLS"], "a") as fh:
|
||||||
|
fh.write(json.dumps(argv) + "\\n")
|
||||||
|
if os.environ.get("FAKE_MISSING") and "AGENTD_SSH_KEY=AGENTD_SSH_KEY" in argv:
|
||||||
|
print("access-hub: credential_unavailable", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
print("ticked")
|
||||||
|
print("agentd stderr line", file=sys.stderr)
|
||||||
|
sys.exit(int(os.environ.get("FAKE_RC", "0")))
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(all(shutil.which(b) for b in ("bash", "flock", "timeout")), "needs bash, flock, timeout")
|
||||||
|
class CronScriptTest(unittest.TestCase):
|
||||||
|
"""cron/agentd-tick-lxc.sh with a fake agent-access.py: a missing AGENTD_SSH_KEY must not stop the tick."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
import re
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
d = Path(self.tmp.name)
|
||||||
|
(d / "client.py").write_text(FAKE_CLIENT)
|
||||||
|
text = (ROOT / "cron" / "agentd-tick-lxc.sh").read_text()
|
||||||
|
for var, value in (("LOCK", d / "cron.lock"), ("LOG", d / "cron.log"), ("CLIENT", "(python3 -I %s)" % (d / "client.py"))):
|
||||||
|
text, n = re.subn(r"(?m)^%s=.*$" % var, "%s=%s" % (var, value), text)
|
||||||
|
self.assertEqual(n, 1, var)
|
||||||
|
self.script, self.calls, self.log = d / "tick.sh", d / "calls", d / "cron.log"
|
||||||
|
self.script.write_text(text)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def run_tick(self, **env):
|
||||||
|
e = dict(os.environ, FAKE_CALLS=str(self.calls), **env)
|
||||||
|
rc = subprocess.run(["bash", str(self.script)], env=e, timeout=60).returncode
|
||||||
|
calls = [json.loads(l) for l in self.calls.read_text().splitlines()] if self.calls.exists() else []
|
||||||
|
return rc, calls, self.log.read_text() if self.log.exists() else ""
|
||||||
|
|
||||||
|
def test_normal_tick_passes_the_key(self):
|
||||||
|
rc, calls, log = self.run_tick()
|
||||||
|
self.assertEqual((rc, len(calls)), (0, 1))
|
||||||
|
self.assertIn("--secret-env", calls[0])
|
||||||
|
self.assertIn("AGENTD_SSH_KEY=AGENTD_SSH_KEY", calls[0])
|
||||||
|
self.assertEqual(calls[0][-3:], ["/opt/plane-agentd/config/agentd.yaml", "tick", "--once"])
|
||||||
|
self.assertIn("ticked", log)
|
||||||
|
self.assertIn("agentd stderr line", log)
|
||||||
|
|
||||||
|
def test_missing_vault_key_repeats_the_tick_without_it(self):
|
||||||
|
rc, calls, log = self.run_tick(FAKE_MISSING="1")
|
||||||
|
self.assertEqual((rc, len(calls)), (0, 2))
|
||||||
|
self.assertNotIn("AGENTD_SSH_KEY=AGENTD_SSH_KEY", calls[1])
|
||||||
|
self.assertIn("PLANE_OREGON_API_TOKEN=PLANE_OREGON_API_TOKEN", calls[1])
|
||||||
|
self.assertIn("AGENTD_SSH_KEY not available from Vault", log)
|
||||||
|
self.assertIn("ticked", log)
|
||||||
|
|
||||||
|
def test_a_failing_tick_is_not_repeated(self):
|
||||||
|
rc, calls, _ = self.run_tick(FAKE_RC="1")
|
||||||
|
self.assertEqual((rc, len(calls)), (1, 1))
|
||||||
|
|
||||||
|
def test_lock_held_exits_75(self):
|
||||||
|
import fcntl
|
||||||
|
with open(Path(self.tmp.name) / "cron.lock", "a") as fh:
|
||||||
|
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
rc, calls, _ = self.run_tick()
|
||||||
|
self.assertEqual((rc, calls), (75, []))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -80,9 +80,11 @@ class PosixRemoteTest(unittest.TestCase):
|
|||||||
|
|
||||||
def test_live_config_runs_oregon_remotely(self):
|
def test_live_config_runs_oregon_remotely(self):
|
||||||
cfg = config.load(ROOT / "config" / "agentd.yaml")
|
cfg = config.load(ROOT / "config" / "agentd.yaml")
|
||||||
t = make_transport(cfg, "oregon")
|
t = make_transport(cfg, "oregon") # PAD-12: direct-ssh over Tailscale, Warpgate relay as fallback
|
||||||
self.assertIsInstance(t, PosixWarpgateTransport)
|
self.assertIsInstance(t.primary, PosixWarpgateTransport)
|
||||||
self.assertEqual((t.target, t.os), ("oregon", "linux"))
|
self.assertIsInstance(t.fallback, PosixWarpgateTransport)
|
||||||
|
self.assertEqual((t.primary.channel.stdin, t.fallback.channel.stdin), (True, False))
|
||||||
|
self.assertEqual((t.fallback.target, t.os), ("oregon", "linux"))
|
||||||
self.assertEqual(cfg.executors["codex"]["tier_defaults"], {"gpt-6.1-sol": "fast"})
|
self.assertEqual(cfg.executors["codex"]["tier_defaults"], {"gpt-6.1-sol": "fast"})
|
||||||
|
|
||||||
def test_relay_argv_and_scrubbed_env(self):
|
def test_relay_argv_and_scrubbed_env(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user