PAD-12: per-terminal transport (direct Tailscale SSH or Warpgate, pre-session fallback, pinned host keys, agent key from Vault), one-call win10 launch and one-call collect, remote call timings, agentd ping
This commit is contained in:
jj.wang
@@ -4,21 +4,162 @@ Interface (duck-typed, see docs/executor-contract.md):
|
||||
os, join(*parts), quote(s), default_run_root()
|
||||
mkdir(path, clean=False), write_text(path, text), read_text(path) -> str|None,
|
||||
grep_first(path, needle) -> str|None, tail(path, n) -> str
|
||||
fetch(run_dir, ops) -> [result per op] (ops: ("read", name) | ("grep", name, needle) | ("tail", name, n))
|
||||
runner(run_dir, cwd, command, stdout_name) -> (script_name, script_text)
|
||||
launch(task, script_path, run_dir), kill(task, run_dir), forget(task)
|
||||
process_count(name) -> int|None (None = unreachable)"""
|
||||
start_run(task, run_dir, files, script_name) (remote only), launch(task, script_path, run_dir), kill(task, run_dir),
|
||||
forget(task), process_count(name) -> int|None (None = unreachable)
|
||||
|
||||
Remote terminals are reached through a channel (design §9): `warpgate-ssh` (agent-access.py run through the
|
||||
Warpgate gateway) or `direct-ssh` (OpenSSH over Tailscale with a dedicated key, optional warpgate fallback)."""
|
||||
import base64
|
||||
import functools
|
||||
import re
|
||||
import time
|
||||
|
||||
|
||||
class TransportError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def make_transport(cfg, terminal):
|
||||
kind = cfg.terminals[terminal]["transport"]
|
||||
class Unreached(TransportError):
|
||||
"""The remote command certainly did not run (connect / auth / agent failure, direct path unavailable): the
|
||||
only kind of failure a direct-ssh terminal may repeat on its fallback channel."""
|
||||
|
||||
|
||||
class HostKeyError(TransportError):
|
||||
"""The terminal presented a host key other than the pinned one: a security error, never retried elsewhere."""
|
||||
|
||||
|
||||
class CallLog:
|
||||
"""One line per remote call -> `path` (remote.log); notable events (fallback, host key) also -> `event`."""
|
||||
|
||||
def __init__(self, path=None, event=None, echo=False):
|
||||
self.path, self.event_fn, self.echo = path, event, echo
|
||||
self._once = set()
|
||||
|
||||
def call(self, terminal, via, op, ms, rc, note=""):
|
||||
line = "term=%s via=%s op=%s ms=%d rc=%s%s" % (terminal, via, op or "-", ms, rc, " " + note if note else "")
|
||||
if self.echo:
|
||||
print(line, flush=True)
|
||||
if self.path:
|
||||
try:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.path, "a", encoding="utf-8") as fh:
|
||||
fh.write(time.strftime("%Y-%m-%dT%H:%M:%S") + " " + line + "\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def event(self, msg, once=None):
|
||||
"""`once` = a key: the message is logged only the first time per process."""
|
||||
if once is not None:
|
||||
if once in self._once:
|
||||
return
|
||||
self._once.add(once)
|
||||
if self.event_fn:
|
||||
self.event_fn(msg)
|
||||
|
||||
|
||||
NULL_LOG = CallLog()
|
||||
|
||||
|
||||
def operation(name):
|
||||
"""Method decorator: names the remote operation for the call log (innermost wins, restored on return)."""
|
||||
def deco(fn):
|
||||
@functools.wraps(fn)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
prev, self._op = getattr(self, "_op", None), name
|
||||
try:
|
||||
return fn(self, *args, **kwargs)
|
||||
finally:
|
||||
self._op = prev
|
||||
return wrapper
|
||||
return deco
|
||||
|
||||
|
||||
def fetch_each(t, run_dir, ops):
|
||||
"""`fetch` built from the single-file primitives (one call per op): local transport and fallbacks."""
|
||||
out = []
|
||||
for o in ops:
|
||||
path = t.join(run_dir, o[1])
|
||||
if o[0] == "read":
|
||||
out.append(t.read_text(path))
|
||||
elif o[0] == "grep":
|
||||
out.append(t.grep_first(path, o[2]))
|
||||
elif o[0] == "tail":
|
||||
out.append(t.tail(path, o[2]))
|
||||
else:
|
||||
raise ValueError("unknown fetch op %r" % (o,))
|
||||
return out
|
||||
|
||||
|
||||
def parse_fetch(out, ops, what=""):
|
||||
"""Decode the `F<i>=<base64>` / `M<i>` (missing) lines of a remote fetch script into results per op:
|
||||
read -> text|None, grep -> line|None, tail -> text ("" when missing)."""
|
||||
res = []
|
||||
for i, o in enumerate(ops):
|
||||
m = re.search(r"^F%d=([A-Za-z0-9+/=]*)$" % i, out, re.M)
|
||||
if m:
|
||||
text = base64.b64decode(m.group(1)).decode("utf-8", "replace") if m.group(1) else ""
|
||||
elif re.search(r"^M%d$" % i, out, re.M):
|
||||
text = None
|
||||
else:
|
||||
raise TransportError("fetch %s: unexpected output: %s" % (what, out.strip()[-200:]))
|
||||
if o[0] == "grep":
|
||||
text = (text or "").rstrip("\n") or None
|
||||
elif o[0] == "tail":
|
||||
text = (text or "").rstrip("\n")
|
||||
res.append(text)
|
||||
return res
|
||||
|
||||
|
||||
class RoutedTransport:
|
||||
"""direct-ssh terminal: every operation runs on the primary (direct) transport; when it fails before the remote
|
||||
command ran (`Unreached`) the operation is repeated on the fallback (warpgate-ssh) transport, which then serves
|
||||
the rest of this transport's life (one tick). Each direct operation is a single remote call, so a repeat never
|
||||
doubles a side effect. Host-key failures and failures after the command may have run propagate unchanged."""
|
||||
PURE = ("join", "quote", "default_run_root", "runner")
|
||||
|
||||
def __init__(self, terminal, primary, fallback=None, log=None):
|
||||
self.terminal, self.primary, self.fallback = terminal, primary, fallback
|
||||
self.log = log or NULL_LOG
|
||||
self.on_fallback = False
|
||||
|
||||
def __getattr__(self, name):
|
||||
attr = getattr(self.primary, name)
|
||||
if name in self.PURE or not callable(attr):
|
||||
return attr
|
||||
|
||||
def routed(*args, **kwargs):
|
||||
if self.on_fallback:
|
||||
return getattr(self.fallback, name)(*args, **kwargs)
|
||||
try:
|
||||
return attr(*args, **kwargs)
|
||||
except Unreached as exc:
|
||||
if self.fallback is None:
|
||||
raise
|
||||
self.on_fallback = True
|
||||
self.log.event("%s: direct-ssh failed before the command ran (%s); using the warpgate-ssh fallback "
|
||||
"for the rest of this tick" % (self.terminal, exc))
|
||||
return getattr(self.fallback, name)(*args, **kwargs)
|
||||
return routed
|
||||
|
||||
|
||||
def make_transport(cfg, terminal, log=None, force=None):
|
||||
"""`force` = direct-ssh | warpgate-ssh: that channel only, no fallback (agentd ping)."""
|
||||
t = cfg.terminals[terminal]
|
||||
kind = force or t["transport"]
|
||||
if kind == "local":
|
||||
from .local import LocalTransport
|
||||
return LocalTransport(cfg, terminal)
|
||||
if kind == "warpgate-ssh":
|
||||
from .warpgate_ssh import WarpgateSshTransport
|
||||
return WarpgateSshTransport(cfg, terminal)
|
||||
return WarpgateSshTransport(cfg, terminal, log=log)
|
||||
if kind == "direct-ssh":
|
||||
from .direct_ssh import DirectChannel
|
||||
from .warpgate_ssh import WarpgateSshTransport
|
||||
primary = WarpgateSshTransport(cfg, terminal, channel=DirectChannel(cfg, terminal, log))
|
||||
fallback = None
|
||||
if not force and t.get("fallback") == "warpgate-ssh":
|
||||
fallback = WarpgateSshTransport(cfg, terminal, log=log, label="fallback")
|
||||
return RoutedTransport(terminal, primary, fallback, log)
|
||||
raise TransportError("unknown transport %s" % kind)
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
"""direct-ssh channel (PAD-12): OpenSSH from the dispatcher straight to the terminal over Tailscale, no Warpgate.
|
||||
|
||||
- Auth: one dedicated agentd key. Its private text arrives in the env var named by `agentd.ssh_key` (default
|
||||
AGENTD_SSH_KEY, injected by `agent-access.py exec --secret-env`), is popped from os.environ (children never see
|
||||
it) and loaded with `ssh-add -` into a per-process ssh-agent on a socket in a private 0700 temp dir. The key is
|
||||
never written to disk, argv, logs or exceptions. No key / agent failure -> the channel is unavailable
|
||||
(`Unreached`), so a terminal with `fallback: warpgate-ssh` uses Warpgate instead.
|
||||
- Host key: pinned per terminal (`ssh.host_key`); a known_hosts file is generated from it (public data) and
|
||||
checked strictly. A mismatch raises HostKeyError: never a fallback.
|
||||
- Reuse: ControlMaster=auto / ControlPersist=60 under `agentd.ssh_dir` (root-only 0700), so the second call of a
|
||||
tick (and the next tick within a minute) skips the handshake. The socket name carries a fingerprint of the pinned
|
||||
host key: a changed pin never rides an open master that was verified against the old one.
|
||||
- Payload on stdin: Windows runs a fixed PowerShell bootstrap (-EncodedCommand) that reads the base64 of the real
|
||||
script from stdin; POSIX runs a fixed bash loader that reads the script from stdin. One call per operation, no
|
||||
argv size limit."""
|
||||
import atexit
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from . import NULL_LOG, HostKeyError, TransportError, Unreached
|
||||
from .local import sh_quote
|
||||
|
||||
CONNECT_TIMEOUT = 8
|
||||
CONTROL_PERSIST = 60
|
||||
SOCKET_MAX = 104 # sun_path; ssh adds ".<16 random chars>" while it sets up the master socket
|
||||
CTL_NAME = 1 + 9 + 40 + 17 # "/<pin8>-<%C: 40 hex>" + that suffix
|
||||
|
||||
# Fixed PowerShell bootstrap: the operation's script arrives on stdin as base64 of its UTF-8 text (ASCII on the
|
||||
# wire, so the console input code page does not matter).
|
||||
BOOTSTRAP = ("$ProgressPreference='SilentlyContinue'\n"
|
||||
"$ErrorActionPreference='Continue'\n"
|
||||
"[Console]::OutputEncoding=[Text.Encoding]::UTF8\n"
|
||||
"$s=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String([Console]::In.ReadToEnd().Trim()))\n"
|
||||
"& ([ScriptBlock]::Create($s))\n")
|
||||
|
||||
|
||||
def encoded_command(script):
|
||||
"""PowerShell -EncodedCommand value: base64 of the UTF-16LE text."""
|
||||
return base64.b64encode(script.encode("utf-16-le")).decode("ascii")
|
||||
|
||||
|
||||
PS_COMMAND = "powershell.exe -NoProfile -NonInteractive -EncodedCommand " + encoded_command(BOOTSTRAP)
|
||||
# POSIX loader: the script is read from stdin into a variable (no MAX_ARG_STRLEN), then runs in a subshell with
|
||||
# stdin = /dev/null. Its exit 255 is reported as 254: 255 stays ssh's own "no session" code. (The PowerShell
|
||||
# bootstrap exits 0/1 by $?; none of our PowerShell scripts calls `exit`.)
|
||||
SH_COMMAND = "bash -c " + sh_quote('exec 3<&0 </dev/null; s=$(cat <&3); exec 3<&-; (eval "$s"); r=$?; '
|
||||
'[ $r = 255 ] && r=254; exit $r')
|
||||
|
||||
|
||||
def ps_payload(script):
|
||||
return (base64.b64encode(script.encode("utf-8")).decode("ascii") + "\n").encode("ascii")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- failure classification
|
||||
# A host offering no ssh-ed25519 key at all counts as a host-key failure too (a downgrade is a security signal).
|
||||
HOSTKEY = re.compile(r"Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED|"
|
||||
r"No \S+ host key is known for|host key for \S+ has changed|no matching host key type", re.I)
|
||||
# ssh exit 255 with one of these = no remote session ran the command. Only ssh's own messages, anchored at the start
|
||||
# of a line (stderr also carries the remote command's stderr).
|
||||
UNREACHED = re.compile(r"^(?:ssh: connect to host |ssh: Could not resolve hostname |banner exchange: |"
|
||||
r"Connection timed out during banner exchange|kex_exchange_identification: |"
|
||||
r"Connection (?:closed|reset) by \S+ port \d+\s*$|"
|
||||
r"\S+: Permission denied \((?:publickey|password|keyboard-interactive|hostbased|gssapi)|"
|
||||
r"Received disconnect from .*Too many authentication failures|"
|
||||
r"sign_and_send_pubkey: |error fetching identities|Error connecting to agent|"
|
||||
r"Unable to negotiate with |Session open refused by peer)", re.I | re.M)
|
||||
|
||||
|
||||
def classify(rc, err, out=""):
|
||||
"""-> 'ok' (the remote command ran, whatever its exit code), 'hostkey', 'unreached' (certainly not run) or
|
||||
'unknown' (ssh exit 255 without a recognisable pre-session error: the command may have run). Any stdout proves
|
||||
that a session ran the command, so it is never 'unreached'."""
|
||||
if rc != 255:
|
||||
return "ok"
|
||||
if (out or "").strip():
|
||||
return "unknown"
|
||||
if HOSTKEY.search(err or ""):
|
||||
return "hostkey"
|
||||
if UNREACHED.search(err or ""):
|
||||
return "unreached"
|
||||
return "unknown"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- per-process key agent
|
||||
def _die_with_parent():
|
||||
"""preexec: the agent gets SIGTERM when agentd dies, even by SIGKILL (Linux PR_SET_PDEATHSIG)."""
|
||||
try:
|
||||
import ctypes
|
||||
ctypes.CDLL("libc.so.6", use_errno=True).prctl(1, signal.SIGTERM)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def normalise_key(text):
|
||||
"""Vault may hand the key with literal `\\n` escapes or without the final newline."""
|
||||
if "\n" not in text.strip() and "\\n" in text:
|
||||
text = text.replace("\\n", "\n")
|
||||
return text.strip() + "\n"
|
||||
|
||||
|
||||
_KEYS = {} # env var name -> key text, taken out of os.environ at CLI start until the agent loads it
|
||||
|
||||
|
||||
def take_key(env_name):
|
||||
"""Move the key out of os.environ right away (called by the CLI before anything else runs), so no child -- not
|
||||
even one started before the first direct call -- can inherit it."""
|
||||
if env_name not in _KEYS:
|
||||
_KEYS[env_name] = os.environ.pop(env_name, None)
|
||||
else:
|
||||
os.environ.pop(env_name, None)
|
||||
|
||||
|
||||
def key_present(env_name):
|
||||
return bool((_KEYS.get(env_name) or os.environ.get(env_name) or "").strip())
|
||||
|
||||
|
||||
class KeyAgent:
|
||||
"""ssh-agent of this process holding the agentd key. `error` set = direct-ssh unavailable here."""
|
||||
|
||||
def __init__(self, env_name):
|
||||
self.env_name = env_name
|
||||
self.dir = self.sock = self.pub = self.proc = None
|
||||
self.error = None
|
||||
|
||||
def start(self):
|
||||
key = os.environ.pop(self.env_name, None) # popped even if loading fails: children never inherit it
|
||||
key = _KEYS.pop(self.env_name, None) or key
|
||||
if not key or not key.strip():
|
||||
self.error = "%s is not set (agent-access.py exec --secret-env %s=<vault key>)" % (self.env_name, self.env_name)
|
||||
return self
|
||||
agent_bin, add_bin = shutil.which("ssh-agent"), shutil.which("ssh-add")
|
||||
if not (agent_bin and add_bin and shutil.which("ssh")):
|
||||
self.error = "ssh / ssh-agent / ssh-add not installed"
|
||||
return self
|
||||
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LANG": "C"}
|
||||
try:
|
||||
self.dir = tempfile.mkdtemp(prefix="agentd-agent-") # 0700
|
||||
self.sock = os.path.join(self.dir, "agent.sock")
|
||||
atexit.register(self.close)
|
||||
self.proc = subprocess.Popen([agent_bin, "-D", "-a", self.sock], stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env,
|
||||
close_fds=True, preexec_fn=_die_with_parent)
|
||||
end = time.monotonic() + 5
|
||||
while not os.path.exists(self.sock):
|
||||
if self.proc.poll() is not None or time.monotonic() > end:
|
||||
self.error = "ssh-agent did not start"
|
||||
return self
|
||||
time.sleep(0.02)
|
||||
env["SSH_AUTH_SOCK"] = self.sock
|
||||
add = subprocess.run([add_bin, "-q", "-"], input=normalise_key(key).encode("utf-8"), env=env,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=15)
|
||||
key = None
|
||||
if add.returncode != 0:
|
||||
why = add.stderr.decode("utf-8", "replace").strip().splitlines()[:1]
|
||||
why = why[0][:120] if why and "-----" not in why[0] else ""
|
||||
self.error = "ssh-add failed (rc %d) %s" % (add.returncode, why)
|
||||
return self
|
||||
listed = subprocess.run([add_bin, "-L"], env=env, capture_output=True, timeout=15)
|
||||
pub = next((l for l in listed.stdout.decode("ascii", "replace").splitlines() if l.startswith("ssh-")), None)
|
||||
if not pub:
|
||||
self.error = "ssh-agent holds no key after ssh-add"
|
||||
return self
|
||||
self.pub = os.path.join(self.dir, "agentd.pub") # public half: selects the agent key (IdentitiesOnly)
|
||||
with open(self.pub, "w", encoding="ascii") as fh:
|
||||
fh.write(pub + "\n")
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
self.error = "ssh-agent setup failed: %s" % type(exc).__name__
|
||||
finally:
|
||||
key = None
|
||||
if self.error:
|
||||
self.close()
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
if self.proc and self.proc.poll() is None:
|
||||
self.proc.terminate()
|
||||
try:
|
||||
self.proc.wait(3)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.proc.kill()
|
||||
if self.dir:
|
||||
shutil.rmtree(self.dir, ignore_errors=True)
|
||||
|
||||
|
||||
_AGENTS = {}
|
||||
|
||||
|
||||
def key_agent(cfg, log=NULL_LOG):
|
||||
"""The process-wide agent for cfg's key env var, started on first use; raises Unreached when unavailable."""
|
||||
name = cfg.ssh_key_env
|
||||
agent = _AGENTS.get(name)
|
||||
if agent is None:
|
||||
agent = _AGENTS[name] = KeyAgent(name).start()
|
||||
if agent.error:
|
||||
log.event("direct-ssh unavailable in this process: %s" % agent.error, once="agent:" + name)
|
||||
raise Unreached("direct-ssh unavailable: %s" % agent.error)
|
||||
return agent
|
||||
|
||||
|
||||
def control_dir(cfg):
|
||||
"""agentd.ssh_dir: ControlMaster sockets + generated known_hosts; created 0700, must be ours."""
|
||||
d = cfg.ssh_dir
|
||||
try:
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(d, 0o700)
|
||||
if os.stat(d).st_uid != os.geteuid():
|
||||
raise OSError("not owned by this user")
|
||||
except OSError as exc:
|
||||
raise Unreached("ssh_dir %s unusable: %s" % (d, exc)) from None
|
||||
return d
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- channel
|
||||
class DirectChannel:
|
||||
"""One terminal's direct OpenSSH path. ps() / sh() mirror the warpgate channel; payloads go on stdin."""
|
||||
stdin = True
|
||||
|
||||
def __init__(self, cfg, terminal, log=None, label="direct"):
|
||||
self.cfg, self.terminal, self.log, self.via = cfg, terminal, log or NULL_LOG, label
|
||||
s = cfg.terminals[terminal]["ssh"]
|
||||
self.host, self.user, self.port = s["host"], s["user"], int(s.get("port", 22))
|
||||
self.host_key = " ".join(s["host_key"].split()[:2])
|
||||
self.pin = hashlib.sha256(self.host_key.encode("ascii")).hexdigest()[:8]
|
||||
self.alias = "agentd-" + terminal
|
||||
self.hostkey_failed = None
|
||||
self.name = "ssh %s@%s" % (self.user, self.host)
|
||||
|
||||
def known_hosts(self, d):
|
||||
path = d / ("known_hosts." + self.terminal)
|
||||
line = "%s %s\n" % (self.alias, self.host_key)
|
||||
try:
|
||||
if not path.exists() or path.read_text(encoding="ascii") != line:
|
||||
tmp = path.with_name("%s.%d.tmp" % (path.name, os.getpid())) # per terminal and process
|
||||
tmp.write_text(line, encoding="ascii")
|
||||
os.replace(tmp, path)
|
||||
except OSError as exc:
|
||||
raise Unreached("known_hosts %s: %s" % (path, exc)) from None
|
||||
return path
|
||||
|
||||
def argv(self, agent, remote_command):
|
||||
d = control_dir(self.cfg)
|
||||
ctl = str(d / (self.pin + "-%C")) # pin fingerprint: a new pin never reuses a master verified on the old one
|
||||
if len(str(d)) + CTL_NAME > SOCKET_MAX:
|
||||
self.log.event("agentd.ssh_dir %s is too long for a control socket; connection reuse off" % d, once="ctl")
|
||||
mux = ["-o", "ControlMaster=no", "-o", "ControlPath=none"]
|
||||
else:
|
||||
mux = ["-o", "ControlMaster=auto", "-o", "ControlPersist=%d" % CONTROL_PERSIST, "-o", "ControlPath=" + ctl]
|
||||
return (["ssh", "-F", "/dev/null", "-T", "-p", str(self.port), "-l", self.user,
|
||||
"-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=" + agent.sock, "-o", "IdentityFile=" + agent.pub,
|
||||
"-o", "PreferredAuthentications=publickey", "-o", "PasswordAuthentication=no",
|
||||
"-o", "KbdInteractiveAuthentication=no",
|
||||
"-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + str(self.known_hosts(d)),
|
||||
"-o", "GlobalKnownHostsFile=/dev/null", "-o", "HostKeyAlias=" + self.alias,
|
||||
"-o", "HostKeyAlgorithms=" + self.host_key.split()[0], "-o", "UpdateHostKeys=no",
|
||||
"-o", "ConnectTimeout=%d" % CONNECT_TIMEOUT, "-o", "ServerAliveInterval=15",
|
||||
"-o", "ServerAliveCountMax=3", "-o", "ForwardAgent=no", "-o", "ForwardX11=no",
|
||||
"-o", "ClearAllForwardings=yes", "-o", "LogLevel=ERROR", "-o", "EscapeChar=none"]
|
||||
+ mux + ["--", self.host, remote_command])
|
||||
|
||||
def run(self, remote_command, payload, timeout, op):
|
||||
"""-> (rc, stdout, stderr) of a call whose command ran; raises HostKeyError / Unreached / TransportError."""
|
||||
if self.hostkey_failed:
|
||||
raise HostKeyError(self.hostkey_failed)
|
||||
agent = key_agent(self.cfg, self.log)
|
||||
argv = self.argv(agent, remote_command)
|
||||
env = self.cfg.scrubbed_env()
|
||||
env.pop("SSH_AUTH_SOCK", None)
|
||||
t0 = time.monotonic()
|
||||
ms = lambda: (time.monotonic() - t0) * 1000 # noqa: E731
|
||||
# output goes to files, not pipes: a ControlPersist master forked by this call must never hold our pipes open
|
||||
with tempfile.TemporaryFile() as fo, tempfile.TemporaryFile() as fe:
|
||||
try:
|
||||
p = subprocess.run(argv, input=payload, stdout=fo, stderr=fe, env=env, timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.log.call(self.terminal, self.via, op, ms(), "timeout")
|
||||
raise TransportError("%s timeout after %ss" % (self.name, timeout)) from None # may have run
|
||||
except OSError as exc:
|
||||
self.log.call(self.terminal, self.via, op, ms(), "launch-failed")
|
||||
raise Unreached("%s: ssh launch failed: %s" % (self.name, type(exc).__name__)) from None
|
||||
fo.seek(0)
|
||||
fe.seek(0)
|
||||
dec = lambda b: b.decode("utf-8", "replace").replace("\r", "") # noqa: E731
|
||||
out, err = dec(fo.read()), dec(fe.read())
|
||||
kind = classify(p.returncode, err, out)
|
||||
self.log.call(self.terminal, self.via, op, ms(), p.returncode, "" if kind == "ok" else kind)
|
||||
if kind == "hostkey":
|
||||
self.hostkey_failed = ("%s:%d host key verification FAILED (pinned %s...; other key or no such key type "
|
||||
"offered): not falling back; check terminals.%s.ssh.host_key out of band"
|
||||
% (self.host, self.port, self.host_key[:40], self.terminal))
|
||||
self.log.event("SECURITY %s: %s" % (self.terminal, self.hostkey_failed), once="hostkey:" + self.terminal)
|
||||
raise HostKeyError(self.hostkey_failed)
|
||||
if kind == "unreached":
|
||||
raise Unreached("%s exit 255: %s" % (self.name, err.strip()[-300:]))
|
||||
if kind == "unknown":
|
||||
raise TransportError("%s exit 255: %s" % (self.name, err.strip()[-300:] or "(no message)"))
|
||||
return p.returncode, out, err
|
||||
|
||||
def ps(self, script, timeout, op=None):
|
||||
rc, out, err = self.run(PS_COMMAND, ps_payload(script), timeout, op)
|
||||
if rc != 0 and not out.strip():
|
||||
raise TransportError("%s exit %s: %s" % (self.name, rc, err.strip()[-300:]))
|
||||
return out
|
||||
|
||||
def sh(self, script, timeout, op=None):
|
||||
return self.run(SH_COMMAND, script.encode("utf-8"), timeout, op)
|
||||
@@ -91,5 +91,5 @@ class LocalTransport:
|
||||
pass
|
||||
|
||||
def process_count(self, name):
|
||||
out = subprocess.run(["pgrep", "-c", "-x", name], capture_output=True, text=True)
|
||||
out = subprocess.run(["pgrep", "-c", "-x", name], capture_output=True, text=True, env=self.cfg.scrubbed_env())
|
||||
return int(out.stdout.strip() or 0)
|
||||
|
||||
@@ -1,17 +1,25 @@
|
||||
"""Terminals behind Warpgate, reached through the server-access relay client:
|
||||
"""Remote terminals (Windows and Linux/macOS) and the Warpgate channel.
|
||||
|
||||
The transport classes build the scripts; a *channel* carries them (design §9):
|
||||
- WarpgateChannel (`transport: warpgate-ssh`, also the direct-ssh fallback): the server-access relay client
|
||||
|
||||
agent-access.py --config <identity> run --target <target> --timeout N -- <argv...>
|
||||
|
||||
The client shlex-joins argv and the target account's login shell runs it (cmd.exe on Windows); stdin is
|
||||
/dev/null, so everything travels in argv. `relay()` is the single place that builds that call.
|
||||
The client shlex-joins argv and the target account's login shell runs it (cmd.exe on Windows); stdin is
|
||||
/dev/null (agent-access.py run_remote uses stdin=DEVNULL), so everything travels in argv. `relay()` is the single
|
||||
place that builds that call.
|
||||
- DirectChannel (`transport: direct-ssh`, agentd/transports/direct_ssh.py): OpenSSH over Tailscale; the script goes
|
||||
on stdin, so every operation (including a launch with its prompt) is one call.
|
||||
|
||||
- os linux / darwin -> PosixWarpgateTransport: every operation is one `bash -c '<script>'` relay call. Files go up
|
||||
as base64 chunks (`printf %s '<b64>' | base64 -d`), the runner starts detached with `setsid nohup bash run.sh`
|
||||
(its own session: pid == pgid == sid), `poll` reads exit.txt, `kill` signals the process group and session.
|
||||
- os windows -> WindowsWarpgateTransport: still drives the helper scripts win10-ps.sh (PowerShell
|
||||
-EncodedCommand) and win10-put.sh (chunked upload, SHA-256 verified), which hard-code `--target win10`;
|
||||
runs are one-shot scheduled tasks (schtasks) so they outlive the SSH session.
|
||||
TODO: move Windows onto relay() (`powershell -EncodedCommand` as argv) and drop the helper scripts."""
|
||||
- os linux / darwin -> PosixWarpgateTransport: every operation is one `bash -c '<script>'` call. On the relay files go
|
||||
up as base64 chunks (`printf %s '<b64>' | base64 -d`), on the direct channel in the same single call. The runner
|
||||
starts detached with `setsid nohup bash run.sh` (its own session: pid == pgid == sid), `poll` reads exit.txt,
|
||||
`fetch` returns everything `collect` needs in one call, `kill` signals the process group and session.
|
||||
- os windows -> WindowsWarpgateTransport: PowerShell scripts; runs are one-shot scheduled tasks (schtasks) so they
|
||||
outlive the SSH session. On the relay it still drives the helper scripts win10-ps.sh (PowerShell -EncodedCommand,
|
||||
argv-size bound) and win10-put.sh (chunked upload, SHA-256 verified), which hard-code `--target win10`; on the
|
||||
direct channel mkdir + clean + prompt.md + run.ps1 + schtasks is one call.
|
||||
The class names are historical: the channel, not the class, decides the path."""
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
@@ -20,8 +28,9 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from . import TransportError
|
||||
from . import NULL_LOG, TransportError, Unreached, fetch_each, operation, parse_fetch
|
||||
from .local import RUNNER as POSIX_RUNNER, sh_quote
|
||||
|
||||
DEFAULT_CLIENT = "~/.local/share/server-access-client/agent-access.py"
|
||||
@@ -55,6 +64,55 @@ def relay(cfg, target, argv, timeout=150):
|
||||
return call(cfg, relay_argv(cfg, target, argv, timeout), timeout)
|
||||
|
||||
|
||||
PS_ARGV_MAX = 7600 # win10-ps.sh refuses longer -EncodedCommand values (cmd.exe line limit 8191)
|
||||
PS_PRELUDE = 120 # characters win10-ps.sh puts in front of every script
|
||||
|
||||
|
||||
class WarpgateChannel:
|
||||
"""Calls through agent-access.py run (argv only). Each call is logged with its duration."""
|
||||
stdin = False
|
||||
|
||||
def __init__(self, cfg, terminal, log=None, label="warpgate"):
|
||||
self.cfg, self.terminal, self.log, self.via = cfg, terminal, log or NULL_LOG, label
|
||||
t = cfg.terminals[terminal]
|
||||
self.target = t.get("target", terminal)
|
||||
self.name = "relay %s" % self.target
|
||||
if t["os"] == "windows":
|
||||
helpers = cfg.helpers_dir(terminal)
|
||||
self.ps_script, self.put_script = str(helpers / "win10-ps.sh"), str(helpers / "win10-put.sh")
|
||||
|
||||
def timed(self, op, fn, *args):
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
res = fn(*args)
|
||||
except TransportError:
|
||||
self.log.call(self.terminal, self.via, op, (time.monotonic() - t0) * 1000, "error")
|
||||
raise
|
||||
self.log.call(self.terminal, self.via, op, (time.monotonic() - t0) * 1000, res[0])
|
||||
return res
|
||||
|
||||
def fits(self, script):
|
||||
"""Whether win10-ps.sh accepts the script (its -EncodedCommand stays under PS_ARGV_MAX)."""
|
||||
units = len(script.encode("utf-16-le")) // 2 + PS_PRELUDE
|
||||
return (2 * units + 2) // 3 * 4 <= PS_ARGV_MAX
|
||||
|
||||
def ps(self, script, timeout, op=None):
|
||||
rc, out, err = self.timed(op, call, self.cfg, [self.ps_script, script], timeout, {"WIN10_TIMEOUT": str(timeout)})
|
||||
if rc != 0 and not out.strip():
|
||||
raise TransportError("relay exit %s: %s" % (rc, err.strip()[-300:]))
|
||||
return out
|
||||
|
||||
def put(self, local, remote, op=None):
|
||||
with open(local, "rb") as fh:
|
||||
sha = hashlib.sha256(fh.read()).hexdigest()
|
||||
rc, out, err = self.timed(op, call, self.cfg, [self.put_script, local, remote], 540)
|
||||
if ("remote sha256 " + sha) not in out:
|
||||
raise TransportError("upload of %s failed: %s" % (remote, (out + err).strip()[-300:]))
|
||||
|
||||
def sh(self, script, timeout, op=None):
|
||||
return self.timed(op, relay, self.cfg, self.target, ["bash", "-c", script], timeout)
|
||||
|
||||
|
||||
RUNNER = r"""$ErrorActionPreference = 'Continue'
|
||||
[Console]::OutputEncoding = [Text.Encoding]::UTF8
|
||||
$OutputEncoding = [Text.Encoding]::UTF8
|
||||
@@ -68,6 +126,15 @@ Set-Content -Path '{d}\exit.txt' -Value ("exit=" + $LASTEXITCODE + " at " + (Get
|
||||
"""
|
||||
|
||||
|
||||
# Windows fetch helpers: F<i>=<base64> per result, M<i> = file missing (parse_fetch). Names must not collide with
|
||||
# PowerShell's built-in aliases (aliases win over functions: `r` is Invoke-History).
|
||||
PS_FETCH = r"""function AdRead($i,$n){$p=Join-Path $d $n; if(Test-Path -LiteralPath $p){"F$i="+[Convert]::ToBase64String([IO.File]::ReadAllBytes($p))}else{"M$i"}}
|
||||
function AdOut($i,$t){"F$i="+[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))}
|
||||
function AdGrep($i,$n,$x){$p=Join-Path $d $n; $m=$null; if(Test-Path -LiteralPath $p){$m=Select-String -LiteralPath $p -SimpleMatch -Pattern $x|Select-Object -First 1}; if($m){AdOut $i $m.Line}else{"F$i="}}
|
||||
function AdTail($i,$n,$k){$p=Join-Path $d $n; $t=''; if(Test-Path -LiteralPath $p){$t=(Get-Content -LiteralPath $p -Encoding UTF8 -Tail $k) -join "`n"}; AdOut $i $t}
|
||||
"""
|
||||
|
||||
|
||||
def _b64_payload(out, marker):
|
||||
m = re.search(marker + r"=([A-Za-z0-9+/=]*)", out)
|
||||
if not m:
|
||||
@@ -78,12 +145,11 @@ def _b64_payload(out, marker):
|
||||
class WindowsWarpgateTransport:
|
||||
os = "windows"
|
||||
|
||||
def __init__(self, cfg, terminal):
|
||||
def __init__(self, cfg, terminal, channel=None, log=None, label="warpgate"):
|
||||
self.cfg, self.terminal = cfg, terminal
|
||||
t = cfg.terminals[terminal]
|
||||
self.target = t.get("target", terminal)
|
||||
helpers = cfg.helpers_dir(terminal)
|
||||
self.ps_script, self.put_script = str(helpers / "win10-ps.sh"), str(helpers / "win10-put.sh")
|
||||
self.channel = channel or WarpgateChannel(cfg, terminal, log, label)
|
||||
self.repo = t["repo"]
|
||||
|
||||
def default_run_root(self):
|
||||
@@ -95,32 +161,38 @@ class WindowsWarpgateTransport:
|
||||
def quote(self, s):
|
||||
return "'" + str(s).replace("'", "''") + "'"
|
||||
|
||||
# ---------------------------------------------------------------- relay primitives
|
||||
# ---------------------------------------------------------------- channel primitives
|
||||
def ps(self, script, timeout=150):
|
||||
rc, out, err = call(self.cfg, [self.ps_script, script], timeout, {"WIN10_TIMEOUT": str(timeout)})
|
||||
if rc != 0 and not out.strip():
|
||||
raise TransportError("relay exit %s: %s" % (rc, err.strip()[-300:]))
|
||||
return out
|
||||
return self.channel.ps(script, timeout, getattr(self, "_op", None))
|
||||
|
||||
def put(self, local, remote):
|
||||
with open(local, "rb") as fh:
|
||||
sha = hashlib.sha256(fh.read()).hexdigest()
|
||||
rc, out, err = call(self.cfg, [self.put_script, local, remote], 540)
|
||||
if ("remote sha256 " + sha) not in out:
|
||||
raise TransportError("upload of %s failed: %s" % (remote, (out + err).strip()[-300:]))
|
||||
self.channel.put(local, remote, getattr(self, "_op", None))
|
||||
|
||||
@operation("ping")
|
||||
def ping(self):
|
||||
out = self.ps("'PONG=' + $PSVersionTable.PSVersion.ToString() + ' ' + $env:COMPUTERNAME", timeout=60)
|
||||
m = re.search(r"PONG=[^\n]*", out)
|
||||
if not m:
|
||||
raise TransportError("ping %s: unexpected output: %s" % (self.terminal, out.strip()[-200:]))
|
||||
return m.group(0)
|
||||
|
||||
# ---------------------------------------------------------------- file ops
|
||||
def mkdir(self, path, clean=False):
|
||||
def mkdir_script(self, path, clean=False):
|
||||
q = self.quote(path)
|
||||
script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q
|
||||
if clean: # never wipe a directory a runner has already started in
|
||||
script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) "
|
||||
"{ 'HASPID'; return }\n" % (q, q))
|
||||
script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q
|
||||
out = self.ps(script + "'MKDIR=ok'")
|
||||
return script
|
||||
|
||||
@operation("mkdir")
|
||||
def mkdir(self, path, clean=False):
|
||||
out = self.ps(self.mkdir_script(path, clean) + "'MKDIR=ok'")
|
||||
if "MKDIR=ok" not in out:
|
||||
raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:]))
|
||||
|
||||
@operation("probe")
|
||||
def probe(self, run_dir, names):
|
||||
out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')"
|
||||
% (self.quote(run_dir), ",".join(self.quote(n) for n in names)))
|
||||
@@ -129,7 +201,20 @@ class WindowsWarpgateTransport:
|
||||
raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:]))
|
||||
return [n for n in m.group(1).strip().split(",") if n]
|
||||
|
||||
def write_script(self, path, data):
|
||||
"""Write bytes exactly as given (UTF-8 without BOM for our texts), echo their SHA-256."""
|
||||
return ("$p = %s\n[IO.File]::WriteAllBytes($p, [Convert]::FromBase64String('%s'))\n"
|
||||
"'WROTE=' + (Get-FileHash -LiteralPath $p -Algorithm SHA256).Hash.ToLower()\n"
|
||||
% (self.quote(path), base64.b64encode(data).decode("ascii")))
|
||||
|
||||
@operation("write")
|
||||
def write_text(self, path, text):
|
||||
if self.channel.stdin:
|
||||
data = text.encode("utf-8")
|
||||
out = self.ps(self.write_script(path, data))
|
||||
if re.findall(r"WROTE=([0-9a-f]{64})", out) != [hashlib.sha256(data).hexdigest()]:
|
||||
raise TransportError("upload of %s failed: %s" % (path, out.strip()[-300:]))
|
||||
return
|
||||
fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="") as fh:
|
||||
@@ -138,6 +223,7 @@ class WindowsWarpgateTransport:
|
||||
finally:
|
||||
os.unlink(tmp)
|
||||
|
||||
@operation("read")
|
||||
def read_text(self, path):
|
||||
q = self.quote(path)
|
||||
out = self.ps("if (Test-Path -LiteralPath %s) { 'B64=' + [Convert]::ToBase64String([IO.File]::ReadAllBytes(%s)) }"
|
||||
@@ -147,8 +233,9 @@ class WindowsWarpgateTransport:
|
||||
text = _b64_payload(out, "B64")
|
||||
if text is None:
|
||||
raise TransportError("read %s: unexpected relay output" % path)
|
||||
return text.lstrip("")
|
||||
return text.lstrip("\ufeff")
|
||||
|
||||
@operation("grep")
|
||||
def grep_first(self, path, needle):
|
||||
q = self.quote(path)
|
||||
out = self.ps("$m = if (Test-Path -LiteralPath %s) { Select-String -LiteralPath %s -SimpleMatch -Pattern %s | Select-Object -First 1 }\n"
|
||||
@@ -156,12 +243,30 @@ class WindowsWarpgateTransport:
|
||||
% (q, q, self.quote(needle)))
|
||||
return _b64_payload(out, "B64") or None
|
||||
|
||||
@operation("tail")
|
||||
def tail(self, path, n):
|
||||
q = self.quote(path)
|
||||
out = self.ps("$t = if (Test-Path -LiteralPath %s) { (Get-Content -LiteralPath %s -Encoding UTF8 -Tail %d) -join \"`n\" } else { '' }\n"
|
||||
"'B64=' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$t))" % (q, q, n))
|
||||
return _b64_payload(out, "B64") or ""
|
||||
|
||||
def fetch_script(self, run_dir, ops):
|
||||
lines = ["$d = %s" % self.quote(run_dir), PS_FETCH.rstrip("\n")]
|
||||
for i, o in enumerate(ops):
|
||||
fn = {"read": "AdRead", "grep": "AdGrep", "tail": "AdTail"}[o[0]]
|
||||
arg = "" if o[0] == "read" else " " + (self.quote(o[2]) if o[0] == "grep" else str(int(o[2])))
|
||||
lines.append("%s %d %s%s" % (fn, i, self.quote(o[1]), arg))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
@operation("fetch")
|
||||
def fetch(self, run_dir, ops):
|
||||
"""Several reads of one run directory in one call (collect)."""
|
||||
script = self.fetch_script(run_dir, ops)
|
||||
if not self.channel.stdin and not self.channel.fits(script):
|
||||
return fetch_each(self, run_dir, ops)
|
||||
res = parse_fetch(self.ps(script, timeout=200), ops, run_dir)
|
||||
return [r.lstrip("\ufeff") if o[0] == "read" and r else r for o, r in zip(ops, res)]
|
||||
|
||||
# ---------------------------------------------------------------- processes
|
||||
def runner(self, run_dir, cwd, command, stdout_name):
|
||||
return "run.ps1", RUNNER.format(cwd=self.quote(cwd), d=run_dir.replace("'", "''"), command=command, stdout=stdout_name)
|
||||
@@ -176,23 +281,60 @@ class WindowsWarpgateTransport:
|
||||
"schtasks /Run /TN '%(t)s' | Out-Null\n"
|
||||
"'LAUNCHED=' + $LASTEXITCODE" % dict(t=task.replace("'", "''"), s=script_path.replace("'", "''")))
|
||||
|
||||
@operation("launch")
|
||||
def launch(self, task, script_path, run_dir):
|
||||
out = self.ps(self.launch_script(task, script_path))
|
||||
if "LAUNCHED=0" not in out:
|
||||
raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:]))
|
||||
|
||||
def start_script(self, task, run_dir, files, script_name):
|
||||
"""mkdir + clean (HASPID refusal) + every file + schtasks, as one script. -> (script, [sha256 per file])"""
|
||||
script, shas = "$ErrorActionPreference = 'Stop'\n" + self.mkdir_script(run_dir, clean=True), []
|
||||
for name, text in files.items():
|
||||
data = text.encode("utf-8")
|
||||
script += self.write_script(self.join(run_dir, name), data)
|
||||
shas.append(hashlib.sha256(data).hexdigest())
|
||||
script += "$ErrorActionPreference = 'Continue'\n" + self.launch_script(task, self.join(run_dir, script_name))
|
||||
return script, shas
|
||||
|
||||
@operation("start")
|
||||
def start_run(self, task, run_dir, files, script_name):
|
||||
"""Stage the run directory and start the task: one call on the direct channel; on the relay (argv only)
|
||||
mkdir, chunked uploads and the launch go one by one."""
|
||||
if not self.channel.stdin:
|
||||
self.mkdir(run_dir, clean=True)
|
||||
for name, text in files.items():
|
||||
self.write_text(self.join(run_dir, name), text)
|
||||
return self.launch(task, self.join(run_dir, script_name), run_dir)
|
||||
script, shas = self.start_script(task, run_dir, files, script_name)
|
||||
out = self.ps(script)
|
||||
if re.search(r"^HASPID$", out, re.M):
|
||||
raise TransportError("refusing to clean %s: it holds a pid file (HASPID)" % run_dir)
|
||||
wrote = re.findall(r"WROTE=([0-9a-f]{64})", out)
|
||||
if wrote != shas:
|
||||
raise TransportError("staging %s: hash mismatch (%d/%d files): %s" % (run_dir, len(wrote), len(shas),
|
||||
out.strip()[-300:]))
|
||||
if "LAUNCHED=0" not in out:
|
||||
raise TransportError("schtasks launch of %s failed: %s" % (task, out.strip()[-300:]))
|
||||
|
||||
@operation("kill")
|
||||
def kill(self, task, run_dir):
|
||||
t = task.replace("'", "''")
|
||||
self.ps("$p = Join-Path %s 'pid.txt'\n"
|
||||
"if (Test-Path $p) { taskkill /T /F /PID (Get-Content $p | Select-Object -First 1) 2>&1 | Out-Null }\n"
|
||||
"schtasks /End /TN '%s' 2>&1 | Out-Null\n"
|
||||
"schtasks /Delete /F /TN '%s' 2>&1 | Out-Null\n'KILLED=ok'" % (self.quote(run_dir), task, task))
|
||||
"schtasks /Delete /F /TN '%s' 2>&1 | Out-Null\n'KILLED=ok'" % (self.quote(run_dir), t, t))
|
||||
|
||||
@operation("forget")
|
||||
def forget(self, task):
|
||||
self.ps("schtasks /Delete /F /TN '%s' 2>&1 | Out-Null; 'FORGOT=ok'" % task)
|
||||
self.ps("schtasks /Delete /F /TN '%s' 2>&1 | Out-Null; 'FORGOT=ok'" % task.replace("'", "''"))
|
||||
|
||||
@operation("busy")
|
||||
def process_count(self, name):
|
||||
try:
|
||||
out = self.ps("'COUNT=' + @(Get-Process -Name %s -ErrorAction SilentlyContinue).Count" % self.quote(name))
|
||||
except Unreached:
|
||||
raise # a direct-ssh terminal retries this on its fallback
|
||||
except TransportError:
|
||||
return None
|
||||
m = re.search(r"COUNT=(\d+)", out)
|
||||
@@ -210,15 +352,17 @@ DETACH = {
|
||||
|
||||
|
||||
class PosixWarpgateTransport:
|
||||
"""Linux/macOS terminal through the relay. Each primitive is one `bash -c` round trip (~10 s through
|
||||
Vault + Warpgate), so `start_run` stages the run directory and launches it in a single call."""
|
||||
CHUNK = 60000 # base64 chars per call: one argv element must stay below 128 KiB (MAX_ARG_STRLEN) on both ends
|
||||
"""Linux/macOS terminal. Each primitive is one `bash -c` round trip (~10 s through Vault + Warpgate, well under
|
||||
a second on a reused direct connection), so `start_run` stages the run directory and launches it in a single
|
||||
call and `fetch` reads everything `collect` needs in one."""
|
||||
CHUNK = 60000 # relay only: base64 chars per call; one argv element must stay below 128 KiB (MAX_ARG_STRLEN)
|
||||
|
||||
def __init__(self, cfg, terminal):
|
||||
def __init__(self, cfg, terminal, channel=None, log=None, label="warpgate"):
|
||||
self.cfg, self.terminal = cfg, terminal
|
||||
t = cfg.terminals[terminal]
|
||||
self.os = t["os"]
|
||||
self.target = t.get("target", terminal)
|
||||
self.channel = channel or WarpgateChannel(cfg, terminal, log, label)
|
||||
self.repo = t["repo"]
|
||||
|
||||
def default_run_root(self):
|
||||
@@ -230,14 +374,19 @@ class PosixWarpgateTransport:
|
||||
def quote(self, s):
|
||||
return sh_quote(s)
|
||||
|
||||
# ---------------------------------------------------------------- relay primitive
|
||||
# ---------------------------------------------------------------- channel primitive
|
||||
def sh(self, script, timeout=150, marker=None):
|
||||
"""Run a bash script on the target. With `marker`, its absence in stdout is an error."""
|
||||
rc, out, err = relay(self.cfg, self.target, ["bash", "-c", script], timeout)
|
||||
rc, out, err = self.channel.sh(script, timeout, getattr(self, "_op", None))
|
||||
if (marker and marker not in out) or (not marker and rc != 0 and not out.strip()):
|
||||
raise TransportError("relay %s exit %s: %s" % (self.target, rc, (out + err).strip()[-300:]))
|
||||
raise TransportError("%s exit %s: %s" % (self.channel.name, rc, (out + err).strip()[-300:]))
|
||||
return out
|
||||
|
||||
@operation("ping")
|
||||
def ping(self):
|
||||
out = self.sh("echo \"PONG=$(uname -sr) $(uname -n)\"", timeout=60, marker="PONG=")
|
||||
return re.search(r"PONG=[^\n]*", out).group(0)
|
||||
|
||||
# ---------------------------------------------------------------- file ops
|
||||
def mkdir_script(self, path, clean=False):
|
||||
q = self.quote(path)
|
||||
@@ -248,6 +397,7 @@ class PosixWarpgateTransport:
|
||||
s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q
|
||||
return s
|
||||
|
||||
@operation("mkdir")
|
||||
def mkdir(self, path, clean=False):
|
||||
self.sh(self.mkdir_script(path, clean) + "echo MKDIR=ok\n", marker="MKDIR=ok")
|
||||
|
||||
@@ -261,10 +411,11 @@ class PosixWarpgateTransport:
|
||||
return ("printf %%s '%s' | base64 -d > %s && mv -f %s %s || exit 1\n"
|
||||
"echo \"WROTE=$(wc -c < %s | tr -d ' ')\"\n" % (b64, part, part, q, q))
|
||||
|
||||
@operation("write")
|
||||
def write_text(self, path, text):
|
||||
data = self.b64(text)
|
||||
size = len(text.encode("utf-8"))
|
||||
if len(data) <= self.CHUNK:
|
||||
if len(data) <= self.CHUNK or self.channel.stdin: # stdin carries the script: no argv limit
|
||||
out = self.sh(self.put_script(path, data), marker="WROTE=")
|
||||
else: # append chunks to <path>.b64, decode on the last call
|
||||
q, qb = self.quote(path), self.quote(path + ".b64")
|
||||
@@ -279,6 +430,7 @@ class PosixWarpgateTransport:
|
||||
if re.findall(r"WROTE=(\d+)", out) != [str(size)]:
|
||||
raise TransportError("upload of %s: size mismatch (%s)" % (path, out.strip()[-200:]))
|
||||
|
||||
@operation("read")
|
||||
def read_text(self, path):
|
||||
q = self.quote(path)
|
||||
out = self.sh("if [ -f %s ]; then printf B64=; base64 < %s | tr -d '\\n'; echo; else echo MISSING; fi" % (q, q),
|
||||
@@ -289,6 +441,7 @@ class PosixWarpgateTransport:
|
||||
raise TransportError("read %s: unexpected relay output" % path)
|
||||
return _b64_payload(out, "B64")
|
||||
|
||||
@operation("probe")
|
||||
def probe(self, run_dir, names):
|
||||
d = self.quote(run_dir)
|
||||
out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo"
|
||||
@@ -302,13 +455,33 @@ class PosixWarpgateTransport:
|
||||
raise TransportError("relay %s: unexpected output" % self.target)
|
||||
return text
|
||||
|
||||
@operation("grep")
|
||||
def grep_first(self, path, needle):
|
||||
line = self._b64_cmd("grep -F -m1 -e %s %s" % (self.quote(needle), self.quote(path)))
|
||||
return line.rstrip("\n") or None
|
||||
|
||||
@operation("tail")
|
||||
def tail(self, path, n):
|
||||
return self._b64_cmd("tail -n %d %s" % (int(n), self.quote(path))).rstrip("\n")
|
||||
|
||||
def fetch_script(self, run_dir, ops):
|
||||
s = "d=%s\n" % self.quote(run_dir)
|
||||
for i, o in enumerate(ops):
|
||||
f = "\"$d\"/%s" % self.quote(o[1])
|
||||
if o[0] == "read":
|
||||
s += ("if [ -f %s ]; then printf F%d=; base64 < %s | tr -d '\\n'; echo; else echo M%d; fi\n"
|
||||
% (f, i, f, i))
|
||||
else:
|
||||
cmd = ("grep -F -m1 -e %s %s" % (self.quote(o[2]), f) if o[0] == "grep"
|
||||
else "tail -n %d %s" % (int(o[2]), f))
|
||||
s += "printf F%d=; { %s; } 2>/dev/null | base64 | tr -d '\\n'; echo\n" % (i, cmd)
|
||||
return s + "echo FETCH=ok\n"
|
||||
|
||||
@operation("fetch")
|
||||
def fetch(self, run_dir, ops):
|
||||
"""Several reads of one run directory in one call (collect)."""
|
||||
return parse_fetch(self.sh(self.fetch_script(run_dir, ops), timeout=200, marker="FETCH=ok"), ops, run_dir)
|
||||
|
||||
# ---------------------------------------------------------------- processes
|
||||
def runner(self, run_dir, cwd, command, stdout_name):
|
||||
return "run.sh", POSIX_RUNNER.format(cwd=self.quote(cwd), d=self.quote(run_dir), command=command,
|
||||
@@ -323,13 +496,15 @@ class PosixWarpgateTransport:
|
||||
"echo $! > launch.pid\n"
|
||||
"echo LAUNCHED=$!\n" % (d, DETACH[self.os] % self.quote(script_name)))
|
||||
|
||||
@operation("launch")
|
||||
def launch(self, task, script_path, run_dir):
|
||||
self.sh(self.launch_script(run_dir, posixpath.basename(script_path)), marker="LAUNCHED=")
|
||||
|
||||
@operation("start")
|
||||
def start_run(self, task, run_dir, files, script_name):
|
||||
"""mkdir + clean, write `files` {name: text}, launch `script_name` -- one relay call unless a file is
|
||||
larger than CHUNK (those go up first through write_text)."""
|
||||
big = {n: t for n, t in files.items() if len(self.b64(t)) > self.CHUNK}
|
||||
"""mkdir + clean, write `files` {name: text}, launch `script_name` -- one call; on the relay a file larger
|
||||
than CHUNK goes up first through write_text."""
|
||||
big = {} if self.channel.stdin else {n: t for n, t in files.items() if len(self.b64(t)) > self.CHUNK}
|
||||
script = self.mkdir_script(run_dir, clean=True)
|
||||
if big:
|
||||
self.mkdir(run_dir, clean=True)
|
||||
@@ -359,23 +534,26 @@ class PosixWarpgateTransport:
|
||||
"kill -KILL -- \"-$p\" 2>/dev/null; pkill -KILL -s \"$p\" 2>/dev/null\n"
|
||||
"sleep 1; echo \"KILLED=ok left=$(pgrep -s \"$p\" 2>/dev/null | wc -l | tr -d ' ')\"\n" % d)
|
||||
|
||||
@operation("kill")
|
||||
def kill(self, task, run_dir):
|
||||
self.sh(self.kill_script(run_dir), marker="KILLED=")
|
||||
|
||||
def forget(self, task):
|
||||
pass
|
||||
|
||||
@operation("busy")
|
||||
def process_count(self, name):
|
||||
try:
|
||||
out = self.sh("echo COUNT=$(pgrep -x %s | wc -l | tr -d ' ')" % self.quote(name), timeout=60)
|
||||
except Unreached:
|
||||
raise # a direct-ssh terminal retries this on its fallback
|
||||
except TransportError:
|
||||
return None
|
||||
m = re.search(r"COUNT=(\d+)", out)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def WarpgateSshTransport(cfg, terminal):
|
||||
"""Factory: the transport class for a warpgate-ssh terminal follows its `os`."""
|
||||
if cfg.terminals[terminal]["os"] == "windows":
|
||||
return WindowsWarpgateTransport(cfg, terminal)
|
||||
return PosixWarpgateTransport(cfg, terminal)
|
||||
def WarpgateSshTransport(cfg, terminal, channel=None, log=None, label="warpgate"):
|
||||
"""Factory: the transport class for a remote terminal follows its `os`; the channel defaults to Warpgate."""
|
||||
cls = WindowsWarpgateTransport if cfg.terminals[terminal]["os"] == "windows" else PosixWarpgateTransport
|
||||
return cls(cfg, terminal, channel=channel, log=log, label=label)
|
||||
|
||||
Reference in New Issue
Block a user