Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick

Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching
marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run
directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same
run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot.
Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped
for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
jj.wang
2026-10-21 11:54:40 +00:00
parent dbe305103c
commit 4f45d368ad
8 changed files with 357 additions and 50 deletions
+9
View File
@@ -4,6 +4,10 @@ import signal
import subprocess
from pathlib import Path
from . import TransportError
PID_FILES = ("pid.txt", "launch.pid") # a run directory holding one of these is never cleaned
# Shared with the remote POSIX transport (warpgate_ssh): plain sh + `date +FMT`, so it also runs on macOS.
RUNNER = """#!/usr/bin/env bash
cd {cwd} || {{ echo "exit=97 at $(date +%Y-%m-%dT%H:%M:%S%z) (cd failed)" > {d}/exit.txt; exit 97; }}
@@ -40,6 +44,8 @@ class LocalTransport:
p = Path(path)
p.mkdir(parents=True, exist_ok=True)
if clean:
if any((p / n).exists() for n in PID_FILES):
raise TransportError("refusing to clean %s: it holds a pid file (HASPID)" % p)
for f in p.iterdir():
if f.is_file():
f.unlink()
@@ -51,6 +57,9 @@ class LocalTransport:
p = Path(path)
return p.read_text(encoding="utf-8", errors="replace") if p.exists() else None
def probe(self, run_dir, names):
return [n for n in names if os.path.exists(os.path.join(run_dir, n))]
def grep_first(self, path, needle):
text = self.read_text(path) or ""
return next((line for line in text.splitlines() if needle in line), None)
+23 -3
View File
@@ -113,9 +113,21 @@ class WindowsWarpgateTransport:
def mkdir(self, path, clean=False):
q = self.quote(path)
script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q
if clean:
if clean: # never wipe a directory a runner has already started in
script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) "
"{ 'HASPID'; return }\n" % (q, q))
script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q
self.ps(script + "'MKDIR=ok'")
out = self.ps(script + "'MKDIR=ok'")
if "MKDIR=ok" not in out:
raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:]))
def probe(self, run_dir, names):
out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')"
% (self.quote(run_dir), ",".join(self.quote(n) for n in names)))
m = re.search(r"PROBE=([^\n]*)", out)
if not m:
raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:]))
return [n for n in m.group(1).strip().split(",") if n]
def write_text(self, path, text):
fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt")
@@ -230,7 +242,9 @@ class PosixWarpgateTransport:
def mkdir_script(self, path, clean=False):
q = self.quote(path)
s = "mkdir -p %s || exit 1\n" % q
if clean:
if clean: # never wipe a directory a runner has already started in
s += "if [ -e %s ] || [ -e %s ]; then echo HASPID; exit 1; fi\n" % (
self.quote(posixpath.join(path, "pid.txt")), self.quote(posixpath.join(path, "launch.pid")))
s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q
return s
@@ -275,6 +289,12 @@ class PosixWarpgateTransport:
raise TransportError("read %s: unexpected relay output" % path)
return _b64_payload(out, "B64")
def probe(self, run_dir, names):
d = self.quote(run_dir)
out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo"
% (" ".join(self.quote(n) for n in names), d), timeout=60, marker="PROBE=")
return [n for n in re.search(r"PROBE=([^\n]*)", out).group(1).split(",") if n]
def _b64_cmd(self, pipeline):
out = self.sh("printf B64=; { %s; } 2>/dev/null | base64 | tr -d '\\n'; echo" % pipeline)
text = _b64_payload(out, "B64")