Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick
Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot. Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
@@ -4,6 +4,10 @@ import signal
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from . import TransportError
|
||||
|
||||
PID_FILES = ("pid.txt", "launch.pid") # a run directory holding one of these is never cleaned
|
||||
|
||||
# Shared with the remote POSIX transport (warpgate_ssh): plain sh + `date +FMT`, so it also runs on macOS.
|
||||
RUNNER = """#!/usr/bin/env bash
|
||||
cd {cwd} || {{ echo "exit=97 at $(date +%Y-%m-%dT%H:%M:%S%z) (cd failed)" > {d}/exit.txt; exit 97; }}
|
||||
@@ -40,6 +44,8 @@ class LocalTransport:
|
||||
p = Path(path)
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
if clean:
|
||||
if any((p / n).exists() for n in PID_FILES):
|
||||
raise TransportError("refusing to clean %s: it holds a pid file (HASPID)" % p)
|
||||
for f in p.iterdir():
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
@@ -51,6 +57,9 @@ class LocalTransport:
|
||||
p = Path(path)
|
||||
return p.read_text(encoding="utf-8", errors="replace") if p.exists() else None
|
||||
|
||||
def probe(self, run_dir, names):
|
||||
return [n for n in names if os.path.exists(os.path.join(run_dir, n))]
|
||||
|
||||
def grep_first(self, path, needle):
|
||||
text = self.read_text(path) or ""
|
||||
return next((line for line in text.splitlines() if needle in line), None)
|
||||
|
||||
@@ -113,9 +113,21 @@ class WindowsWarpgateTransport:
|
||||
def mkdir(self, path, clean=False):
|
||||
q = self.quote(path)
|
||||
script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q
|
||||
if clean:
|
||||
if clean: # never wipe a directory a runner has already started in
|
||||
script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) "
|
||||
"{ 'HASPID'; return }\n" % (q, q))
|
||||
script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q
|
||||
self.ps(script + "'MKDIR=ok'")
|
||||
out = self.ps(script + "'MKDIR=ok'")
|
||||
if "MKDIR=ok" not in out:
|
||||
raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:]))
|
||||
|
||||
def probe(self, run_dir, names):
|
||||
out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')"
|
||||
% (self.quote(run_dir), ",".join(self.quote(n) for n in names)))
|
||||
m = re.search(r"PROBE=([^\n]*)", out)
|
||||
if not m:
|
||||
raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:]))
|
||||
return [n for n in m.group(1).strip().split(",") if n]
|
||||
|
||||
def write_text(self, path, text):
|
||||
fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt")
|
||||
@@ -230,7 +242,9 @@ class PosixWarpgateTransport:
|
||||
def mkdir_script(self, path, clean=False):
|
||||
q = self.quote(path)
|
||||
s = "mkdir -p %s || exit 1\n" % q
|
||||
if clean:
|
||||
if clean: # never wipe a directory a runner has already started in
|
||||
s += "if [ -e %s ] || [ -e %s ]; then echo HASPID; exit 1; fi\n" % (
|
||||
self.quote(posixpath.join(path, "pid.txt")), self.quote(posixpath.join(path, "launch.pid")))
|
||||
s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q
|
||||
return s
|
||||
|
||||
@@ -275,6 +289,12 @@ class PosixWarpgateTransport:
|
||||
raise TransportError("read %s: unexpected relay output" % path)
|
||||
return _b64_payload(out, "B64")
|
||||
|
||||
def probe(self, run_dir, names):
|
||||
d = self.quote(run_dir)
|
||||
out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo"
|
||||
% (" ".join(self.quote(n) for n in names), d), timeout=60, marker="PROBE=")
|
||||
return [n for n in re.search(r"PROBE=([^\n]*)", out).group(1).split(",") if n]
|
||||
|
||||
def _b64_cmd(self, pipeline):
|
||||
out = self.sh("printf B64=; { %s; } 2>/dev/null | base64 | tr -d '\\n'; echo" % pipeline)
|
||||
text = _b64_payload(out, "B64")
|
||||
|
||||
Reference in New Issue
Block a user