Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick
Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot. Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
@@ -113,9 +113,21 @@ class WindowsWarpgateTransport:
|
||||
def mkdir(self, path, clean=False):
|
||||
q = self.quote(path)
|
||||
script = "New-Item -ItemType Directory -Force -Path %s | Out-Null\n" % q
|
||||
if clean:
|
||||
if clean: # never wipe a directory a runner has already started in
|
||||
script += ("if ((Test-Path -LiteralPath (Join-Path %s 'pid.txt')) -or (Test-Path -LiteralPath (Join-Path %s 'launch.pid'))) "
|
||||
"{ 'HASPID'; return }\n" % (q, q))
|
||||
script += "Get-ChildItem -LiteralPath %s -File | Remove-Item -Force\n" % q
|
||||
self.ps(script + "'MKDIR=ok'")
|
||||
out = self.ps(script + "'MKDIR=ok'")
|
||||
if "MKDIR=ok" not in out:
|
||||
raise TransportError("mkdir %s failed: %s" % (path, out.strip()[-300:]))
|
||||
|
||||
def probe(self, run_dir, names):
|
||||
out = self.ps("$d = %s\n'PROBE=' + ((@(%s) | Where-Object { Test-Path -LiteralPath (Join-Path $d $_) }) -join ',')"
|
||||
% (self.quote(run_dir), ",".join(self.quote(n) for n in names)))
|
||||
m = re.search(r"PROBE=([^\n]*)", out)
|
||||
if not m:
|
||||
raise TransportError("probe %s: unexpected relay output: %s" % (run_dir, out.strip()[-200:]))
|
||||
return [n for n in m.group(1).strip().split(",") if n]
|
||||
|
||||
def write_text(self, path, text):
|
||||
fd, tmp = tempfile.mkstemp(prefix="agentd-", suffix=".txt")
|
||||
@@ -230,7 +242,9 @@ class PosixWarpgateTransport:
|
||||
def mkdir_script(self, path, clean=False):
|
||||
q = self.quote(path)
|
||||
s = "mkdir -p %s || exit 1\n" % q
|
||||
if clean:
|
||||
if clean: # never wipe a directory a runner has already started in
|
||||
s += "if [ -e %s ] || [ -e %s ]; then echo HASPID; exit 1; fi\n" % (
|
||||
self.quote(posixpath.join(path, "pid.txt")), self.quote(posixpath.join(path, "launch.pid")))
|
||||
s += "find %s -maxdepth 1 -type f -exec rm -f {} + || exit 1\n" % q
|
||||
return s
|
||||
|
||||
@@ -275,6 +289,12 @@ class PosixWarpgateTransport:
|
||||
raise TransportError("read %s: unexpected relay output" % path)
|
||||
return _b64_payload(out, "B64")
|
||||
|
||||
def probe(self, run_dir, names):
|
||||
d = self.quote(run_dir)
|
||||
out = self.sh("printf PROBE=; for f in %s; do [ -e %s/\"$f\" ] && printf '%%s,' \"$f\"; done; echo"
|
||||
% (" ".join(self.quote(n) for n in names), d), timeout=60, marker="PROBE=")
|
||||
return [n for n in re.search(r"PROBE=([^\n]*)", out).group(1).split(",") if n]
|
||||
|
||||
def _b64_cmd(self, pipeline):
|
||||
out = self.sh("printf B64=; { %s; } 2>/dev/null | base64 | tr -d '\\n'; echo" % pipeline)
|
||||
text = _b64_payload(out, "B64")
|
||||
|
||||
Reference in New Issue
Block a user