Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick
Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot. Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
+27
-2
@@ -6,6 +6,7 @@ from agentd.config import Config
|
||||
from agentd.core import Agentd
|
||||
from agentd.executors import ExecResult
|
||||
from agentd.store import Store
|
||||
from agentd.transports import TransportError
|
||||
|
||||
STATES = {"Backlog": "s-bl", "Todo": "s-todo", "In Progress": "s-run", "Waiting": "s-wait", "Review": "s-rev", "Done": "s-done"}
|
||||
MEMBERS = {"window10": "u-w10", "oregon": "u-or", "kappiee": "u-k"}
|
||||
@@ -89,15 +90,39 @@ class FakeExecutor:
|
||||
self.resume = resume
|
||||
self.started, self.killed, self.done = [], [], {}
|
||||
self.fail_starts, self.fail_kills = 0, 0
|
||||
self.lost_starts = 0 # the run launches but the relay reports a failure (e.g. timeout)
|
||||
self.fail_probes = self.fail_polls = 0
|
||||
self.dirs = {} # run_dir -> files on the "terminal"
|
||||
self.polls = []
|
||||
|
||||
def handle(self, run):
|
||||
return {"run_dir": "/runs/" + (run.dir_name or run.run_id), "run_id": run.run_id, "thread_id": run.thread_id}
|
||||
|
||||
def start(self, run):
|
||||
if self.fail_starts:
|
||||
self.fail_starts -= 1
|
||||
raise RuntimeError("relay down")
|
||||
raise TransportError("relay down")
|
||||
h = self.handle(run)
|
||||
if "pid.txt" in self.dirs.get(h["run_dir"], ()):
|
||||
raise TransportError("HASPID")
|
||||
self.started.append(run)
|
||||
return {"run_dir": "/runs/" + run.run_id, "run_id": run.run_id, "thread_id": run.thread_id}
|
||||
self.dirs[h["run_dir"]] = {"launch.pid", "pid.txt"}
|
||||
if self.lost_starts:
|
||||
self.lost_starts -= 1
|
||||
raise TransportError("relay timeout after 150s")
|
||||
return h
|
||||
|
||||
def probe(self, handle):
|
||||
if self.fail_probes:
|
||||
self.fail_probes -= 1
|
||||
raise TransportError("relay down")
|
||||
return sorted(self.dirs.get(handle["run_dir"], ()))
|
||||
|
||||
def poll(self, handle):
|
||||
self.polls.append(handle["run_id"])
|
||||
if self.fail_polls:
|
||||
self.fail_polls -= 1
|
||||
raise TransportError("relay down")
|
||||
return "finished" if handle["run_id"] in self.done else "running"
|
||||
|
||||
def collect(self, handle):
|
||||
|
||||
Reference in New Issue
Block a user