Double-start guard: launching marker, probe-and-adopt, fresh-dir relaunch; skip unreachable terminals per tick
Before the launch call the planned run (run id, terminal, handle, bookkeeping) is saved as a launching marker. If the launch call fails or agentd dies before recording the run, the next tick probes the run directory (exit.txt / launch.pid / pid.txt): evidence -> adopt the run; nothing -> relaunch with the same run id in <run_id>-r2 (-r3 ...); probe unreachable -> keep the marker, which holds the terminal slot. Transports refuse to clean a run directory holding a pid file. A terminal whose relay fails is skipped for the rest of the tick (heartbeat.down) while other terminals carry on.
This commit is contained in:
@@ -203,3 +203,17 @@ class PosixRemoteTest(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
def test_probe_dir_name_and_pid_guard(self):
|
||||
ex = make_executor(self.cfg, "mac", "shell", self.t)
|
||||
run = Run("GW-5-1", self.cfg.terminals["mac"]["repo"], "", script="sleep 0.2", dir_name="GW-5-1-r2")
|
||||
self.assertEqual(ex.probe(ex.handle(run)), [])
|
||||
h = ex.start(run)
|
||||
self.assertEqual((os.path.basename(h["run_dir"]), h["task"]), ("GW-5-1-r2", "agentd-GW-5-1-r2"))
|
||||
self.assertTrue(wait(ex, h))
|
||||
self.assertEqual(ex.probe(h), ["exit.txt", "launch.pid", "pid.txt"])
|
||||
n = len(self.t.calls)
|
||||
with self.assertRaises(AssertionError): # LocalBash: marker LAUNCHED= missing (HASPID)
|
||||
ex.start(run)
|
||||
self.assertIn("HASPID", self.t.calls[n])
|
||||
self.assertTrue(os.path.exists(os.path.join(h["run_dir"], "exit.txt"))) # nothing wiped
|
||||
|
||||
Reference in New Issue
Block a user