Skip to content

Commit 9c799c5

Browse files
joshavantjoshavant
committed
fix main session startup recovery
1 parent dfb4491 commit 9c799c5

4 files changed

Lines changed: 607 additions & 33 deletions

File tree

‎src/agents/main-session-restart-recovery.test.ts‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import {
1616
import {
1717
markRestartAbortedMainSessions,
1818
markRestartAbortedMainSessionsFromLocks,
19+
markStartupOrphanedMainSessionsForRecovery,
20+
recoverStartupOrphanedMainSessions,
1921
recoverRestartAbortedMainSessions,
2022
} from "./main-session-restart-recovery.js";
2123
import type { SessionLockInspection } from "./session-write-lock.js";
@@ -637,6 +639,37 @@ describe("main-session-restart-recovery", () => {
637639
expect(store["agent:main:main"]?.pendingFinalDeliveryText).toBe("The final answer is 42.");
638640
});
639641

642+
it("resumes pending final delivery even when the transcript tail is assistant output", async () => {
643+
const sessionsDir = await makeSessionsDir();
644+
await writeStore(sessionsDir, {
645+
"agent:main:main": {
646+
sessionId: "main-session",
647+
updatedAt: Date.now() - 10_000,
648+
status: "running",
649+
abortedLastRun: true,
650+
pendingFinalDelivery: true,
651+
pendingFinalDeliveryText: "assistant final was already captured",
652+
pendingFinalDeliveryCreatedAt: Date.now() - 5_000,
653+
},
654+
});
655+
await writeTranscript(sessionsDir, "main-session", [
656+
{ role: "user", content: "finish" },
657+
{ role: "assistant", content: "assistant final was already captured" },
658+
]);
659+
660+
const result = await recoverRestartAbortedMainSessions({ stateDir: tmpDir });
661+
662+
expect(result).toEqual({ recovered: 1, failed: 0, skipped: 0 });
663+
expect(callGateway).toHaveBeenCalledOnce();
664+
expect(firstGatewayParams().message).toContain("assistant final was already captured");
665+
const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json"));
666+
expect(store["agent:main:main"]?.status).toBe("running");
667+
expect(store["agent:main:main"]?.pendingFinalDelivery).toBe(true);
668+
expect(store["agent:main:main"]?.pendingFinalDeliveryText).toBe(
669+
"assistant final was already captured",
670+
);
671+
});
672+
640673
it("does not scan ordinary running sessions without the restart-aborted marker", async () => {
641674
const sessionsDir = await makeSessionsDir();
642675
await writeStore(sessionsDir, {
@@ -657,6 +690,207 @@ describe("main-session-restart-recovery", () => {
657690
expect(callGateway).not.toHaveBeenCalled();
658691
});
659692

693+
it("skips restart-aborted sessions that a current process owns", async () => {
694+
const sessionsDir = await makeSessionsDir();
695+
await writeStore(sessionsDir, {
696+
"agent:main:active-key": {
697+
sessionId: "active-key-session",
698+
updatedAt: Date.now() - 10_000,
699+
status: "running",
700+
abortedLastRun: true,
701+
},
702+
"agent:main:active-id": {
703+
sessionId: "active-id-session",
704+
updatedAt: Date.now() - 10_000,
705+
status: "running",
706+
abortedLastRun: true,
707+
},
708+
"agent:main:recoverable": {
709+
sessionId: "recoverable-session",
710+
updatedAt: Date.now() - 10_000,
711+
status: "running",
712+
abortedLastRun: true,
713+
},
714+
});
715+
await writeTranscript(sessionsDir, "active-key-session", [
716+
{ role: "user", content: "new run owns this key" },
717+
{ role: "toolResult", content: "done" },
718+
]);
719+
await writeTranscript(sessionsDir, "active-id-session", [
720+
{ role: "user", content: "new run owns this id" },
721+
{ role: "toolResult", content: "done" },
722+
]);
723+
await writeTranscript(sessionsDir, "recoverable-session", [
724+
{ role: "user", content: "recover this one" },
725+
{ role: "toolResult", content: "done" },
726+
]);
727+
728+
const result = await recoverRestartAbortedMainSessions({
729+
stateDir: tmpDir,
730+
activeSessionKeys: ["agent:main:active-key"],
731+
activeSessionIds: ["active-key-session", "active-id-session"],
732+
});
733+
734+
expect(result).toEqual({ recovered: 1, failed: 0, skipped: 2 });
735+
expect(callGateway).toHaveBeenCalledOnce();
736+
const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json"));
737+
expect(store["agent:main:active-key"]?.abortedLastRun).toBe(true);
738+
expect(store["agent:main:active-id"]?.abortedLastRun).toBe(true);
739+
expect(store["agent:main:recoverable"]?.abortedLastRun).toBe(false);
740+
});
741+
742+
it("recovers duplicate-key restart-aborted rows when the active run owns a different session id", async () => {
743+
const sessionsDir = await makeSessionsDir();
744+
await writeStore(sessionsDir, {
745+
"agent:main:main": {
746+
sessionId: "stale-session",
747+
updatedAt: Date.now() - 10_000,
748+
status: "running",
749+
abortedLastRun: true,
750+
},
751+
});
752+
await writeTranscript(sessionsDir, "stale-session", [
753+
{ role: "user", content: "recover the stale duplicate" },
754+
{ role: "toolResult", content: "done" },
755+
]);
756+
757+
const result = await recoverRestartAbortedMainSessions({
758+
stateDir: tmpDir,
759+
activeSessionKeys: ["agent:main:main"],
760+
activeSessionIds: ["new-current-session"],
761+
});
762+
763+
expect(result).toEqual({ recovered: 1, failed: 0, skipped: 0 });
764+
expect(callGateway).toHaveBeenCalledOnce();
765+
const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json"));
766+
expect(store["agent:main:main"]?.abortedLastRun).toBe(false);
767+
});
768+
769+
it("marks startup-orphaned running main sessions before recovery", async () => {
770+
const sessionsDir = await makeSessionsDir();
771+
const cutoff = Date.now();
772+
await writeStore(sessionsDir, {
773+
"agent:main:main": {
774+
sessionId: "main-session",
775+
updatedAt: cutoff - 10_000,
776+
status: "running",
777+
},
778+
"agent:main:active-key": {
779+
sessionId: "active-key-session",
780+
updatedAt: cutoff - 10_000,
781+
status: "running",
782+
},
783+
"agent:main:active-id": {
784+
sessionId: "active-id-session",
785+
updatedAt: cutoff - 10_000,
786+
status: "running",
787+
},
788+
"agent:main:fresh": {
789+
sessionId: "fresh-session",
790+
updatedAt: cutoff + 1,
791+
status: "running",
792+
},
793+
"agent:main:subagent:child": {
794+
sessionId: "child-session",
795+
updatedAt: cutoff - 10_000,
796+
status: "running",
797+
spawnDepth: 1,
798+
},
799+
"agent:main:cron:nightly": {
800+
sessionId: "cron-session",
801+
updatedAt: cutoff - 10_000,
802+
status: "running",
803+
},
804+
"agent:main:completed": {
805+
sessionId: "completed-session",
806+
updatedAt: cutoff - 10_000,
807+
status: "done",
808+
},
809+
"agent:main:already-marked": {
810+
sessionId: "already-marked-session",
811+
updatedAt: cutoff - 10_000,
812+
status: "running",
813+
abortedLastRun: true,
814+
},
815+
});
816+
await writeTranscript(sessionsDir, "main-session", [
817+
{ role: "user", content: "run the tool" },
818+
{ role: "toolResult", content: "done" },
819+
]);
820+
await writeTranscript(sessionsDir, "already-marked-session", [
821+
{ role: "user", content: "already interrupted" },
822+
{ role: "toolResult", content: "done" },
823+
]);
824+
825+
const marked = await markStartupOrphanedMainSessionsForRecovery({
826+
stateDir: tmpDir,
827+
activeSessionKeys: ["agent:main:active-key"],
828+
activeSessionIds: ["active-key-session", "active-id-session"],
829+
updatedBeforeMs: cutoff,
830+
});
831+
832+
expect(marked).toEqual({ marked: 1, skipped: 2 });
833+
let store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json"));
834+
expect(store["agent:main:main"]?.abortedLastRun).toBe(true);
835+
expect(store["agent:main:active-key"]?.abortedLastRun).toBeUndefined();
836+
expect(store["agent:main:active-id"]?.abortedLastRun).toBeUndefined();
837+
expect(store["agent:main:fresh"]?.abortedLastRun).toBeUndefined();
838+
expect(store["agent:main:subagent:child"]?.abortedLastRun).toBeUndefined();
839+
expect(store["agent:main:cron:nightly"]?.abortedLastRun).toBeUndefined();
840+
expect(store["agent:main:completed"]?.abortedLastRun).toBeUndefined();
841+
expect(store["agent:main:already-marked"]?.abortedLastRun).toBe(true);
842+
843+
const recovered = await recoverRestartAbortedMainSessions({ stateDir: tmpDir });
844+
845+
expect(recovered).toEqual({ recovered: 2, failed: 0, skipped: 0 });
846+
expect(callGateway).toHaveBeenCalledTimes(2);
847+
store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json"));
848+
expect(store["agent:main:main"]?.abortedLastRun).toBe(false);
849+
expect(store["agent:main:already-marked"]?.abortedLastRun).toBe(false);
850+
});
851+
852+
it("recovers only the configured store for duplicate startup-orphaned session keys", async () => {
853+
const cutoff = Date.now();
854+
const defaultSessionsDir = await makeSessionsDir();
855+
await writeStore(defaultSessionsDir, {
856+
"agent:main:main": {
857+
sessionId: "default-main-session",
858+
updatedAt: cutoff - 10_000,
859+
status: "running",
860+
},
861+
});
862+
await writeTranscript(defaultSessionsDir, "default-main-session", [
863+
{ role: "user", content: "continue default" },
864+
{ role: "toolResult", content: "default result" },
865+
]);
866+
867+
const customStorePath = path.join(tmpDir, "custom-startup-duplicate", "sessions.json");
868+
await writeSessionStoreForTestAsync(customStorePath, {
869+
"agent:main:main": {
870+
sessionId: "custom-main-session",
871+
updatedAt: cutoff - 10_000,
872+
status: "running",
873+
},
874+
});
875+
await writeTranscript(path.dirname(customStorePath), "custom-main-session", [
876+
{ role: "user", content: "continue custom" },
877+
{ role: "toolResult", content: "custom result" },
878+
]);
879+
880+
const result = await recoverStartupOrphanedMainSessions({
881+
cfg: { session: { store: customStorePath } },
882+
stateDir: tmpDir,
883+
updatedBeforeMs: cutoff,
884+
});
885+
886+
expect(result).toEqual({ marked: 2, recovered: 1, failed: 0, skipped: 1 });
887+
expect(callGateway).toHaveBeenCalledOnce();
888+
const defaultStore = readSessionStoreForTest(path.join(defaultSessionsDir, "sessions.json"));
889+
const customStore = readSessionStoreForTest(customStorePath);
890+
expect(defaultStore["agent:main:main"]?.abortedLastRun).toBe(true);
891+
expect(customStore["agent:main:main"]?.abortedLastRun).toBe(false);
892+
});
893+
660894
it("fails marked sessions whose transcript tail cannot be resumed", async () => {
661895
const sessionsDir = await makeSessionsDir();
662896
await writeStore(sessionsDir, {

0 commit comments

Comments
 (0)