diff --git a/README.md b/README.md index f2dba8b..5d2fbe5 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,23 @@ required capability (a named `node` target passes through), the node runs the agent in its mapped checkout, and the orchestrator detects exits by reconciling its tracked agents against the engine roster. +Relay dispatch is lifecycle-owned, not fire-and-forget. A one-shot `factory +dispatch --backend relay` keeps a small publisher runtime alive until the +remote branch has produced a PR, terminal issue writeback is acknowledged, and +remote agents are released. The lifecycle (including a per-run branch, +placement results, PR receipt, and a fenced owner lease) is persisted beside +the configured loop registry so `factory start` or a replacement dispatch +process on the same control-plane host can take over after a crash. Execution +nodes never need access to that state file, and remote PIDs are never signalled +as local processes. + +The supported topology is one Factory control-plane host per workspace, with +any number of relay execution nodes. Multiple Factory processes on that host +are fenced through the shared `FileStateStore` lock/lease. Active/active Factory +control planes on different hosts are intentionally unsupported: the current +Relayfile and Relay messaging APIs do not expose a shared compare-and-set lease, +so separate local state files cannot provide a truthful cross-host fence. + Tokens involved — set only the first one on the orchestrator host: | Token | Prefix | Who holds it | diff --git a/src/cli/fleet.test.ts b/src/cli/fleet.test.ts index ac3775a..97c79f5 100644 --- a/src/cli/fleet.test.ts +++ b/src/cli/fleet.test.ts @@ -7,6 +7,7 @@ import type { CloseProbePrInput, Factory, FactoryPorts, createFactory } from '.. import { stateResolutionFromIds } from '../index' import { FileStateStore } from '../state/file-state-store' import { FakeFleetClient, FakeMountClient } from '../testing' +import type { GithubConnectionWrite, SpawnInput, SpawnResult } from '../ports' import { formatLogArgs, installFactoryStopSignalHandlers, parseFleetCommand, parseGlobalOptions, resolveBrokerConnectionPath, runFleetCli } from './fleet' const issuePath = '/linear/issues/AR-77__uuid-77.json' @@ -48,6 +49,29 @@ const issueFile = { }, } +class CompletingRemoteFleetClient extends FakeFleetClient { + override readonly placementLocality = 'remote' as const + readonly lifecycleOrder: string[] = [] + + override async spawn(input: SpawnInput): Promise { + const result = await super.spawn(input) + if (input.name.includes('-impl-')) { + setTimeout(() => this.emitAgentExit(input.name, 'exited'), 0) + } + return { ...result, node: 'sf-mini', locality: 'remote' } + } + + override async release(name: string, reason?: string): Promise { + this.lifecycleOrder.push(`release:${name}`) + await super.release(name, reason) + } + + override async dispose(): Promise { + this.lifecycleOrder.push('dispose') + await super.dispose() + } +} + const githubIssueFile = (repo: string, number = 48) => ({ provider: 'github', objectType: 'issue', @@ -547,6 +571,69 @@ describe('fleet CLI runtime', () => { } }) + it('keeps relay dispatch ownership until the remote PR is published and the issue is parked', async () => { + const root = await mkdtemp(join(tmpdir(), 'fleet-cli-relay-owner-')) + try { + const configPath = await writeConfig(root, { + loop: { + heartbeatPath: join(root, 'heartbeat.json'), + registryPath: join(root, 'registry.json'), + heartbeatStaleMs: 10_000, + }, + }) + const publishes: Parameters[0][] = [] + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => { + publishes.push(input) + return { + repo: input.repo, + number: 77, + url: 'https://github.com/AgentWorkforce/pear/pull/77', + headRef: input.headRef ?? 'unexpected-local-head', + } + }, + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath]: issueFile, + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const fleet = new CompletingRemoteFleetClient() + const output = buffer() + const errors = buffer() + + const code = await runFleetCli([ + 'dispatch', + 'AR-77', + '--backend', + 'relay', + '--config', + configPath, + ], { + fleet, + mount, + stdout: output, + stderr: errors, + probePrGhRunner: async () => ({ stdout: '[]' }), + }) + + expect(code).toBe(0) + expect(JSON.parse(output.text())).toMatchObject({ issue: { key: 'AR-77' }, dryRun: false }) + expect(publishes).toEqual([expect.objectContaining({ + repo: 'AgentWorkforce/pear', + headRef: expect.stringMatching(/^factory\/ar-77-agentworkforce-pear-[0-9a-f]{8}$/u), + })]) + expect(fleet.releases.map((release) => release.reason)).toEqual(['issue-human-review', 'issue-human-review']) + const firstDispose = fleet.lifecycleOrder.indexOf('dispose') + expect(firstDispose).toBeGreaterThanOrEqual(0) + expect(fleet.lifecycleOrder.slice(0, firstDispose).every((event) => event.startsWith('release:'))).toBe(true) + expect(fleet.lifecycleOrder.slice(firstDispose).every((event) => event === 'dispose')).toBe(true) + expect(errors.text()).not.toContain('[factory] error') + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + it('does not infer or preflight clone paths for a maintenance command', async () => { const root = await mkdtemp(join(tmpdir(), 'fleet-cli-maintenance-clone-')) try { diff --git a/src/cli/fleet.ts b/src/cli/fleet.ts index 600e2e0..d18eb48 100644 --- a/src/cli/fleet.ts +++ b/src/cli/fleet.ts @@ -444,6 +444,21 @@ async function runFactoryCommand( return 0 } + if (command.kind === 'factory-dispatch' && globals.backend === 'relay' && !globals.dryRun) { + // A relay dispatch is not placement-only. This process becomes the durable + // lifecycle publisher (or attaches to the current owner's durable row), + // and stays through takeover/publication/writeback/release to terminal. + await factory.start({ mode: 'dispatch-owner' }) + try { + const result = await factory.dispatch(decision, { dryRun: false }) + writeJson(out, result) + await factory.waitForDispatchTerminal(result.issue) + return 0 + } finally { + await factory.stop() + } + } + writeJson(out, await factory.dispatch(decision, { dryRun: globals.dryRun })) return 0 } diff --git a/src/dispatch/templates.ts b/src/dispatch/templates.ts index 0ebe0b2..0045a8f 100644 --- a/src/dispatch/templates.ts +++ b/src/dispatch/templates.ts @@ -54,6 +54,8 @@ export interface RenderAgentTaskInput { } /** Pre-rendered writeback instructions for connected integrations. */ integrationInstructions?: string + /** Exact branch Factory will publish after the implementer pushes it. */ + branchName?: string /** * Absolute path to the .integrations mount root. The agent runs in its repo * clonePath, not the daemon cwd where .integrations lives, so every @@ -84,7 +86,9 @@ export function renderAgentTask(input: RenderAgentTaskInput): string { const common = [ ...header, '', - 'Create a branch for this issue before editing.', + input.branchName + ? `Create a branch for this issue before editing. Create or reset the exact branch \`${input.branchName}\` from the repository default branch, then commit and push only this branch.` + : 'Create a branch for this issue before editing.', 'Commit the implementation and tests.', 'Push the branch to origin.', 'When implementation is complete, finish your session normally; Factory will open the PR targeting the repository default branch through the connected GitHub workspace.', diff --git a/src/fleet/internal-fleet-client.ts b/src/fleet/internal-fleet-client.ts index bdb7593..718d68c 100644 --- a/src/fleet/internal-fleet-client.ts +++ b/src/fleet/internal-fleet-client.ts @@ -95,6 +95,7 @@ const RELEASE_RETRY_MAX_ATTEMPTS = 3 const RELEASE_RETRY_BACKOFF_MS = 250 export class InternalFleetClient implements FleetClient { + readonly placementLocality = 'local' as const readonly #client: HarnessDriverClientLike #ownsBroker: boolean readonly #ownedBrokerAgentExitTimeoutMs: number @@ -178,7 +179,14 @@ export class InternalFleetClient implements FleetClient { return spawnResultFrom(handle) } - async resume(input: { name?: string; sessionRef: string; node?: 'self' | string; capability?: Capability }): Promise { + async resume(input: { + name?: string + sessionRef: string + node?: 'self' | string + capability?: Capability + repo?: string + clonePath?: string + }): Promise { assertSelfNode(input.node) this.#ensureEventSubscription() this.#readyAgentNames.delete(input.name ?? input.sessionRef) diff --git a/src/fleet/relay-fleet-client.test.ts b/src/fleet/relay-fleet-client.test.ts index b0aac42..b73125d 100644 --- a/src/fleet/relay-fleet-client.test.ts +++ b/src/fleet/relay-fleet-client.test.ts @@ -163,7 +163,7 @@ describe('RelayFleetClient', () => { cwd: '/checkout', invocationId: 'factory-inv-1', channel: 'wf-factory', - })).resolves.toEqual({ name: 'ar-1-impl', sessionRef: 'session-1', pid: 123 }) + })).resolves.toEqual({ name: 'ar-1-impl', sessionRef: 'session-1', pid: 123, node: 'mac-mini', locality: 'remote' }) expect(messaging.placements).toHaveLength(1) const placement = messaging.placements[0]! diff --git a/src/fleet/relay-fleet-client.ts b/src/fleet/relay-fleet-client.ts index 0090f2a..1777653 100644 --- a/src/fleet/relay-fleet-client.ts +++ b/src/fleet/relay-fleet-client.ts @@ -77,6 +77,7 @@ const DEFAULT_NODE_OFFLINE_GRACE_MS = 90_000 const DEFAULT_REGISTRATION_GRACE_MS = 60_000 export class RelayFleetClient implements FleetClient { + readonly placementLocality = 'remote' as const readonly #options: RelayFleetClientOptions readonly #agentName: string readonly #spawnAckTimeoutMs: number @@ -164,17 +165,26 @@ export class RelayFleetClient implements FleetClient { log: this.#log, }) const invocation = await this.#awaitInvocation(ack.actionName || 'spawn', ack) - const result = spawnResultFromInvocation(input.name, input.sessionRef, invocation) + const result = spawnResultFromInvocation(input.name, input.sessionRef, invocation, ack) this.#track(result.name, ack) return result } - async resume(input: { name?: string; sessionRef: string; node?: 'self' | string; capability?: Capability }): Promise { + async resume(input: { + name?: string + sessionRef: string + node?: 'self' | string + capability?: Capability + repo?: string + clonePath?: string + }): Promise { const name = input.name ?? input.sessionRef return await this.spawn({ name, capability: input.capability ?? 'spawn:codex', node: input.node, + repo: input.repo, + clonePath: input.clonePath, sessionRef: input.sessionRef, }) } @@ -204,7 +214,11 @@ export class RelayFleetClient implements FleetClient { messaging.nodes.list(), ]) return { - agents: agents.map((agent) => ({ name: agent.name })), + agents: agents.map((agent) => { + const record = asRecord(agent) + const node = readString(record, 'node', 'node_id', 'nodeId') + return { name: agent.name, ...(node ? { node } : {}) } + }), nodes: nodes.map((node) => ({ name: node.name, capabilities: normalizeCapabilities(node.capabilities), @@ -515,7 +529,12 @@ function spawnActionInput(input: SpawnInput): Record { }) } -function spawnResultFromInvocation(fallbackName: string, fallbackSessionRef: string | undefined, invocation: RelayActionInvocation): SpawnResult { +function spawnResultFromInvocation( + fallbackName: string, + fallbackSessionRef: string | undefined, + invocation: RelayActionInvocation, + ack: RelayActionInvocationAck & { placement?: { node?: string } }, +): SpawnResult { const output = asRecord(invocation.output) const agent = asRecord(output?.agent) const name = readString(output, 'name', 'agent_name', 'agentName') ?? readString(agent, 'name') ?? fallbackName @@ -524,11 +543,14 @@ function spawnResultFromInvocation(fallbackName: string, fallbackSessionRef: str ?? fallbackSessionRef const pid = readNumber(output, 'pid') ?? readNumber(agent, 'pid') const pids = readNumberArray(output, 'pids') ?? readNumberArray(agent, 'pids') + const node = readString(output, 'node') ?? readString(agent, 'node') ?? ack.placement?.node ?? ack.dispatchedNodeId ?? undefined return { name, ...(sessionRef ? { sessionRef } : {}), ...(pid !== undefined ? { pid } : {}), ...(pids ? { pids } : {}), + ...(node ? { node } : {}), + locality: 'remote', } } diff --git a/src/mount/relayfile-github-connection-write.test.ts b/src/mount/relayfile-github-connection-write.test.ts index 1c73bdb..bd93dbc 100644 --- a/src/mount/relayfile-github-connection-write.test.ts +++ b/src/mount/relayfile-github-connection-write.test.ts @@ -16,6 +16,50 @@ const gitRunnerForBranch = (branch: string): GitCommandRunner => vi.fn(async (ar }) describe('RelayfileGithubConnectionWrite', () => { + it('publishes an already-pushed remote branch without reading an orchestrator-local clone', async () => { + const pullRequestPath = '/github/repos/AgentWorkforce/factory/pull-requests/factory-factory-ar-85-agentworkforce-factory-pushed.json' + class ReceiptMount extends FakeMountClient { + override async writeFile(path: string, content: unknown, opts?: { guarded?: boolean }): Promise { + await super.writeFile(path, content, opts) + if (path === pullRequestPath) { + this.files.set(path, { content: { created: 85, url: 'https://github.com/AgentWorkforce/factory/pull/85' } }) + } + } + } + const mount = new ReceiptMount() + const git = vi.fn(async () => { throw new Error('remote publication must not inspect local git') }) + const write = new RelayfileGithubConnectionWrite({ mount, gitRunner: git }) + + const input = { + repo: 'AgentWorkforce/factory', + headRef: 'factory/ar-85-agentworkforce-factory', + baseRef: 'main', + title: 'Issue 85', + body: 'Fixes #85', + } + await expect(write.publishPullRequest(input)).resolves.toEqual({ + repo: 'AgentWorkforce/factory', + number: 85, + url: 'https://github.com/AgentWorkforce/factory/pull/85', + headRef: 'factory/ar-85-agentworkforce-factory', + headSha: undefined, + }) + expect(git).not.toHaveBeenCalled() + expect(mount.writes).toEqual([{ + path: pullRequestPath, + content: { + title: 'Issue 85', + head: 'factory/ar-85-agentworkforce-factory', + base: 'main', + body: 'Fixes #85', + author: 'app', + }, + }]) + + await expect(write.publishPullRequest(input)).resolves.toMatchObject({ number: 85 }) + expect(mount.writes[1]?.path).toBe(pullRequestPath) + }) + it('pushes the current ref before creating a pull request through Relayfile', async () => { const draft = 'factory-fix-issue-52-1234567890ab' const pullRequestPath = `/github/repos/AgentWorkforce/factory/pull-requests/${draft}.json` diff --git a/src/mount/relayfile-github-connection-write.ts b/src/mount/relayfile-github-connection-write.ts index 0713682..857dc45 100644 --- a/src/mount/relayfile-github-connection-write.ts +++ b/src/mount/relayfile-github-connection-write.ts @@ -42,8 +42,15 @@ export class RelayfileGithubConnectionWrite implements GithubConnectionWrite { async publishPullRequest(input: GithubPublishPullRequestInput): Promise { const { owner, repo } = githubRepoParts(input.repo) - const headRef = await this.#gitValue(['-C', input.clonePath, 'symbolic-ref', '--short', 'HEAD'], 'current branch') - const headSha = await this.#gitValue(['-C', input.clonePath, 'rev-parse', 'HEAD'], 'HEAD commit') + const headRef = input.headRef ?? (input.clonePath + ? await this.#gitValue(['-C', input.clonePath, 'symbolic-ref', '--short', 'HEAD'], 'current branch') + : undefined) + const headSha = input.headSha ?? (input.clonePath && !input.headRef + ? await this.#gitValue(['-C', input.clonePath, 'rev-parse', 'HEAD'], 'HEAD commit') + : undefined) + if (!headRef) { + throw new Error('GitHub PR publication requires headRef or clonePath') + } if (headRef === input.baseRef) { throw new Error(`Refusing to publish GitHub PR with head equal to base branch: ${headRef}`) } @@ -52,10 +59,14 @@ export class RelayfileGithubConnectionWrite implements GithubConnectionWrite { const fullHeadRef = `refs/heads/${headRef}` const refPath = `${repoRoot}/refs/${encodeURIComponent(fullHeadRef)}.json` - await this.#writeAndConfirm(refPath, { - ref: fullHeadRef, - sha: headSha, - }) + // A remote implementer already pushed its branch. Only synthesize/update + // the ref for the legacy local-clone path where the publisher owns HEAD. + if (headSha) { + await this.#writeAndConfirm(refPath, { + ref: fullHeadRef, + sha: headSha, + }) + } const pullRequestPath = `${repoRoot}/pull-requests/${draftName}.json` await this.#writeAndConfirm(pullRequestPath, { @@ -139,9 +150,10 @@ const githubRepoParts = (value: string): { owner: string; repo: string } => { return { owner: match[1], repo: match[2] } } -const githubDraftName = (headRef: string, headSha: string): string => { +const githubDraftName = (headRef: string, headSha?: string): string => { const branch = headRef.toLowerCase().replace(/[^a-z0-9]+/gu, '-').replace(/^-|-$/gu, '') || 'branch' - return `factory-${branch.slice(0, 80)}-${headSha.slice(0, 12)}` + const identity = headSha?.slice(0, 12) ?? 'pushed' + return `factory-${branch.slice(0, 80)}-${identity}` } const record = (value: unknown): Record => diff --git a/src/orchestrator/batch-tracker.ts b/src/orchestrator/batch-tracker.ts index c945807..21a985a 100644 --- a/src/orchestrator/batch-tracker.ts +++ b/src/orchestrator/batch-tracker.ts @@ -160,6 +160,11 @@ export class BatchTracker { }) } + recordPlanned(record: InFlightIssue, spec: AgentSpec): void { + if (record.agents.has(spec.name)) return + record.agents.set(spec.name, { spec: { ...spec }, sessionRef: spec.sessionRef }) + } + recordDryRun(record: InFlightIssue, spec: AgentSpec, invocationId: string): void { record.invocationIds.add(invocationId) this.#invocationIds.add(invocationId) @@ -169,6 +174,28 @@ export class BatchTracker { sessionRef: spec.sessionRef, }) } + + /** Restore a crash-safe lifecycle before the fleet emits reconciled exits. */ + restore(record: InFlightIssue): InFlightIssue { + const key = issueKey(record.issue) + const existing = this.#inFlight.get(key) + if (existing) return existing + const restored: InFlightIssue = { + issue: { ...record.issue }, + decision: structuredClone(record.decision), + dryRun: record.dryRun, + agents: new Map([...record.agents].map(([name, tracked]) => [name, { + spec: structuredClone(tracked.spec), + result: tracked.result ? { ...tracked.result } : undefined, + sessionRef: tracked.sessionRef, + }])), + invocationIds: new Set(record.invocationIds), + result: record.result ? structuredClone(record.result) : undefined, + } + this.#inFlight.set(key, restored) + for (const invocationId of restored.invocationIds) this.#invocationIds.add(invocationId) + return restored + } } export const issueKey = (issue: IssueRef): string => `${issue.key}:${issue.uuid}:${issue.path}` diff --git a/src/orchestrator/factory.test.ts b/src/orchestrator/factory.test.ts index 1570f3f..bc0f76c 100644 --- a/src/orchestrator/factory.test.ts +++ b/src/orchestrator/factory.test.ts @@ -449,6 +449,92 @@ class ManualClock { } } +class RemoteLifecycleFleetClient extends FakeFleetClient { + override readonly placementLocality = 'remote' as const + exitImplementerOnReconcile = false + + override async spawn(input: SpawnInput): Promise { + const result = await super.spawn(input) + return { ...result, node: 'sf-mini', locality: 'remote' } + } + + override async roster() { + const roster = await super.roster() + return { ...roster, agents: roster.agents.map((agent) => ({ ...agent, node: 'sf-mini' })) } + } + + override async reconcileTrackedAgents(): Promise { + await super.reconcileTrackedAgents() + if (!this.exitImplementerOnReconcile) return + const implementer = this.hydrated.find((agent) => agent.name.includes('-impl-')) + if (implementer) this.emitAgentExit(implementer.name, 'exited') + } +} + +class TransientRemoteReleaseFleetClient extends RemoteLifecycleFleetClient { + failReleaseFor?: string + releaseFailures = 0 + + override async release(name: string, reason?: string): Promise { + if (name === this.failReleaseFor && this.releaseFailures === 0) { + this.releaseFailures += 1 + throw new Error(`transient release failure for ${name}`) + } + await super.release(name, reason) + } +} + +class BlockingClarificationReleaseFleetClient extends RemoteLifecycleFleetClient { + readonly clarificationReleaseStarted: Promise + #signalClarificationReleaseStarted!: () => void + #allowClarificationRelease!: () => void + readonly #clarificationReleaseAllowed: Promise + #blocked = false + + constructor() { + super() + this.clarificationReleaseStarted = new Promise((resolve) => { + this.#signalClarificationReleaseStarted = resolve + }) + this.#clarificationReleaseAllowed = new Promise((resolve) => { + this.#allowClarificationRelease = resolve + }) + } + + allowClarificationRelease(): void { + this.#allowClarificationRelease() + } + + override async release(name: string, reason?: string): Promise { + if (!this.#blocked && reason === 'waiting-for-human') { + this.#blocked = true + this.#signalClarificationReleaseStarted() + await this.#clarificationReleaseAllowed + } + await super.release(name, reason) + } +} + +class FailingClarificationReleaseFleetClient extends RemoteLifecycleFleetClient { + override async release(name: string, reason?: string): Promise { + if (reason === 'waiting-for-human') { + throw new Error(`relay release unavailable for ${name}`) + } + await super.release(name, reason) + } +} + +class ParkingTakeoverReconcileFleetClient extends BlockingClarificationReleaseFleetClient { + override async reconcileTrackedAgents(): Promise { + await super.reconcileTrackedAgents() + // Relay reconciliation reports hydrated agents absent from the broker as + // exits immediately. Emit independently of FakeFleetClient's hydration so + // this regression also proves parking agents were excluded from hydration. + this.emitAgentExit('ar-991-impl-pear', 'exited') + this.emitAgentExit('ar-991-review', 'exited') + } +} + class TimestampFailingFleetClient extends FakeFleetClient { readonly attemptTimes: number[] = [] @@ -1410,6 +1496,8 @@ describe('FactoryLoop', () => { sessionRef: 'session-review-hoopsheet-26', node: 'self', capability: 'spawn:claude', + repo: 'AgentWorkforce/hoopsheet', + clonePath: '/work/hoopsheet', })) fleet.emitAgentExit('ar-26-impl-pear', 'issue-done') @@ -2166,15 +2254,14 @@ describe('FactoryLoop', () => { it('re-dispatches a terminal issue after a canonical Done to Ready re-open', async () => { const mount = new FakeMountClient({ [issuePath(364)]: issueFile(364) }) - const fleet = new FakeFleetClient() + const fleet = new RemoteLifecycleFleetClient() const factory = createFactory(config(), { mount, fleet, triage: new StaticTriage() }) const first = await factory.runOnce() expect(first.dispatched.map((result) => result.issue.key)).toEqual(['AR-364']) fleet.emitAgentExit('ar-364-impl-pear', 'issue-done') - await flush() - expect(factory.status().counters.done).toBe(1) + await vi.waitFor(() => expect(factory.status().counters.done).toBe(1)) await mount.writeFile(issuePath(364), issuePayload(364, ready)) const reopened = await factory.runOnce() @@ -2187,6 +2274,13 @@ describe('FactoryLoop', () => { 'ar-364-impl-pear', 'ar-364-review', ]) + const branchInstructions = fleet.spawns + .filter((spawn) => spawn.name === 'ar-364-impl-pear') + .map((spawn) => /Factory publication branch: ([^\s]+)/u.exec(spawn.task ?? '')?.[1]) + expect(branchInstructions).toHaveLength(2) + expect(branchInstructions[0]).not.toBe(branchInstructions[1]) + expect(fleet.spawns[0]?.invocationId).not.toBe(fleet.spawns[2]?.invocationId) + expect(fleet.spawns[1]?.invocationId).not.toBe(fleet.spawns[3]?.invocationId) expect(factory.status().counters.dispatchTerminalReopened).toBe(1) }) @@ -2202,8 +2296,7 @@ describe('FactoryLoop', () => { expect(first.dispatched.map((result) => result.issue.key)).toEqual(['AR-366']) fleet.emitAgentExit('ar-366-impl-pear', 'issue-done') - await flush() - expect(factory.status().counters.humanReview).toBe(1) + await vi.waitFor(() => expect(factory.status().counters.humanReview).toBe(1)) await mount.writeFile(issuePath(366), issuePayload(366, ready)) const reopened = await factory.runOnce() @@ -2712,100 +2805,1009 @@ describe('FactoryLoop', () => { expect(fleet.spawns.map((spawn) => spawn.name)).toEqual(['ar-133-impl-pear', 'ar-133-review']) }) - it('skips missing canonical-shaped issue paths without aborting discovery', async () => { - const missingCanonicalPath = '/linear/issues/ZZ-404__00000000-0000-4000-8000-000000000404.json' - const debugLogs: unknown[][] = [] - const mount = new ListingReadTrackingMount({ - [capturedReadyCanaryPath]: realIssueFile(133, ready, { id: 'dac27fce-e8de-4910-bbf6-98ad436df3dd' }), - }, [missingCanonicalPath]) - const fleet = new FakeFleetClient() + it('skips missing canonical-shaped issue paths without aborting discovery', async () => { + const missingCanonicalPath = '/linear/issues/ZZ-404__00000000-0000-4000-8000-000000000404.json' + const debugLogs: unknown[][] = [] + const mount = new ListingReadTrackingMount({ + [capturedReadyCanaryPath]: realIssueFile(133, ready, { id: 'dac27fce-e8de-4910-bbf6-98ad436df3dd' }), + }, [missingCanonicalPath]) + const fleet = new FakeFleetClient() + const factory = createFactory(config(), { + mount, + fleet, + triage: new StaticTriage(), + logger: { + debug: (...args: unknown[]) => debugLogs.push(args), + warn: () => undefined, + error: () => undefined, + }, + }) + + const report = await factory.runOnce() + + expect(report.pulled.map((issue) => issue.path)).toEqual([capturedReadyCanaryPath]) + expect(mount.readPaths).toContain(missingCanonicalPath) + expect(factory.status().counters.phantomSkipped).toBe(1) + expect(debugLogs).toContainEqual([ + '[factory] skipped missing issue file discovered from issue tree', + { path: missingCanonicalPath }, + ]) + }) + + it('runLoop stops at the configured iteration cap, preserves the batch cap, and advances heartbeat liveness', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-loop-heartbeat-')) + const heartbeatPath = join(root, 'heartbeat.json') + const registryPath = join(root, 'registry.json') + try { + const mount = new FakeMountClient(Object.fromEntries( + [51, 52, 53, 54, 55, 56].map((n) => [issuePath(n), issueFile(n)]), + )) + const fleet = new FakeFleetClient() + const factory = createFactory(config({ + batchSize: 5, + loop: { maxIterations: 3, heartbeatPath, registryPath, heartbeatStaleMs: 10_000 }, + }), { mount, fleet, triage: new StaticTriage() }) + + const reports = await factory.runLoop({ dryRun: true }) + + expect(reports).toHaveLength(3) + expect(factory.status().counters.loopIdle).toBe(1) + expect(factory.status().inFlight).toHaveLength(5) + expect(factory.status().queued).toHaveLength(1) + const heartbeat = await readFactoryLoopHeartbeat(heartbeatPath) + expect(heartbeat).toMatchObject({ status: 'idle', iteration: 3, maxIterations: 3, pid: process.pid }) + expect(checkFactoryLoopLiveness(heartbeat, { nowMs: heartbeat!.updatedAtMs + 500, staleMs: 10_000 })).toMatchObject({ + ok: true, + stale: false, + }) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('start re-adopts remote in-flight registry agents into the fleet and reconciles once', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-adopt-inflight-')) + const registryPath = join(root, 'registry.json') + try { + await writeFile(registryPath, JSON.stringify({ + pid: 12345, + updatedAt: new Date(0).toISOString(), + updatedAtMs: 0, + agents: [ + { name: 'ar-1-impl', pids: [], invocationId: 'inv-1', node: 'mac-mini' }, + { name: 'ar-1-review', pids: [], node: 'mac-mini' }, + // Local-only agents (pids, no placement facts) are not fleet-tracked. + { name: 'ar-2-impl', pids: [4242] }, + ], + })) + const fleet = new FakeFleetClient() + const factory = createFactory(config({ + loop: { maxIterations: 1, heartbeatPath: join(root, 'heartbeat.json'), registryPath, heartbeatStaleMs: 1_000 }, + }), { + mount: new FakeMountClient(), + fleet, + triage: new StaticTriage(), + }) + + await factory.start({ mode: 'live', liveSubscription: { transport: 'subscribe' } }) + + expect(fleet.hydrated).toEqual([ + { name: 'ar-1-impl', invocationId: 'inv-1', node: 'mac-mini' }, + { name: 'ar-1-review', invocationId: undefined, node: 'mac-mini' }, + ]) + expect(fleet.reconciles).toBe(1) + + await factory.stop() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('rehydrates durable remote lifecycle before reconciliation and publishes one PR after owner crash', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-remote-lifecycle-')) + const watchStatePath = join(root, 'state.json') + const clock = new ManualClock() + const publishInputs: GithubPublishPullRequestInput[] = [] + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => { + publishInputs.push(input) + return { + repo: input.repo, + number: 85, + url: 'https://github.com/AgentWorkforce/pear/pull/85', + headRef: input.headRef ?? 'unexpected-local-head', + } + }, + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(85)]: issueFile(85), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + try { + const firstFleet = new RemoteLifecycleFleetClient() + const first = createFactory(config(), { + mount, + fleet: firstFleet, + stateStore: new FileStateStore({ batchSize: 2, watchStatePath }), + triage: new StaticTriage(), + clock, + probePrResolver: async () => undefined, + }) + const decision = await first.triageIssue(parseLinearIssue(issuePath(85), issueFile(85))) + const originalResult = await first.dispatch(decision) + + const key = issueKey(decision.issue) + const crashedState = new FileStateStore({ batchSize: 2, watchStatePath }) + const beforeCrash = await crashedState.getDispatchLifecycle('factory-test', key) + expect(beforeCrash?.phase).toBe('running') + expect(beforeCrash?.agents.find((agent) => agent.name === 'ar-85-impl-pear')?.tracked.result?.node).toBe('sf-mini') + + // The replacement starts while the crashed owner's nominal lease is + // still live. It must attach without hydrating/spawning/publishing, then + // autonomously take over after expiry with no second start or event. + const restartedFleet = new RemoteLifecycleFleetClient() + restartedFleet.exitImplementerOnReconcile = true + const restarted = createFactory(config(), { + mount, + fleet: restartedFleet, + stateStore: new FileStateStore({ batchSize: 2, watchStatePath }), + triage: new StaticTriage(), + clock, + probePrResolver: async () => undefined, + }) + await restarted.start({ mode: 'dispatch-owner' }) + await expect(restarted.dispatch(decision)).resolves.toEqual(originalResult) + const terminal = restarted.waitForDispatchTerminal(decision.issue) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(restartedFleet.hydrated).toEqual([]) + expect(restartedFleet.spawns).toEqual([]) + expect(publishInputs).toEqual([]) + + clock.advance(5 * 60_000 + 1) + await terminal + await vi.waitFor(async () => { + expect(await crashedState.getDispatchLifecycle('factory-test', key)).toMatchObject({ phase: 'complete' }) + }) + + expect(restartedFleet.hydrated.map((agent) => agent.name).sort()).toEqual(['ar-85-impl-pear', 'ar-85-review']) + expect(publishInputs).toEqual([expect.objectContaining({ + repo: 'AgentWorkforce/pear', + headRef: expect.stringMatching(/^factory\/ar-85-agentworkforce-pear-[0-9a-f]{8}$/u), + })]) + expect(publishInputs[0]).not.toHaveProperty('clonePath') + expect(restartedFleet.releases.map((release) => release.name).sort()).toEqual(['ar-85-impl-pear', 'ar-85-review']) + await first.stop() + await restarted.stop() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it.each([ + ['healthy owner', 1091, false], + ['waiting owner crash', 1092, true], + ] as const)('keeps an attached terminal waiter live across clarification with a %s', async (_scenario, number, crashAfterPark) => { + const root = await mkdtemp(join(tmpdir(), `factory-attached-clarification-${number}-`)) + const watchStatePath = join(root, 'state.json') + const clock = new ManualClock() + const state = () => new FileStateStore({ batchSize: 1, watchStatePath }) + const mount = new ConfirmRecordingSlackMountClient({ [issuePath(number)]: issueFile(number) }) + const factoryConfig = config({ batchSize: 1, slack: slackConfig() }) + const firstFleet = new RemoteLifecycleFleetClient() + firstFleet.setSessionRef(`ar-${number}-impl-pear`, `session-ar-${number}-impl-pear`) + firstFleet.setSessionRef(`ar-${number}-review`, `session-ar-${number}-review`) + const first = createFactory(factoryConfig, { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + clock, + }) + const attachedFleet = new RemoteLifecycleFleetClient() + const attached = createFactory(factoryConfig, { + mount, + fleet: attachedFleet, + stateStore: state(), + triage: new StaticTriage(), + clock, + }) + try { + const decision = await first.triageIssue(parseLinearIssue(issuePath(number), issueFile(number))) + const result = await first.dispatch(decision) + await attached.start({ mode: 'dispatch-owner' }) + await expect(attached.dispatch(decision)).resolves.toEqual(result) + const terminal = attached.waitForDispatchTerminal(decision.issue) + + mount.files.set(issuePath(number), { content: issueFile(number, implementing) }) + firstFleet.emitAgentMessage({ + from: `ar-${number}-impl-pear`, + target: 'factory', + body: `[factory-needs-input] Keep the attached owner live for ${number}.`, + eventId: `agent-question-${number}`, + }) + await vi.waitFor(() => expect(first.status().counters.agentQuestionTeamsReleased).toBe(1)) + await vi.waitFor(() => expect(first.status().counters.clarificationQuestionsDelivered).toBe(1)) + await vi.waitFor(() => expect(attached.status().counters.dispatchTerminalWaitingObserved) + .toBeGreaterThanOrEqual(1), { timeout: 4_000 }) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(attachedFleet.hydrated).toEqual([]) + expect(attachedFleet.spawns).toEqual([]) + expect(attachedFleet.resumes).toEqual([]) + expect(attached.status().counters.githubPullRequestsPublished).toBeUndefined() + + if (crashAfterPark) { + // Drop the healthy owner's subscriptions, then leave behind the same + // nominal foreign-lease window an ungraceful crash would expose. + await first.stop() + const persisted = state() + const key = issueKey(decision.issue) + const waitingLifecycle = await persisted.getDispatchLifecycle('factory-test', key) + expect(waitingLifecycle).toMatchObject({ phase: 'waiting-for-human' }) + const crashLease = await persisted.claimDispatchLifecycle( + 'factory-test', + key, + waitingLifecycle!, + 'crashed-waiting-owner', + clock.now(), + 5 * 60_000, + ) + expect(crashLease.acquired).toBe(true) + const waitingObservations = attached.status().counters.dispatchTerminalWaitingObserved ?? 0 + clock.advance(5 * 60_000 + 1) + await vi.waitFor(() => expect(attached.status().counters.dispatchTerminalWaitingObserved ?? 0) + .toBeGreaterThan(waitingObservations), { timeout: 4_000 }) + await vi.waitFor(() => expect(attached.status().counters.slackWatchersRearmed).toBe(1), { timeout: 4_000 }) + } + emitSlackReply( + mount, + slackReplyFixturePath('C0FACTORY__factory-e2e', mount.threadTs, `human-answer-${number}`), + `human-answer-${number}`, + { + text: `Resume issue ${number}.`, + user: 'U123', + user_is_bot: false, + }, + ) + + const completingFleet = crashAfterPark ? attachedFleet : firstFleet + await vi.waitFor(() => expect(completingFleet.resumes).toHaveLength(2), { timeout: 4_000 }) + if (crashAfterPark) expect(firstFleet.resumes).toEqual([]) + else expect(attachedFleet.resumes).toEqual([]) + completingFleet.emitAgentExit(`ar-${number}-review`, 'completed') + await terminal + expect(await state().getDispatchLifecycle('factory-test', issueKey(decision.issue))) + .toMatchObject({ phase: 'complete' }) + expect(attachedFleet.spawns).toEqual([]) + expect(attached.status().counters.githubPullRequestsPublished).toBeUndefined() + } finally { + await attached.stop() + await first.stop() + await rm(root, { recursive: true, force: true }) + } + }, 15_000) + + it('fences a duplicate lifecycle owner before it can spawn a second remote team', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-duplicate-owner-')) + const watchStatePath = join(root, 'state.json') + const mount = new FakeMountClient({ [issuePath(85)]: issueFile(85) }) + try { + const firstFleet = new RemoteLifecycleFleetClient() + const first = createFactory(config(), { + mount, + fleet: firstFleet, + stateStore: new FileStateStore({ batchSize: 2, watchStatePath }), + triage: new StaticTriage(), + }) + const decision = await first.triageIssue(parseLinearIssue(issuePath(85), issueFile(85))) + await first.dispatch(decision) + + const duplicateFleet = new RemoteLifecycleFleetClient() + const duplicate = createFactory(config(), { + mount: new FakeMountClient({ [issuePath(85)]: issueFile(85) }), + fleet: duplicateFleet, + stateStore: new FileStateStore({ batchSize: 2, watchStatePath }), + triage: new StaticTriage(), + }) + await expect(duplicate.dispatch(decision)).rejects.toThrow(/lifecycle is owned by/) + expect(firstFleet.spawns).toHaveLength(2) + expect(duplicateFleet.spawns).toEqual([]) + await first.stop() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('keeps a durable queued issue from spawning after restart until the running slot is released', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-durable-queue-restart-')) + const watchStatePath = join(root, 'state.json') + const clock = new ManualClock() + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => ({ + repo: input.repo, + number: 985, + url: 'https://github.com/AgentWorkforce/pear/pull/985', + headRef: input.headRef!, + }), + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(985)]: issueFile(985), + [issuePath(986)]: issueFile(986), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const state = () => new FileStateStore({ batchSize: 1, watchStatePath }) + const firstFleet = new RemoteLifecycleFleetClient() + const first = createFactory(config({ batchSize: 1 }), { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + clock, + probePrResolver: async () => undefined, + }) + let restarted: ReturnType | undefined + try { + const running = await first.triageIssue(parseLinearIssue(issuePath(985), issueFile(985))) + const queued = await first.triageIssue(parseLinearIssue(issuePath(986), issueFile(986))) + await first.dispatch(running) + await first.dispatch(queued) + expect(firstFleet.spawns.map((spawn) => spawn.name)).toEqual(['ar-985-impl-pear', 'ar-985-review']) + expect(await state().getDispatchLifecycle('factory-test', issueKey(queued.issue))).toMatchObject({ phase: 'queued' }) + + clock.advance(5 * 60_000 + 1) + const restartedFleet = new RemoteLifecycleFleetClient() + restarted = createFactory(config({ batchSize: 1 }), { + mount, + fleet: restartedFleet, + stateStore: state(), + triage: new StaticTriage(), + clock, + probePrResolver: async () => undefined, + }) + await restarted.start({ mode: 'dispatch-owner' }) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(restartedFleet.spawns).toEqual([]) + + restartedFleet.emitAgentExit('ar-985-impl-pear', 'exited') + await vi.waitFor(() => expect(restartedFleet.spawns.map((spawn) => spawn.name)) + .toEqual(['ar-986-impl-pear', 'ar-986-review']), { timeout: 4_000 }) + await first.stop() + } finally { + await restarted?.stop() + await first.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('enforces one global durable slot across concurrent same-host control-plane processes', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-global-capacity-')) + const watchStatePath = join(root, 'state.json') + const state = () => new FileStateStore({ batchSize: 1, watchStatePath }) + const firstFleet = new RemoteLifecycleFleetClient() + const secondFleet = new RemoteLifecycleFleetClient() + const firstMount = new FakeMountClient({ [issuePath(987)]: issueFile(987) }) + const secondMount = new FakeMountClient({ [issuePath(988)]: issueFile(988) }) + const first = createFactory(config({ batchSize: 1 }), { + mount: firstMount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + const second = createFactory(config({ batchSize: 1 }), { + mount: secondMount, + fleet: secondFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + try { + await first.dispatch(await first.triageIssue(parseLinearIssue(issuePath(987), issueFile(987)))) + const secondDecision = await second.triageIssue(parseLinearIssue(issuePath(988), issueFile(988))) + await second.dispatch(secondDecision) + + expect(firstFleet.spawns).toHaveLength(2) + expect(secondFleet.spawns).toEqual([]) + expect(await state().getDispatchLifecycle('factory-test', issueKey(secondDecision.issue))).toMatchObject({ phase: 'queued' }) + } finally { + await second.stop() + await first.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('keeps global capacity occupied until a clarification team is confirmed parked', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-global-capacity-clarification-park-')) + const watchStatePath = join(root, 'state.json') + const state = () => new FileStateStore({ batchSize: 1, watchStatePath }) + const firstFleet = new BlockingClarificationReleaseFleetClient() + const secondFleet = new RemoteLifecycleFleetClient() + const firstMount = new ConfirmRecordingSlackMountClient({ [issuePath(989)]: issueFile(989) }) + const secondMount = new FakeMountClient({ [issuePath(990)]: issueFile(990) }) + const first = createFactory(config({ batchSize: 1, slack: slackConfig() }), { + mount: firstMount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + const second = createFactory(config({ batchSize: 1 }), { + mount: secondMount, + fleet: secondFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + try { + const firstDecision = await first.triageIssue(parseLinearIssue(issuePath(989), issueFile(989))) + const secondDecision = await second.triageIssue(parseLinearIssue(issuePath(990), issueFile(990))) + await first.dispatch(firstDecision) + firstFleet.emitAgentMessage({ + from: 'ar-989-impl-pear', + target: 'factory', + body: '[factory-needs-input] Keep my slot until the remote team is absent.', + eventId: 'agent-question-989', + }) + await firstFleet.clarificationReleaseStarted + await vi.waitFor(async () => { + expect(await state().getDispatchLifecycle('factory-test', issueKey(firstDecision.issue))) + .toMatchObject({ phase: 'parking' }) + }) + + await second.dispatch(secondDecision) + expect(await state().getDispatchLifecycle('factory-test', issueKey(secondDecision.issue))) + .toMatchObject({ phase: 'queued' }) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(secondFleet.spawns).toEqual([]) + + firstFleet.allowClarificationRelease() + await vi.waitFor(async () => { + expect(await state().getDispatchLifecycle('factory-test', issueKey(firstDecision.issue))) + .toMatchObject({ phase: 'waiting-for-human' }) + }) + await vi.waitFor(() => expect(secondFleet.spawns.map((spawn) => spawn.name)) + .toEqual(['ar-990-impl-pear', 'ar-990-review']), { timeout: 4_000 }) + } finally { + firstFleet.allowClarificationRelease() + await second.stop() + await first.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('recovers a durable clarification parking phase after the release owner stops', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-clarification-parking-takeover-')) + const watchStatePath = join(root, 'state.json') + const state = () => new FileStateStore({ batchSize: 1, watchStatePath }) + const mount = new ConfirmRecordingSlackMountClient({ + [issuePath(991)]: issueFile(991), + [issuePath(992)]: issueFile(992), + }) + const firstFleet = new FailingClarificationReleaseFleetClient() + firstFleet.setSessionRef('ar-991-impl-pear', 'session-ar-991-impl-pear') + firstFleet.setSessionRef('ar-991-review', 'session-ar-991-review') + const first = createFactory(config({ batchSize: 1, slack: slackConfig() }), { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + let restarted: ReturnType | undefined + let restartedFleet: ParkingTakeoverReconcileFleetClient | undefined + try { + const decision = await first.triageIssue(parseLinearIssue(issuePath(991), issueFile(991))) + await first.dispatch(decision) + firstFleet.emitAgentMessage({ + from: 'ar-991-impl-pear', + target: 'factory', + body: '[factory-needs-input] Recover my interrupted parking transition.', + eventId: 'agent-question-991', + }) + await vi.waitFor(() => expect(first.status().counters.clarificationParkReleasePending).toBe(1)) + await vi.waitFor(async () => { + expect(await state().getDispatchLifecycle('factory-test', issueKey(decision.issue))) + .toMatchObject({ phase: 'parking' }) + }) + const queuedDecision = await first.triageIssue(parseLinearIssue(issuePath(992), issueFile(992))) + await first.dispatch(queuedDecision) + expect(await state().getDispatchLifecycle('factory-test', issueKey(queuedDecision.issue))) + .toMatchObject({ phase: 'queued' }) + await first.stop() + + restartedFleet = new ParkingTakeoverReconcileFleetClient() + restarted = createFactory(config({ batchSize: 1, slack: slackConfig() }), { + mount, + fleet: restartedFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + const starting = restarted.start({ mode: 'dispatch-owner' }) + await restartedFleet.clarificationReleaseStarted + await vi.waitFor(() => expect(restarted?.status().counters.clarificationParkingExitsSuppressed).toBe(2)) + expect(restartedFleet.hydrated).toEqual([]) + expect(restartedFleet.resumes).toEqual([]) + expect(restartedFleet.spawns).toEqual([]) + expect(restarted.status().counters.exitPrPublishSkipped).toBeUndefined() + expect(await state().getDispatchLifecycle('factory-test', issueKey(decision.issue))) + .toMatchObject({ phase: 'parking' }) + + restartedFleet.allowClarificationRelease() + await starting + await vi.waitFor(async () => { + expect(await state().getDispatchLifecycle('factory-test', issueKey(decision.issue))) + .toMatchObject({ phase: 'waiting-for-human' }) + }) + expect(restartedFleet.releases.map((release) => release.name).sort()) + .toEqual(['ar-991-impl-pear', 'ar-991-review']) + expect(restarted.status().counters.clarificationParksRecovered).toBe(1) + await vi.waitFor(() => expect(restartedFleet.spawns.map((spawn) => spawn.name)) + .toEqual(['ar-992-impl-pear', 'ar-992-review']), { timeout: 4_000 }) + } finally { + restartedFleet?.allowClarificationRelease() + await restarted?.stop() + await first.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('fences a duplicate owner while the active owner is inside a slow PR publication', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-slow-publish-owner-')) + const watchStatePath = join(root, 'state.json') + let signalPublishStarted!: () => void + let releasePublish!: () => void + const publishStarted = new Promise((resolve) => { signalPublishStarted = resolve }) + const publishReleased = new Promise((resolve) => { releasePublish = resolve }) + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => { + signalPublishStarted() + await publishReleased + return { + repo: input.repo, + number: 685, + url: 'https://github.com/AgentWorkforce/pear/pull/685', + headRef: input.headRef!, + } + }, + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(685)]: issueFile(685), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const state = () => new FileStateStore({ batchSize: 2, watchStatePath }) + const firstFleet = new RemoteLifecycleFleetClient() + const first = createFactory(config(), { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + try { + const decision = await first.triageIssue(parseLinearIssue(issuePath(685), issueFile(685))) + await first.dispatch(decision) + firstFleet.emitAgentExit('ar-685-impl-pear', 'exited') + await publishStarted + + const duplicateFleet = new RemoteLifecycleFleetClient() + const duplicate = createFactory(config(), { + mount: new FakeMountClient({ [issuePath(685)]: issueFile(685) }), + fleet: duplicateFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + await expect(duplicate.dispatch(decision)).rejects.toThrow(/lifecycle is owned by/) + expect(duplicateFleet.spawns).toEqual([]) + + releasePublish() + await vi.waitFor(() => expect(first.status().counters.done).toBe(1)) + expect(firstFleet.spawns).toHaveLength(2) + await duplicate.stop() + } finally { + releasePublish() + await first.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('adopts a roster-visible remote spawn after crashing across the ack persistence gap', async () => { + class AckGapFleet extends RemoteLifecycleFleetClient { + failed = false + + override async spawn(input: SpawnInput): Promise { + const result = await super.spawn(input) + if (!this.failed) { + this.failed = true + throw new Error('owner crashed after remote spawn ack') + } + return result + } + } + const root = await mkdtemp(join(tmpdir(), 'factory-ack-gap-')) + const watchStatePath = join(root, 'state.json') + const fleet = new AckGapFleet() + const mount = new FakeMountClient({ [issuePath(585)]: issueFile(585) }) + const factory = createFactory(config(), { + mount, + fleet, + stateStore: new FileStateStore({ batchSize: 2, watchStatePath }), + triage: new StaticTriage(), + }) + try { + const decision = await factory.triageIssue(parseLinearIssue(issuePath(585), issueFile(585))) + await expect(factory.dispatch(decision)).rejects.toThrow('owner crashed after remote spawn ack') + await vi.waitFor(async () => expect(await new FileStateStore({ batchSize: 2, watchStatePath }) + .getDispatchLifecycle('factory-test', issueKey(decision.issue))).toMatchObject({ phase: 'running' }), { timeout: 4_000 }) + + expect(fleet.spawns.filter((spawn) => spawn.name === 'ar-585-impl-pear')).toHaveLength(1) + expect(fleet.spawns.filter((spawn) => spawn.name === 'ar-585-review')).toHaveLength(1) + expect((await new FileStateStore({ batchSize: 2, watchStatePath }) + .getDispatchLifecycle('factory-test', issueKey(decision.issue)))?.agents + .find((agent) => agent.name === 'ar-585-impl-pear')?.tracked.result?.node).toBe('sf-mini') + } finally { + await factory.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('autonomously retries a transient remote PR publication without another exit event', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-publish-retry-')) + const watchStatePath = join(root, 'state.json') + let attempts = 0 + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => { + attempts += 1 + if (attempts === 1) throw new Error('transient provider outage') + return { + repo: input.repo, + number: 185, + url: 'https://github.com/AgentWorkforce/pear/pull/185', + headRef: input.headRef!, + } + }, + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(185)]: issueFile(185), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const fleet = new RemoteLifecycleFleetClient() + const factory = createFactory(config(), { + mount, + fleet, + stateStore: new FileStateStore({ batchSize: 2, watchStatePath }), + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + try { + const decision = await factory.triageIssue(parseLinearIssue(issuePath(185), issueFile(185))) + await factory.dispatch(decision) + fleet.emitAgentExit('ar-185-impl-pear', 'exited') + + await vi.waitFor(() => expect(attempts).toBe(2), { timeout: 4_000 }) + await vi.waitFor(() => expect(factory.status().counters.done).toBe(1), { timeout: 4_000 }) + expect(fleet.resumes).toEqual([]) + expect(fleet.releases.map((release) => release.name).sort()).toEqual(['ar-185-impl-pear', 'ar-185-review']) + } finally { + await factory.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('reuses the provider receipt after losing the lifecycle lease between publication and persistence', async () => { + class RejectFirstPublishedSaveStore extends FileStateStore { + rejected = false + + override async saveDispatchLifecycle(...args: Parameters): Promise { + const lifecycle = args[5] + if (lifecycle.phase === 'published' && !this.rejected) { + this.rejected = true + return false + } + return super.saveDispatchLifecycle(...args) + } + } + + const root = await mkdtemp(join(tmpdir(), 'factory-publish-receipt-lease-loss-')) + const watchStatePath = join(root, 'state.json') + let publishAttempts = 0 + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => { + publishAttempts += 1 + return { + repo: input.repo, + number: 1085, + url: 'https://github.com/AgentWorkforce/pear/pull/1085', + headRef: input.headRef!, + } + }, + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(1085)]: issueFile(1085), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const fleet = new RemoteLifecycleFleetClient() + const stateStore = new RejectFirstPublishedSaveStore({ batchSize: 2, watchStatePath }) + const factory = createFactory(config(), { + mount, + fleet, + stateStore, + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + try { + await factory.dispatch(await factory.triageIssue(parseLinearIssue(issuePath(1085), issueFile(1085)))) + fleet.emitAgentExit('ar-1085-impl-pear', 'exited') + + await vi.waitFor(() => expect(factory.status().counters.done).toBe(1), { timeout: 4_000 }) + expect(stateStore.rejected).toBe(true) + expect(publishAttempts).toBe(1) + } finally { + await factory.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('waits for mounted metadata to confirm the exact remote PR is open and non-draft', async () => { + const prPath = '/github/repos/AgentWorkforce/pear/pulls/885/metadata.json' + let signalStaleRead!: () => void + const staleRead = new Promise((resolve) => { signalStaleRead = resolve }) + class ConfirmingMount extends FakeMountClient { + override async readFile(path: string): Promise<{ content: unknown; revision?: string }> { + const value = await super.readFile(path) + if (path === prPath) signalStaleRead() + return value + } + } + const publishInputs: GithubPublishPullRequestInput[] = [] + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => { + publishInputs.push(input) + return { + repo: input.repo, + number: 885, + url: 'https://github.com/AgentWorkforce/pear/pull/885', + headRef: input.headRef!, + } + }, + closePullRequest: async () => undefined, + } + const mount = new ConfirmingMount({ + [issuePath(885)]: issueFile(885), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + [prPath]: { + number: 885, + state: 'open', + head_ref: 'factory/ar-885-agentworkforce-pear-stale-run', + isDraft: false, + }, + }, githubWrite) + Object.defineProperty(mount, 'writebackTransport', { value: 'relayfile-cloud' }) + const fleet = new RemoteLifecycleFleetClient() + const factory = createFactory(config(), { + mount, + fleet, + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + try { + await factory.dispatch(await factory.triageIssue(parseLinearIssue(issuePath(885), issueFile(885)))) + fleet.emitAgentExit('ar-885-impl-pear', 'exited') + await staleRead + expect(factory.status().counters.done ?? 0).toBe(0) + + mount.files.set(prPath, { content: { + number: 885, + state: 'open', + head_ref: publishInputs[0]?.headRef, + isDraft: false, + } }) + await vi.waitFor(() => expect(factory.status().counters.done).toBe(1)) + expect(fleet.releases.map((release) => release.name).sort()).toEqual(['ar-885-impl-pear', 'ar-885-review']) + } finally { + await factory.stop() + } + }) + + it('takes over a persisted publishing phase after the owner stops', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-publishing-takeover-')) + const watchStatePath = join(root, 'state.json') + let providerReady = false + let attempts = 0 + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => { + attempts += 1 + if (!providerReady) throw new Error('publisher unavailable') + return { + repo: input.repo, + number: 485, + url: 'https://github.com/AgentWorkforce/pear/pull/485', + headRef: input.headRef!, + } + }, + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(485)]: issueFile(485), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const state = () => new FileStateStore({ batchSize: 2, watchStatePath }) + const firstFleet = new RemoteLifecycleFleetClient() + const first = createFactory(config(), { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + try { + const decision = await first.triageIssue(parseLinearIssue(issuePath(485), issueFile(485))) + await first.dispatch(decision) + firstFleet.emitAgentExit('ar-485-impl-pear', 'exited') + await vi.waitFor(() => expect(attempts).toBe(1)) + await vi.waitFor(async () => expect(await state().getDispatchLifecycle('factory-test', issueKey(decision.issue))) + .toMatchObject({ phase: 'publishing' })) + await first.stop() + + providerReady = true + const restartedFleet = new RemoteLifecycleFleetClient() + const restarted = createFactory(config(), { + mount, + fleet: restartedFleet, + stateStore: state(), + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + await restarted.start({ mode: 'dispatch-owner' }) + await vi.waitFor(() => expect(restarted.status().counters.done).toBe(1), { timeout: 4_000 }) + expect(attempts).toBe(2) + expect(restartedFleet.releases.map((release) => release.name).sort()).toEqual(['ar-485-impl-pear', 'ar-485-review']) + await restarted.stop() + } finally { + await first.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('pins a resumed session to its placed node/repo and never signals its remote PID locally', async () => { + class RemotePidFleet extends RemoteLifecycleFleetClient { + override async spawn(input: SpawnInput): Promise { + return { ...await super.spawn(input), pid: 4242 } + } + } + const fleet = new RemotePidFleet() + fleet.setSessionRef('ar-385-review', 'remote-review-session') + const killed: Array<{ pid: number; signal?: NodeJS.Signals | number }> = [] + const mount = new FakeMountClient({ [issuePath(385)]: issueFile(385) }) const factory = createFactory(config(), { mount, fleet, triage: new StaticTriage(), - logger: { - debug: (...args: unknown[]) => debugLogs.push(args), - warn: () => undefined, - error: () => undefined, - }, + kill: (pid, signal) => { killed.push({ pid, signal }) }, + probePrResolver: async () => undefined, }) - const report = await factory.runOnce() + await factory.dispatch(await factory.triageIssue(parseLinearIssue(issuePath(385), issueFile(385)))) + fleet.emitAgentExit('ar-385-review', 'crash') + await vi.waitFor(() => expect(fleet.resumes).toHaveLength(1)) - expect(report.pulled.map((issue) => issue.path)).toEqual([capturedReadyCanaryPath]) - expect(mount.readPaths).toContain(missingCanonicalPath) - expect(factory.status().counters.phantomSkipped).toBe(1) - expect(debugLogs).toContainEqual([ - '[factory] skipped missing issue file discovered from issue tree', - { path: missingCanonicalPath }, - ]) + expect(fleet.resumes[0]).toMatchObject({ + name: 'ar-385-review', + sessionRef: 'remote-review-session', + node: 'sf-mini', + repo: 'AgentWorkforce/pear', + clonePath: '/work/pear', + }) + expect(killed).toEqual([]) + await factory.stop() }) - it('runLoop stops at the configured iteration cap, preserves the batch cap, and advances heartbeat liveness', async () => { - const root = await mkdtemp(join(tmpdir(), 'factory-loop-heartbeat-')) - const heartbeatPath = join(root, 'heartbeat.json') - const registryPath = join(root, 'registry.json') + it('frees the batch slot before retrying a transient remote release failure', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-release-retry-')) + const watchStatePath = join(root, 'state.json') + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => ({ + repo: input.repo, + number: 285, + url: 'https://github.com/AgentWorkforce/pear/pull/285', + headRef: input.headRef!, + }), + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(285)]: issueFile(285), + [issuePath(286)]: issueFile(286), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const fleet = new TransientRemoteReleaseFleetClient() + fleet.failReleaseFor = 'ar-285-impl-pear' + const factory = createFactory(config({ batchSize: 1 }), { + mount, + fleet, + stateStore: new FileStateStore({ batchSize: 1, watchStatePath }), + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) try { - const mount = new FakeMountClient(Object.fromEntries( - [51, 52, 53, 54, 55, 56].map((n) => [issuePath(n), issueFile(n)]), - )) - const fleet = new FakeFleetClient() - const factory = createFactory(config({ - batchSize: 5, - loop: { maxIterations: 3, heartbeatPath, registryPath, heartbeatStaleMs: 10_000 }, - }), { mount, fleet, triage: new StaticTriage() }) - - const reports = await factory.runLoop({ dryRun: true }) - - expect(reports).toHaveLength(3) - expect(factory.status().counters.loopIdle).toBe(1) - expect(factory.status().inFlight).toHaveLength(5) - expect(factory.status().queued).toHaveLength(1) - const heartbeat = await readFactoryLoopHeartbeat(heartbeatPath) - expect(heartbeat).toMatchObject({ status: 'idle', iteration: 3, maxIterations: 3, pid: process.pid }) - expect(checkFactoryLoopLiveness(heartbeat, { nowMs: heartbeat!.updatedAtMs + 500, staleMs: 10_000 })).toMatchObject({ - ok: true, - stale: false, + const first = await factory.triageIssue(parseLinearIssue(issuePath(285), issueFile(285))) + const second = await factory.triageIssue(parseLinearIssue(issuePath(286), issueFile(286))) + await factory.dispatch(first) + await factory.dispatch(second) + fleet.emitAgentExit('ar-285-impl-pear', 'exited') + + await vi.waitFor(() => expect(fleet.spawns.map((spawn) => spawn.name)).toContain('ar-286-impl-pear'), { + timeout: 4_000, }) + await vi.waitFor(() => expect(fleet.releases.filter((release) => release.name === 'ar-285-impl-pear')).toHaveLength(1), { + timeout: 4_000, + }) + expect(fleet.releaseFailures).toBe(1) } finally { + await factory.stop() await rm(root, { recursive: true, force: true }) } }) - it('start re-adopts remote in-flight registry agents into the fleet and reconciles once', async () => { - const root = await mkdtemp(join(tmpdir(), 'factory-adopt-inflight-')) - const registryPath = join(root, 'registry.json') + it('takes over persisted release cleanup after the publishing owner stops', async () => { + class ReleaseOutageFleet extends RemoteLifecycleFleetClient { + override async release(name: string, reason?: string): Promise { + if (name === 'ar-785-impl-pear') throw new Error('release service unavailable') + await super.release(name, reason) + } + } + + const root = await mkdtemp(join(tmpdir(), 'factory-release-takeover-')) + const watchStatePath = join(root, 'state.json') + const githubWrite: GithubConnectionWrite = { + publishPullRequest: async (input) => ({ + repo: input.repo, + number: 785, + url: 'https://github.com/AgentWorkforce/pear/pull/785', + headRef: input.headRef!, + }), + closePullRequest: async () => undefined, + } + const mount = new FakeMountClient({ + [issuePath(785)]: issueFile(785), + '/github/repos/AgentWorkforce/pear/meta.json': { default_branch: 'main' }, + }, githubWrite) + const state = () => new FileStateStore({ batchSize: 1, watchStatePath }) + const firstFleet = new ReleaseOutageFleet() + const first = createFactory(config({ batchSize: 1 }), { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + let restarted: ReturnType | undefined try { - await writeFile(registryPath, JSON.stringify({ - pid: 12345, - updatedAt: new Date(0).toISOString(), - updatedAtMs: 0, - agents: [ - { name: 'ar-1-impl', pids: [], invocationId: 'inv-1', node: 'mac-mini' }, - { name: 'ar-1-review', pids: [], node: 'mac-mini' }, - // Local-only agents (pids, no placement facts) are not fleet-tracked. - { name: 'ar-2-impl', pids: [4242] }, - ], - })) - const fleet = new FakeFleetClient() - const factory = createFactory(config({ - loop: { maxIterations: 1, heartbeatPath: join(root, 'heartbeat.json'), registryPath, heartbeatStaleMs: 1_000 }, - }), { - mount: new FakeMountClient(), - fleet, - triage: new StaticTriage(), + const decision = await first.triageIssue(parseLinearIssue(issuePath(785), issueFile(785))) + await first.dispatch(decision) + firstFleet.emitAgentExit('ar-785-impl-pear', 'exited') + await vi.waitFor(async () => { + const lifecycle = await state().getDispatchLifecycle('factory-test', issueKey(decision.issue)) + expect(lifecycle).toMatchObject({ phase: 'releasing' }) + expect(lifecycle?.agents.find((agent) => agent.name === 'ar-785-review')?.releasedAtMs) + .toEqual(expect.any(Number)) }) + await first.stop() - await factory.start({ mode: 'live', liveSubscription: { transport: 'subscribe' } }) - - expect(fleet.hydrated).toEqual([ - { name: 'ar-1-impl', invocationId: 'inv-1', node: 'mac-mini' }, - { name: 'ar-1-review', invocationId: undefined, node: 'mac-mini' }, - ]) - expect(fleet.reconciles).toBe(1) - - await factory.stop() + const restartedFleet = new RemoteLifecycleFleetClient() + restarted = createFactory(config({ batchSize: 1 }), { + mount, + fleet: restartedFleet, + stateStore: state(), + triage: new StaticTriage(), + probePrResolver: async () => undefined, + }) + await restarted.start({ mode: 'dispatch-owner' }) + await vi.waitFor(async () => expect(await state().getDispatchLifecycle('factory-test', issueKey(decision.issue))) + .toMatchObject({ phase: 'complete' }), { timeout: 4_000 }) + // The reviewer acknowledgement was fenced before the first owner + // stopped, so takeover retries only the failed implementer release. + expect(restartedFleet.releases.map((release) => release.name)).toEqual(['ar-785-impl-pear']) } finally { + await restarted?.stop() + await first.stop() await rm(root, { recursive: true, force: true }) } }) @@ -5518,6 +6520,8 @@ describe('FactoryLoop', () => { sessionRef: 'session-review-6', node: 'self', capability: 'spawn:claude', + repo: 'AgentWorkforce/pear', + clonePath: '/work/pear', }]) expect(fleet.spawns.map((spawn) => spawn.name)).toEqual(['ar-6-impl-pear', 'ar-6-review']) }) @@ -5960,6 +6964,8 @@ describe('FactoryLoop', () => { sessionRef: 'session-impl-256', node: 'self', capability: 'spawn:codex', + repo: 'AgentWorkforce/pear', + clonePath: '/work/pear', }]) }) @@ -5992,6 +6998,8 @@ describe('FactoryLoop', () => { sessionRef: 'session-impl-255', node: 'self', capability: 'spawn:codex', + repo: 'AgentWorkforce/pear', + clonePath: '/work/pear', }]) expect(fleet.releases).toEqual([]) expect(factory.status().counters.done).toBeUndefined() @@ -6016,6 +7024,8 @@ describe('FactoryLoop', () => { sessionRef: 'session-review-10', node: 'self', capability: 'spawn:claude', + repo: 'AgentWorkforce/pear', + clonePath: '/work/pear', }]) }) @@ -8702,8 +9712,8 @@ describe('FactoryLoop', () => { { name: 'ar-36-review', data: '\nHuman reply in the Slack thread:\nUse the shared retry helper in factory.ts.\n\r' }, ]) expect(fleet.resumes).toEqual([ - { name: 'ar-36-impl-pear', sessionRef: 'session-ar-36-impl-pear', node: 'self', capability: 'spawn:codex' }, - { name: 'ar-36-review', sessionRef: 'session-ar-36-review', node: 'self', capability: 'spawn:claude' }, + { name: 'ar-36-impl-pear', sessionRef: 'session-ar-36-impl-pear', node: 'self', capability: 'spawn:codex', repo: 'AgentWorkforce/pear', clonePath: '/work/pear' }, + { name: 'ar-36-review', sessionRef: 'session-ar-36-review', node: 'self', capability: 'spawn:claude', repo: 'AgentWorkforce/pear', clonePath: '/work/pear' }, ]) expect(factory.status().inFlight.map((issue) => issue.key)).toEqual(['AR-36']) expect(fleet.messages).toHaveLength(2) @@ -8931,7 +9941,7 @@ describe('FactoryLoop', () => { expect(pending?.releasedAgents).toEqual(['ar-50-impl-pear', 'ar-50-review']) expect(pending?.parkedAtMs).toBeUndefined() expect(factory.status().counters.agentQuestionTeamsReleased).toBeUndefined() - expect(factory.status().inFlight).toEqual([]) + expect(factory.status().inFlight.map((issue) => issue.key)).toEqual(['AR-50']) mount.files.set(issuePath(50), { content: issueFile(50, implementing) }) await factory.runLoop({ maxIterations: 1 }) @@ -9499,6 +10509,147 @@ describe('FactoryLoop', () => { } }) + it('dispatch-owner recovers a remote team parked for clarification and preserves its prior dispatch result', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-clarification-dispatch-owner-')) + try { + const watchStatePath = join(root, 'factory-state.json') + const mount = new ConfirmRecordingSlackMountClient({ [issuePath(955)]: issueFile(955) }) + const firstFleet = new RemoteLifecycleFleetClient() + firstFleet.setSessionRef('ar-955-impl-pear', 'session-ar-955-impl-pear') + firstFleet.setSessionRef('ar-955-review', 'session-ar-955-review') + const factoryConfig = config({ batchSize: 1, slack: slackConfig() }) + const state = () => new FileStateStore({ batchSize: 1, watchStatePath }) + const first = createFactory(factoryConfig, { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + const decision = await first.triageIssue(parseLinearIssue(issuePath(955), issueFile(955))) + const originalResult = await first.dispatch(decision) + mount.files.set(issuePath(955), { content: issueFile(955, implementing) }) + firstFleet.emitAgentMessage({ + from: 'ar-955-impl-pear', + target: 'factory', + body: '[factory-needs-input] Recover this from a replacement dispatch owner.', + eventId: 'agent-question-955', + }) + await vi.waitFor(() => expect(first.status().counters.agentQuestionTeamsReleased).toBe(1)) + await vi.waitFor(() => expect(first.status().counters.clarificationQuestionsDelivered).toBe(1)) + await first.stop() + + const restartedFleet = new RemoteLifecycleFleetClient() + const restarted = createFactory(factoryConfig, { + mount, + fleet: restartedFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + await restarted.start({ mode: 'dispatch-owner' }) + await expect(restarted.dispatch(decision)).resolves.toEqual(originalResult) + + const terminal = restarted.waitForDispatchTerminal(decision.issue) + emitSlackReply( + mount, + slackReplyFixturePath('C0FACTORY__factory-e2e', mount.threadTs, 'human-answer-955'), + 'persisted-answer-955', + { + text: 'Resume on the original remote node.', + user: 'U123', + user_is_bot: false, + }, + ) + await vi.waitFor(() => expect(restarted.status().counters.clarificationTeamsWoken).toBe(1)) + expect(restartedFleet.resumes).toEqual([ + expect.objectContaining({ node: 'sf-mini', repo: 'AgentWorkforce/pear', clonePath: '/work/pear' }), + expect.objectContaining({ node: 'sf-mini', repo: 'AgentWorkforce/pear', clonePath: '/work/pear' }), + ]) + restartedFleet.emitAgentExit('ar-955-review', 'completed') + await terminal + expect(await state().getDispatchLifecycle('factory-test', issueKey(decision.issue))).toMatchObject({ phase: 'complete' }) + await restarted.stop() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('preserves remote placement when adopting an online clarification wake across the spawn-ack crash gap', async () => { + class OnlineClarificationFleet extends RemoteLifecycleFleetClient { + override async roster() { + return { + agents: [ + { name: 'ar-956-impl-pear', node: 'sf-mini' }, + { name: 'ar-956-review', node: 'sf-mini' }, + ], + nodes: [], + } + } + } + + const root = await mkdtemp(join(tmpdir(), 'factory-clarification-placement-gap-')) + try { + const watchStatePath = join(root, 'factory-state.json') + const mount = new ConfirmRecordingSlackMountClient({ [issuePath(956)]: issueFile(956) }) + const firstFleet = new RemoteLifecycleFleetClient() + firstFleet.setSessionRef('ar-956-impl-pear', 'session-ar-956-impl-pear') + firstFleet.setSessionRef('ar-956-review', 'session-ar-956-review') + const factoryConfig = config({ slack: slackConfig() }) + const state = () => new FileStateStore({ batchSize: 2, watchStatePath }) + const first = createFactory(factoryConfig, { + mount, + fleet: firstFleet, + stateStore: state(), + triage: new StaticTriage(), + }) + await first.dispatch(await first.triageIssue(parseLinearIssue(issuePath(956), issueFile(956)))) + mount.files.set(issuePath(956), { content: issueFile(956, implementing) }) + firstFleet.emitAgentMessage({ + from: 'ar-956-impl-pear', + target: 'factory', + body: '[factory-needs-input] Preserve placement across wake adoption.', + eventId: 'agent-question-956', + }) + await vi.waitFor(() => expect(first.status().counters.agentQuestionTeamsReleased).toBe(1)) + await vi.waitFor(() => expect(first.status().counters.clarificationQuestionsDelivered).toBe(1)) + await first.stop() + + const persisted = state() + const [key] = (await persisted.listWaitingClarifications('factory-test'))[0]! + const claimed = await persisted.claimClarificationReply('factory-test', key, { + id: 'persisted-answer-956', + text: 'Adopt the already-online wake.', + receivedAtMs: 500, + }) + expect(claimed?.reply?.id).toBe('persisted-answer-956') + + const fleet = new OnlineClarificationFleet() + const killed: number[] = [] + const restarted = createFactory(factoryConfig, { + mount, + fleet, + stateStore: state(), + triage: new StaticTriage(), + kill: (pid) => { killed.push(pid) }, + }) + await restarted.start({ mode: 'dispatch-owner' }) + await vi.waitFor(() => expect(restarted.status().counters.clarificationTeamsWoken).toBe(1)) + expect(fleet.resumes).toEqual([]) + + fleet.emitAgentExit('ar-956-review', 'crash') + await vi.waitFor(() => expect(fleet.resumes).toHaveLength(1)) + expect(fleet.resumes[0]).toMatchObject({ + name: 'ar-956-review', + node: 'sf-mini', + repo: 'AgentWorkforce/pear', + clonePath: '/work/pear', + }) + expect(killed).toEqual([]) + await restarted.stop() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + it('rearms a persisted mid-task GitHub question and replays a reply received while stopped', async () => { const path = githubIssuePath('AgentWorkforce', 'pear', 63) const issue = githubIssueFile(63, { labels: ['factory'], author: 'reporter' }) @@ -10246,6 +11397,153 @@ describe('FactoryLoop PR babysitter', () => { }) } + it('recovers a remote babysitter across the spawn-ack crash gap without replaying the original team tasks', async () => { + class BabysitterAckGapFleet extends RemoteLifecycleFleetClient { + crashed = false + + override async spawn(input: SpawnInput): Promise { + const result = await super.spawn(input) + if (input.name.includes('-babysit') && !this.crashed) { + this.crashed = true + throw new Error('owner crashed after remote babysitter spawn ack') + } + return result + } + } + + const root = await mkdtemp(join(tmpdir(), 'factory-remote-babysitter-ack-gap-')) + const watchStatePath = join(root, 'state.json') + const issue = realIssueFile(493, ready, { title: 'Real remote babysitter ack gap' }) + const prPath = '/github/repos/AgentWorkforce/pear/pulls/493/metadata.json' + const mount = new FakeMountClient({ [issuePath(493)]: issue }) + seedPrMeta(mount, 'AgentWorkforce/pear', 493, { state: 'open', draft: false }) + const fleet = new BabysitterAckGapFleet() + const state = () => new FileStateStore({ batchSize: 2, watchStatePath }) + const first = createFactory(babysitterConfig(), { + mount, + fleet, + triage: new StaticTriage(), + stateStore: state(), + }) + let restarted: ReturnType | undefined + try { + await first.start({ mode: 'live', liveSubscription: { transport: 'subscribe' } }) + const decision = await first.triageIssue(parseLinearIssue(issuePath(493), issue)) + await first.dispatch(decision) + mount.emit(changeEvent(prPath, 'remote-pr-493-open')) + await vi.waitFor(() => expect(first.status().counters.babysitterSpawnFailures).toBe(1)) + + const key = issueKey(decision.issue) + await vi.waitFor(async () => { + const lifecycle = await state().getDispatchLifecycle('factory-test', key) + expect(lifecycle?.phase).toBe('dispatching') + expect(lifecycle?.agents.find((agent) => agent.name === 'ar-493-babysit')?.tracked.result).toBeUndefined() + }) + await first.stop() + + const messagesBeforeRestart = fleet.messages.length + restarted = createFactory(babysitterConfig(), { + mount, + fleet, + triage: new StaticTriage(), + stateStore: state(), + }) + await restarted.start({ mode: 'dispatch-owner' }) + await vi.waitFor(async () => expect(await state().getDispatchLifecycle('factory-test', key)).toMatchObject({ + phase: 'running', + agents: expect.arrayContaining([ + expect.objectContaining({ + name: 'ar-493-babysit', + tracked: expect.objectContaining({ result: expect.objectContaining({ node: 'sf-mini', locality: 'remote' }) }), + }), + ]), + }), { timeout: 4_000 }) + await vi.waitFor(async () => expect(await state().listBabysitterSessions('factory-test')).toEqual([ + [key, expect.objectContaining({ agentName: 'ar-493-babysit', repo: 'AgentWorkforce/pear', prNumber: 493 })], + ])) + + expect(fleet.spawns.filter((spawn) => spawn.name === 'ar-493-babysit')).toHaveLength(1) + expect(fleet.messages.slice(messagesBeforeRestart).filter((message) => + message.to === 'ar-493-impl-pear' || message.to === 'ar-493-review')).toEqual([]) + + fleet.emitAgentMessage({ + from: 'ar-493-babysit', + target: 'factory', + body: '[factory-pr-ready] AR-493', + }) + await vi.waitFor(async () => expect(await state().getDispatchLifecycle('factory-test', key)) + .toMatchObject({ phase: 'complete' })) + expect(await state().listBabysitterSessions('factory-test')).toEqual([]) + } finally { + await first.stop() + await restarted?.stop() + await rm(root, { recursive: true, force: true }) + } + }) + + it('keeps a healthy remote lifecycle owner authoritative for restored babysitter wakes and critical state', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-remote-babysitter-owner-')) + const watchStatePath = join(root, 'state.json') + const issue = realIssueFile(494, ready, { title: 'Real remote babysitter owner fence' }) + const prPath = '/github/repos/AgentWorkforce/pear/pulls/494/metadata.json' + const reviewPath = '/github/repos/AgentWorkforce/pear/pulls/494/comments/9404.json' + const mount = new FakeMountClient({ [issuePath(494)]: issue }) + seedPrMeta(mount, 'AgentWorkforce/pear', 494, { state: 'open', draft: false }) + const state = () => new FileStateStore({ batchSize: 2, watchStatePath }) + const ownerFleet = new RemoteLifecycleFleetClient() + const owner = createFactory(babysitterConfig(), { + mount, + fleet: ownerFleet, + triage: new StaticTriage(), + stateStore: state(), + }) + const duplicateFleet = new RemoteLifecycleFleetClient() + const duplicate = createFactory(babysitterConfig(), { + mount, + fleet: duplicateFleet, + triage: new StaticTriage(), + stateStore: state(), + }) + try { + await owner.start({ mode: 'live', liveSubscription: { transport: 'subscribe' } }) + const decision = await owner.triageIssue(parseLinearIssue(issuePath(494), issue)) + await owner.dispatch(decision) + mount.emit(changeEvent(prPath, 'remote-pr-494-open')) + await vi.waitFor(() => expect(ownerFleet.spawns.map((spawn) => spawn.name)).toContain('ar-494-babysit')) + ownerFleet.emitAgentMessage({ + from: 'ar-494-babysit', + target: 'factory', + body: '[factory-babysitter-critical] AR-494 begin', + }) + await vi.waitFor(async () => expect((await state().listBabysitterSessions('factory-test'))[0]?.[1].critical).toBe(true)) + mount.emit(changeEvent(reviewPath, 'remote-pr-494-review')) + await vi.waitFor(async () => expect((await state().listBabysitterSessions('factory-test'))[0]?.[1].pendingKinds) + .toContain('review-comment')) + + await duplicate.start({ mode: 'dispatch-owner' }) + expect(duplicate.status().counters.babysitterOwnershipRestored).toBeUndefined() + expect(duplicate.status().counters.babysitterOwnershipRestoreSkippedNonOwner).toBe(1) + duplicateFleet.emitAgentMessage({ + from: 'ar-494-babysit', + target: 'factory', + body: '[factory-babysitter-critical] AR-494 end', + }) + await flush() + + expect((await state().listBabysitterSessions('factory-test'))[0]?.[1]).toMatchObject({ + critical: true, + pendingKinds: expect.arrayContaining(['review-comment']), + }) + expect(duplicateFleet.messages).toEqual([]) + expect(duplicateFleet.inputs).toEqual([]) + expect(duplicate.status().counters.babysitterCriticalSignalsIgnored).toBe(1) + } finally { + await duplicate.stop() + await owner.stop() + await rm(root, { recursive: true, force: true }) + } + }) + it('keeps equal-number GitHub babysitters and reviewer prompts isolated across repos', async () => { const number = 26 const pearPath = githubIssuePath('AgentWorkforce', 'pear', number) diff --git a/src/orchestrator/factory.ts b/src/orchestrator/factory.ts index 2ccc5ae..c5a25a1 100644 --- a/src/orchestrator/factory.ts +++ b/src/orchestrator/factory.ts @@ -18,11 +18,14 @@ import type { MountClient, ProviderSyncStatus, SlackWriteback, + SpawnResult, Subscription, } from '../ports' import type { BatchSnapshot, BabysitterSessionState, + DispatchLifecycle, + DispatchLifecyclePhase, GithubIssueCommentWatchPending, GithubIssueCommentWatchState, RegistryHandoffAgent, @@ -41,7 +44,7 @@ import { } from '@agent-relay/integration-prompts' import { renderAgentTask } from '../dispatch/templates' import { HeuristicTriage, TieredTriage, babysitterSpec, isShapeLabel, scopeFromLabels } from '../triage' -import { agentNameForRole } from '../triage/agent-names' +import { agentNameForRole, sanitizeAgentSlug } from '../triage/agent-names' import type { DispatchResult, Factory, @@ -187,6 +190,8 @@ const COMPLETION_SWEEP_INTERVAL_MS = 15_000 const COMPLETION_SWEEP_BATCH_SIZE = 2 const PROBE_PR_GH_BACKOFF_MS = 60_000 const PROBE_PR_GH_CANDIDATE_LIMIT = 200 +const PUBLISHED_PR_CONFIRM_ATTEMPTS = 20 +const PUBLISHED_PR_CONFIRM_DELAY_MS = 100 const SLACK_REPLY_EVENTS_LIMIT = 100 const SLACK_REPLY_POLL_INTERVAL_MS = 5_000 const AGENT_QUESTION_DEDUPE_LIMIT = 500 @@ -213,6 +218,9 @@ const CLARIFICATION_ESCALATION_LEASE_MS = 2 * 60_000 const CLARIFICATION_ESCALATION_RETRY_MS = 5_000 const CLARIFICATION_STALE_WARN_MS = 7 * 24 * 60 * 60_000 const STOP_TEARDOWN_TIMEOUT_MS = 2_500 +const DISPATCH_LIFECYCLE_LEASE_MS = 5 * 60_000 +const DISPATCH_LIFECYCLE_RENEW_MS = 60_000 +const DISPATCH_LIFECYCLE_RETRY_MS = 1_000 const SLACK_EVENT_WATERMARK_CACHE_MS = 60_000 const MERGE_GATE_MAX_ATTEMPTS = 12 const MERGE_GATE_POLL_DELAY_MS = 10_000 @@ -288,6 +296,12 @@ export class FactoryLoop implements Factory { readonly #clarificationWakeInFlight = new Map>() readonly #clarificationWakeRetryTimers = new Map>() readonly #clarificationWakeOwner = `${process.pid}:${randomUUID()}` + readonly #dispatchLifecycleOwner = `${process.pid}:${randomUUID()}` + readonly #dispatchLifecycleEpochs = new Map() + readonly #dispatchTerminalWaiters = new Map void>>() + readonly #dispatchLifecycleRetryTimers = new Map>() + readonly #dispatchLifecycleDrives = new Set>() + #dispatchLifecycleRenewTimer?: ReturnType #clarificationSweepTimer?: ReturnType #clarificationSweepDueAtMs?: number #clarificationSweepInFlight?: Promise @@ -334,7 +348,7 @@ export class FactoryLoop implements Factory { // and clears it afterward. Event text can be broker-delivered while a prompt // is active, but the PTY submit must never land in that critical window. readonly #babysitterCriticalAgents = new Set() - readonly #publishedPullRequests = new Set() + readonly #publishedPullRequests = new Map() readonly #probePrGhBackoffUntilMs = new Map() readonly #probePrResolvedCache = new Map() // GitHub issue mirror-id -> resolved Linear mirror path, so repeat ingestion @@ -352,6 +366,7 @@ export class FactoryLoop implements Factory { #integrationInstructionsRefresh?: Promise #starting?: Promise #started = false + #startMode?: FactoryStartOptions['mode'] #stopping = false constructor(config: FactoryConfig, ports: FactoryPorts) { @@ -464,6 +479,71 @@ export class FactoryLoop implements Factory { return batch } + async waitForDispatchTerminal(issue: IssueRef): Promise { + const key = issueKey(issue) + const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key) + if (lifecycle && isTerminalDispatchLifecycle(lifecycle)) return + await new Promise((resolve) => { + let settled = false + let timer: ReturnType | undefined + let waiters = this.#dispatchTerminalWaiters.get(key) + if (!waiters) { + waiters = new Set() + this.#dispatchTerminalWaiters.set(key, waiters) + } + const finish = (): void => { + if (settled) return + settled = true + if (timer) clearTimeout(timer) + const current = this.#dispatchTerminalWaiters.get(key) + current?.delete(finish) + if (current?.size === 0) this.#dispatchTerminalWaiters.delete(key) + resolve() + } + waiters.add(finish) + + // FileStateStore has no cross-process notification. Poll the durable row + // so an attached one-shot owner observes another healthy owner's + // terminal commit, including after an intermediate clarification park. + const poll = async (): Promise => { + if (settled) return + if (this.#stopping) { + finish() + return + } + try { + const latest = await this.#state.getDispatchLifecycle(this.#workspaceId, key) + if (latest && isTerminalDispatchLifecycle(latest)) { + this.#resolveDispatchTerminalWaiters(issue) + return + } + if (latest?.phase === 'waiting-for-human' && this.#startMode === 'dispatch-owner') { + this.#increment('dispatchTerminalWaitingObserved') + const canRecoverWaiting = !latest.lease || + latest.lease.owner === this.#dispatchLifecycleOwner || + latest.lease.leaseUntilMs <= this.#clock.now() + if (canRecoverWaiting) { + // The row may have entered waiting after this process's one-time + // startup recovery. Arm its durable reply channels only when + // ownership is reclaimable; while a healthy foreign owner holds + // the lease, its watcher must remain the sole wake driver. + await this.#rearmSlackReplyWatchers() + await this.#rearmGithubIssueCommentWatchers() + await this.#drainReadyClarificationWake() + } + } + } catch (error) { + this.#logger.warn?.('[factory] durable terminal observation failed; retrying', { + issue: issue.key, + error: describeError(error).errorMessage, + }) + } + if (!settled) timer = setTimeout(() => { void poll() }, DISPATCH_LIFECYCLE_RETRY_MS) + } + void poll() + }) + } + async start(opts: FactoryStartOptions = {}): Promise { if (this.#started) { return @@ -483,6 +563,7 @@ export class FactoryLoop implements Factory { async #start(opts: FactoryStartOptions): Promise { this.#stopping = false + this.#startMode = opts.mode ?? 'live' const issueSource = await this.#issueSource() if (issueSource === 'linear') { const ready = await this.#mount.ensureSubRoot(ISSUE_ROOT, { timeoutMs: 90_000 }) @@ -501,6 +582,18 @@ export class FactoryLoop implements Factory { await this.#adoptInFlightAgents() await this.#restoreBabysitterOwnership() + if (opts.mode === 'dispatch-owner') { + this.#started = true + this.#scheduleDispatchLifecycleRenewal() + // A replacement one-shot owner must also recover a team parked for + // human input; it intentionally does not subscribe to the full issue + // stream, but it does rearm the durable clarification channels. + await this.#rearmSlackReplyWatchers() + await this.#drainReadyClarificationWake() + await this.#rearmGithubIssueCommentWatchers() + return + } + if ((opts.mode ?? 'live') === 'live') { this.#started = true try { @@ -551,10 +644,15 @@ export class FactoryLoop implements Factory { async stop(): Promise { this.#started = false this.#stopping = true + if (this.#dispatchLifecycleRenewTimer) clearInterval(this.#dispatchLifecycleRenewTimer) + this.#dispatchLifecycleRenewTimer = undefined + for (const timer of this.#dispatchLifecycleRetryTimers.values()) clearTimeout(timer) + this.#dispatchLifecycleRetryTimers.clear() if (this.#completionSweepTimer) clearTimeout(this.#completionSweepTimer) this.#completionSweepTimer = undefined this.#stoppingHeartbeatRefreshActive = await this.#stopLiveHeartbeat('stopping') try { + await Promise.allSettled([...this.#dispatchLifecycleDrives]) // Fence every source of new clarification work before touching the fleet. // A wake already past the fence is allowed to unwind, and is awaited // without a timeout so it can never race fleet disposal. @@ -570,7 +668,20 @@ export class FactoryLoop implements Factory { await this.#drainBabysitterWakesForStop() - await this.#releaseInFlightAgents('factory-stopped') + // Durable relay placements must survive an owner restart so a successor + // can adopt them. The one-shot/daemon stop path releases only + // non-durable (local/internal) records; terminal completion performs the + // normal remote release before clearing the lifecycle. + await this.#releaseInFlightAgents('factory-stopped', { preserveDurable: true }) + for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) { + await this.#state.releaseDispatchLifecycleLease( + this.#workspaceId, + key, + this.#dispatchLifecycleOwner, + epoch, + ) + } + this.#dispatchLifecycleEpochs.clear() if (this.#livePollTimer) clearTimeout(this.#livePollTimer) this.#livePollTimer = undefined this.#livePollInFlight = false @@ -1527,6 +1638,19 @@ export class FactoryLoop implements Factory { if (existingRecord?.result) { return existingRecord.result } + if (!dryRun && this.#fleet.placementLocality === 'remote') { + const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(decision.issue)) + if (durable && !isTerminalDispatchLifecycle(durable)) { + if (durable.result && this.#dispatchLifecycleEpochs.has(issueKey(decision.issue))) { + return durable.result + } + if (this.#startMode === 'dispatch-owner') { + this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(durable)) + this.#increment('dispatchLifecycleForeignOwnerAttached') + return dispatchResultFromLifecycle(durable) + } + } + } const blockReason = await this.#dispatchBlockReason(decision.issue) if (blockReason) { @@ -1583,10 +1707,28 @@ export class FactoryLoop implements Factory { return { issue: decision.issue, agents: [], comments: [comment], dryRun } } - const dispatchDecision = labelDispatch.decision + let dispatchDecision = labelDispatch.decision // A valid label resolution clears any prior failure notice so a later // regression posts a fresh, actionable comment instead of being deduped. this.#labelDispatchFailures.delete(issueStateKey(dispatchDecision.issue)) + if (!dryRun && this.#fleet.placementLocality === 'remote') { + const lifecycleClaim = await this.#claimDispatchLifecycle(dispatchDecision, dryRun) + dispatchDecision = structuredClone(lifecycleClaim.lifecycle.decision) + if (lifecycleClaim.lifecycle.phase === 'waiting-for-human') { + return lifecycleClaim.lifecycle.result ?? { issue: dispatchDecision.issue, agents: [], dryRun } + } + if (lifecycleClaim.lifecycle.phase === 'queued') { + const queuedRecord = inFlightRecordFromLifecycle(lifecycleClaim.lifecycle) + this.#scheduleDispatchLifecycleRetry(queuedRecord) + this.#increment('queued') + this.#emit('issue-queued', { issue: dispatchDecision.issue }) + return lifecycleClaim.lifecycle.result ?? { issue: dispatchDecision.issue, agents: [], dryRun } + } + if (!lifecycleClaim.created) { + const restored = batch.restore(inFlightRecordFromLifecycle(lifecycleClaim.lifecycle)) + if (restored.result) return restored.result + } + } await this.#recordDispatchAttempt(dispatchDecision.issue) const record = batch.start(dispatchDecision, dryRun) if (!record) { @@ -1599,6 +1741,7 @@ export class FactoryLoop implements Factory { if (record.result) { return record.result } + await this.#saveDispatchLifecycle(record, 'dispatching') const spawnedForReaperHandoff: RegistryHandoffAgent[] = [] try { @@ -1648,6 +1791,7 @@ export class FactoryLoop implements Factory { dryRun, } record.result = result + await this.#saveDispatchLifecycle(record, 'running') this.#increment('dispatched') this.#emit('dispatched', { issue: dispatchDecision.issue, result }) if (!dryRun) { @@ -1661,7 +1805,10 @@ export class FactoryLoop implements Factory { } catch (error) { await this.#persistDispatchFailureReaperHandoff(record, spawnedForReaperHandoff) await this.#recordDispatchFailure(decision.issue) + const failedState = await this.#state.getDispatchAttempts(this.#workspaceId, decision.issue.key) + await this.#saveDispatchLifecycle(record, failedState?.terminal ? 'abandoned' : 'retryable') batch.abandon(decision.issue) + if (!failedState?.terminal) this.#scheduleDispatchLifecycleRetry(record) this.#error(error, decision.issue) throw error } @@ -1709,24 +1856,430 @@ export class FactoryLoop implements Factory { } // Remote backends survive orchestrator restarts: re-adopt the agents recorded - // in the in-flight registry, then reconcile once so exits that happened while - // this process was down are handled before any new dispatch. + // in the durable lifecycle store, restore their full batch/spec association, + // then reconcile once so exits that happened while this process was down are + // handled instead of being dropped as unknown agents. async #adoptInFlightAgents(): Promise { - if (!this.#fleet.hydrateTracked) return try { - const registry = await readFactoryInFlightRegistry(this.#config.loop.registryPath) - const agents = (registry?.agents ?? []).filter((agent) => agent.invocationId || agent.node) - if (agents.length > 0) { - this.#fleet.hydrateTracked(agents.map((agent) => ({ + const batch = await this.#batch() + const agents: Array<{ name: string; invocationId?: string; node?: string }> = [] + let hasNonterminalDurableLifecycle = false + for (const [key, lifecycle] of await this.#state.listDispatchLifecycles(this.#workspaceId)) { + if (isTerminalDispatchLifecycle(lifecycle)) continue + hasNonterminalDurableLifecycle = true + const claim = await this.#state.claimDispatchLifecycle( + this.#workspaceId, + key, + lifecycle, + this.#dispatchLifecycleOwner, + this.#clock.now(), + DISPATCH_LIFECYCLE_LEASE_MS, + ) + if (!claim.acquired || !claim.lease) { + // The other process may have crashed while its nominal lease is + // still live. Keep this process attached so it reclaims the row + // after expiry without another start/dispatch/fleet event. + this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(claim.lifecycle)) + continue + } + this.#dispatchLifecycleEpochs.set(key, claim.lease.epoch) + if (claim.lifecycle.phase === 'waiting-for-human') continue + const durableRecord = inFlightRecordFromLifecycle(claim.lifecycle) + const restored = claim.lifecycle.phase === 'queued' || claim.lifecycle.phase === 'releasing' + ? durableRecord + : batch.restore(durableRecord) + if (claim.lifecycle.phase !== 'running') this.#scheduleDispatchLifecycleRetry(restored) + // Parking agents are cleanup-only. Hydrating them makes relay + // reconciliation report their expected absence as an ordinary exit + // before the durable parking driver can release/confirm them. + if ( + claim.lifecycle.phase === 'queued' || + claim.lifecycle.phase === 'parking' || + claim.lifecycle.phase === 'releasing' + ) continue + for (const agent of claim.lifecycle.agents) { + const invocationId = agent.tracked.spec.invocationId + const node = agent.tracked.result?.node + if (invocationId || node) agents.push({ name: agent.name, invocationId, node }) + } + } + // Migration fallback for registries written before durable lifecycle + // records existed. It preserves observation, but only new lifecycle rows + // carry enough decision/spec state to process the reconciled exit. + if (agents.length === 0 && !hasNonterminalDurableLifecycle) { + const registry = await readFactoryInFlightRegistry(this.#config.loop.registryPath) + agents.push(...(registry?.agents ?? []) + .filter((agent) => agent.invocationId || agent.node) + .map((agent) => ({ name: agent.name, invocationId: agent.invocationId, node: agent.node }))) + } + if (agents.length > 0 && this.#fleet.hydrateTracked) { + this.#fleet.hydrateTracked(agents) + } + this.#scheduleDispatchLifecycleRenewal() + if (this.#fleet.hydrateTracked) await this.#fleet.reconcileTrackedAgents?.() + } catch (error) { + this.#logger.warn?.('[factory] failed to re-adopt durable in-flight agents', { error }) + } + } + + #scheduleDispatchLifecycleRenewal(): void { + if (this.#dispatchLifecycleRenewTimer || this.#dispatchLifecycleEpochs.size === 0) return + this.#dispatchLifecycleRenewTimer = setInterval(() => { + void this.#renewDispatchLifecycles() + }, DISPATCH_LIFECYCLE_RENEW_MS) + this.#dispatchLifecycleRenewTimer.unref?.() + } + + async #renewDispatchLifecycles(): Promise { + for (const [key, epoch] of [...this.#dispatchLifecycleEpochs]) { + const renewed = await this.#state.renewDispatchLifecycle( + this.#workspaceId, + key, + this.#dispatchLifecycleOwner, + epoch, + this.#clock.now(), + DISPATCH_LIFECYCLE_LEASE_MS, + ) + if (!renewed) { + this.#dispatchLifecycleEpochs.delete(key) + this.#increment('dispatchLifecycleLeasesLost') + const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key) + if (lifecycle && !isTerminalDispatchLifecycle(lifecycle) && lifecycle.phase !== 'waiting-for-human') { + this.#scheduleDispatchLifecycleRetry(inFlightRecordFromLifecycle(lifecycle)) + } + } + } + if (this.#dispatchLifecycleEpochs.size === 0 && this.#dispatchLifecycleRenewTimer) { + clearInterval(this.#dispatchLifecycleRenewTimer) + this.#dispatchLifecycleRenewTimer = undefined + } + } + + async #claimDispatchLifecycle(decision: TriageDecision, dryRun: boolean): Promise<{ created: boolean; lifecycle: DispatchLifecycle }> { + const key = issueKey(decision.issue) + const seed: DispatchLifecycle = { + runId: randomUUID(), + issue: { ...decision.issue }, + decision: structuredClone(decision), + dryRun, + phase: 'dispatching', + agents: [], + invocationIds: [], + updatedAtMs: this.#clock.now(), + } + seed.decision = decisionWithLifecycleBranches(seed.decision, seed.runId) + const claim = await this.#state.claimDispatchLifecycle( + this.#workspaceId, + key, + seed, + this.#dispatchLifecycleOwner, + this.#clock.now(), + DISPATCH_LIFECYCLE_LEASE_MS, + ) + if (!claim.acquired || !claim.lease) { + const reason = isTerminalDispatchLifecycle(claim.lifecycle) + ? 'dispatch lifecycle is already terminal' + : `dispatch lifecycle is owned by ${claim.lifecycle.lease?.owner ?? 'another publisher'}` + throw new Error(`Refusing to dispatch ${decision.issue.key}: ${reason}`) + } + this.#dispatchLifecycleEpochs.set(key, claim.lease.epoch) + this.#scheduleDispatchLifecycleRenewal() + return { created: claim.created, lifecycle: claim.lifecycle } + } + + async #saveDispatchLifecycle( + record: InFlightIssue, + phase: DispatchLifecyclePhase, + pullRequest?: GithubPublishPullRequestResult, + releaseReason?: string, + releasedAgentNames: ReadonlySet = new Set(), + ): Promise { + if (record.dryRun || this.#fleet.placementLocality !== 'remote') return true + const key = issueKey(record.issue) + const epoch = this.#dispatchLifecycleEpochs.get(key) + if (epoch === undefined) { + this.#scheduleDispatchLifecycleRetry(record) + return false + } + const previous = await this.#state.getDispatchLifecycle(this.#workspaceId, key) + const lifecycle = lifecycleFromInFlightRecord( + record, + previous?.runId ?? randomUUID(), + phase, + this.#clock.now(), + pullRequest ?? previous?.pullRequest, + releaseReason ?? previous?.releaseReason, + ) + for (const agent of lifecycle.agents) { + const previouslyReleasedAtMs = previous?.agents.find((candidate) => candidate.name === agent.name)?.releasedAtMs + if (previouslyReleasedAtMs !== undefined) agent.releasedAtMs = previouslyReleasedAtMs + if (releasedAgentNames.has(agent.name)) agent.releasedAtMs ??= this.#clock.now() + } + const saved = await this.#state.saveDispatchLifecycle( + this.#workspaceId, + key, + this.#dispatchLifecycleOwner, + epoch, + this.#clock.now(), + lifecycle, + ) + if (!saved) { + this.#dispatchLifecycleEpochs.delete(key) + this.#increment('dispatchLifecycleFencesRejected') + this.#scheduleDispatchLifecycleRetry(record) + return false + } + if (isTerminalDispatchLifecycle(lifecycle)) { + this.#dispatchLifecycleEpochs.delete(key) + } + return true + } + + #resolveDispatchTerminalWaiters(issue: IssueRef): void { + const key = issueKey(issue) + for (const resolve of this.#dispatchTerminalWaiters.get(key) ?? []) resolve() + this.#dispatchTerminalWaiters.delete(key) + } + + #scheduleDispatchLifecycleRetry(record: InFlightIssue): void { + const key = issueKey(record.issue) + if (this.#stopping || this.#dispatchLifecycleRetryTimers.has(key)) return + const timer = setTimeout(() => { + this.#dispatchLifecycleRetryTimers.delete(key) + const drive = this.#driveDispatchLifecycle(key) + .catch((error) => { + this.#logger.warn?.('[factory] durable dispatch lifecycle retry failed', { + issue: record.issue.key, + error: describeError(error).errorMessage, + }) + this.#scheduleDispatchLifecycleRetry(record) + }) + .finally(() => this.#dispatchLifecycleDrives.delete(drive)) + this.#dispatchLifecycleDrives.add(drive) + }, DISPATCH_LIFECYCLE_RETRY_MS) + this.#dispatchLifecycleRetryTimers.set(key, timer) + } + + async #driveDispatchLifecycle(key: string): Promise { + if (this.#stopping) return + let lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, key) + if (!lifecycle) return + if (isTerminalDispatchLifecycle(lifecycle)) { + this.#resolveDispatchTerminalWaiters(lifecycle.issue) + return + } + if (lifecycle.phase === 'waiting-for-human') return + let acquiredNow = false + if (!this.#dispatchLifecycleEpochs.has(key)) { + const claim = await this.#state.claimDispatchLifecycle( + this.#workspaceId, + key, + lifecycle, + this.#dispatchLifecycleOwner, + this.#clock.now(), + DISPATCH_LIFECYCLE_LEASE_MS, + ) + if (!claim.acquired || !claim.lease) { + throw new Error(`durable dispatch ${lifecycle.issue.key} is still owned by another publisher`) + } + this.#dispatchLifecycleEpochs.set(key, claim.lease.epoch) + this.#scheduleDispatchLifecycleRenewal() + lifecycle = claim.lifecycle + acquiredNow = true + } + if (lifecycle.phase === 'queued') { + const epoch = this.#dispatchLifecycleEpochs.get(key) + if (epoch === undefined || !await this.#state.promoteDispatchLifecycle( + this.#workspaceId, + key, + this.#dispatchLifecycleOwner, + epoch, + this.#clock.now(), + )) { + throw new Error(`durable dispatch ${lifecycle.issue.key} is waiting for batch capacity`) + } + const promoted = await this.#state.getDispatchLifecycle(this.#workspaceId, key) + if (!promoted || promoted.phase !== 'dispatching') { + throw new Error(`durable dispatch ${lifecycle.issue.key} lost its promoted lifecycle`) + } + lifecycle = promoted + } + const batch = await this.#batch() + const durableRecord = inFlightRecordFromLifecycle(lifecycle) + const record = lifecycle.phase === 'releasing' ? durableRecord : batch.restore(durableRecord) + if (!await this.#assertDispatchLifecycleOwner(record)) return + if (acquiredNow && this.#config.babysitter.enabled) await this.#restoreBabysitterOwnership() + + if (acquiredNow && lifecycle.phase === 'running') { + if (this.#fleet.hydrateTracked) { + this.#fleet.hydrateTracked(lifecycle.agents.map((agent) => ({ name: agent.name, - invocationId: agent.invocationId, - node: agent.node, + invocationId: agent.tracked.spec.invocationId, + node: agent.tracked.result?.node, }))) + await this.#fleet.reconcileTrackedAgents?.() } - await this.#fleet.reconcileTrackedAgents?.() - } catch (error) { - this.#logger.warn?.('[factory] failed to re-adopt in-flight agents from the registry', { error }) + return + } + + if (lifecycle.phase === 'parking') { + const waiting = await this.#state.getWaitingClarification(this.#workspaceId, key) + if (!waiting) { + throw new Error(`durable dispatch ${record.issue.key} has no clarification to finish parking`) + } + await this.#finishClarificationPark(waiting, true) + return + } + if (lifecycle.phase === 'dispatching' || lifecycle.phase === 'retryable') { + await this.#resumeDurableDispatch(record) + return + } + if (lifecycle.phase === 'publishing') { + const implementer = [...record.agents.values()].find((agent) => agent.spec.role === 'implementer') + if (!implementer) throw new Error(`durable dispatch ${record.issue.key} has no implementer to publish`) + const published = await this.#publishImplementerPullRequest(record, implementer) + if (!published) throw new Error(`durable dispatch ${record.issue.key} did not produce a pull request`) + if (!await this.#saveDispatchLifecycle(record, 'published', published)) return + if (this.#config.babysitter.enabled) { + await this.#ensureBabysitter(record, { + repo: published.repo, + prNumber: published.number, + url: published.url, + }) + return + } + await this.#completeIssue(record) + return + } + if (lifecycle.phase === 'published' && this.#config.babysitter.enabled && lifecycle.pullRequest) { + await this.#ensureBabysitter(record, { + repo: lifecycle.pullRequest.repo, + prNumber: lifecycle.pullRequest.number, + url: lifecycle.pullRequest.url, + }) + return + } + if (lifecycle.phase === 'published' || lifecycle.phase === 'writeback-applied') { + await this.#completeIssue(record) + return + } + if (lifecycle.phase === 'releasing') { + await this.#finishDurableRelease(record, lifecycle.releaseReason) + } + } + + async #resumeDurableDispatch(record: InFlightIssue): Promise { + const hadResult = Boolean(record.result) + const agents: DispatchResult['agents'] = [] + const specs = dispatchSpecs(record.decision) + const plannedNames = new Set(specs.map((spec) => spec.name)) + for (const tracked of record.agents.values()) { + if (plannedNames.has(tracked.spec.name)) continue + plannedNames.add(tracked.spec.name) + specs.push(tracked.spec) + } + for (const spec of specs) { + const spawned = await this.#spawnAgent(record, spec, record.dryRun) + agents.push({ name: spawned.name, role: spec.role }) + } + await this.#writeInFlightRegistry() + if (!record.dryRun) { + const issue = await this.#readIssue(record.issue.path) + if (!issue) throw new Error(`Unable to recover durable dispatch ${record.issue.key}: issue is no longer readable`) + if (isGithubIssue(issue)) { + await this.#githubWriteback.setStatus(issue, 'in-progress') + } else { + await this.#linear.setState(issue, this.#states.idFor(issue.team, 'agentImplementing')) + } + } + record.result ??= { + issue: record.issue, + agents, + comments: [dispatchComment(record.decision, agents)], + dryRun: record.dryRun, + } + if (!await this.#saveDispatchLifecycle(record, 'running')) return + if (!record.dryRun) { + if (!hadResult) { + await this.#sendImplementerTask(record) + await this.#sendCriticalReviewerMessage(record) + } + for (const tracked of record.agents.values()) { + const owned = tracked.spec.ownedPullRequest + if (tracked.spec.role !== 'babysitter' || !owned) continue + await this.#ensureBabysitter(record, { + repo: owned.repo, + prNumber: owned.number, + path: owned.path, + }) + } + } + } + + async #finishDurableRelease(record: InFlightIssue, releaseReason?: string): Promise { + const batch = await this.#batch() + const next = this.#fleet.placementLocality === 'remote' ? undefined : batch.complete(record.issue) + const reason = releaseReason ?? (this.#config.terminalState === 'human-review' ? 'issue-human-review' : 'issue-done') + const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)) + const released = new Set(lifecycle?.agents + .filter((agent) => agent.releasedAtMs !== undefined) + .map((agent) => agent.name) ?? []) + const failed: string[] = [] + for (const agent of record.agents) { + if (released.has(agent[0])) continue + const releaseFailed = await this.#releaseAndTerminateAgents([agent], reason, 'completion') + if (releaseFailed.length > 0) { + failed.push(...releaseFailed) + continue + } + released.add(agent[0]) + // Persist each acknowledged release independently. A takeover retries + // only agents whose release did not reach a fenced durable checkpoint. + if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, reason, released)) return false + } + if (next) await this.dispatch(next.decision, { dryRun: next.dryRun }) + await this.#writeInFlightRegistry() + if (failed.length > 0) { + this.#increment('dispatchLifecycleReleaseRetries') + this.#scheduleDispatchLifecycleRetry(record) + return false + } + // Terminal lifecycle saves intentionally relinquish the owner epoch. Clear + // the babysitter's durable ownership/wake/critical state while that epoch + // is still valid so a later reopened issue cannot inherit a stale PR owner. + if (this.#fleet.placementLocality === 'remote' && this.#config.babysitter.enabled) { + await this.#cancelBabysitterWake(issueKey(record.issue)) } + if (!await this.#saveDispatchLifecycle(record, 'complete')) return false + this.#increment(releaseReason === 'issue-human-review' ? 'humanReview' : 'done') + this.#emit('issue-done', { issue: record.issue }) + await this.#writeInFlightRegistry() + this.#resolveDispatchTerminalWaiters(record.issue) + return true + } + + async #assertDispatchLifecycleOwner(record: InFlightIssue): Promise { + return await this.#assertIssueDispatchLifecycleOwner(record.issue) + } + + async #assertIssueDispatchLifecycleOwner(issue: IssueRef): Promise { + if (this.#fleet.placementLocality !== 'remote') return true + const key = issueKey(issue) + const epoch = this.#dispatchLifecycleEpochs.get(key) + if (epoch === undefined) return false + const renewed = await this.#state.renewDispatchLifecycle( + this.#workspaceId, + key, + this.#dispatchLifecycleOwner, + epoch, + this.#clock.now(), + DISPATCH_LIFECYCLE_LEASE_MS, + ) + if (!renewed) { + this.#dispatchLifecycleEpochs.delete(key) + this.#increment('dispatchLifecycleFencesRejected') + } + return renewed } async #backfillReadyIssues(): Promise { @@ -2190,6 +2743,11 @@ export class FactoryLoop implements Factory { await this.#state.recordDispatchAttempt(this.#workspaceId, key, dispatchState) this.#increment('dispatchTerminalReopened') } + for (const [key, lifecycle] of await this.#state.listDispatchLifecycles(this.#workspaceId)) { + if (lifecycle.issue.key !== issue.key || !isTerminalDispatchLifecycle(lifecycle)) continue + await this.#state.clearDispatchLifecycle(this.#workspaceId, key) + this.#dispatchLifecycleEpochs.delete(key) + } } await this.#state.recordCanonicalState(this.#workspaceId, key, issue.stateId) } @@ -2363,12 +2921,15 @@ export class FactoryLoop implements Factory { } } - async #releaseInFlightAgents(reason: string): Promise { + async #releaseInFlightAgents(reason: string, opts: { preserveDurable?: boolean } = {}): Promise { const agents = new Map() for (const record of (await this.#batch()).inFlight) { if (record.dryRun) { continue } + if (opts.preserveDurable && [...record.agents.values()].some((tracked) => tracked.result?.locality === 'remote')) { + continue + } for (const [agentName, tracked] of record.agents) { agents.set(agentName, tracked) } @@ -2382,7 +2943,8 @@ export class FactoryLoop implements Factory { agents: Array<[string, TrackedAgent]>, reason: string, context: 'stop' | 'completion' | 'clarification', - ): Promise { + ): Promise { + const failed: string[] = [] const protectedPids = await this.#protectedPids() for (const [agentName, tracked] of agents) { if (context === 'stop') { @@ -2419,15 +2981,23 @@ export class FactoryLoop implements Factory { try { await this.#fleet.release(agentName, reason) } catch (error) { + failed.push(agentName) this.#logger.warn?.(`[factory] failed to release ${agentName} during ${context}`, error) } if (context === 'stop') { await this.#refreshStoppingHeartbeat() } } + return failed } async #terminationRoots(agentName: string, tracked: TrackedAgent, protectedPids: number[] = []): Promise { + // Relay placement PIDs belong to the recorded node, never this + // orchestrator. Release through the control plane; do not signal a + // coincidentally reused local PID. + if (tracked.result?.locality === 'remote') { + return { pids: [], status: 'missing' } + } const pids = pidsFromSpawnResult(tracked.result) if (!this.#fleet.resolveAgentPid) { return pids.length > 0 ? { pids, status: 'found' } : { pids: [], status: 'unresolved' } @@ -2571,7 +3141,7 @@ export class FactoryLoop implements Factory { const batch = await this.#batch() const invocationId = batch.invocationIdFor(record.issue, spec) const existing = record.agents.get(spec.name) - if (existing) { + if (existing?.result) { return { name: existing.result?.name ?? spec.name } } @@ -2584,14 +3154,32 @@ export class FactoryLoop implements Factory { return { name: spec.name } } + // Persist intent before the remote side effect. If the owner crashes after + // the spawn ack but before recording its result, takeover retries the same + // deterministic invocation id instead of inventing a second worker. + batch.recordPlanned(record, { ...spec, invocationId }) + if (!await this.#saveDispatchLifecycle(record, 'dispatching')) { + throw new Error(`Dispatch lifecycle ownership lost before spawning ${spec.name}`) + } + let roster try { roster = await retryOnTimeout(() => this.#fleet.roster(), { attempts: 3, delayMs: 2000 }) } catch (error) { throw contextualError(`Dispatch roster lookup failed for ${record.issue.key}`, error) } - if (roster.agents.some((agent) => agent.name === spec.name)) { - batch.recordSpawn(record, spec, invocationId, { name: spec.name, sessionRef: spec.sessionRef }) + const rosterAgent = roster.agents.find((agent) => agent.name === spec.name) + if (rosterAgent) { + const trackedPlacement = this.#fleet.trackedAgents?.().get(spec.name) + batch.recordSpawn(record, spec, invocationId, { + name: spec.name, + sessionRef: existing?.sessionRef ?? spec.sessionRef, + node: existing?.result?.node ?? trackedPlacement?.node ?? rosterAgent.node, + locality: existing?.result?.locality ?? this.#fleet.placementLocality, + }) + if (!await this.#saveDispatchLifecycle(record, 'dispatching')) { + throw new Error(`Dispatch lifecycle ownership lost after adopting ${spec.name}`) + } return { name: spec.name } } @@ -2619,6 +3207,9 @@ export class FactoryLoop implements Factory { ) } batch.recordSpawn(record, spec, invocationId, result) + if (!await this.#saveDispatchLifecycle(record, 'dispatching')) { + throw new Error(`Dispatch lifecycle ownership lost after spawning ${spec.name}`) + } return { name: result.name } } @@ -2642,10 +3233,30 @@ export class FactoryLoop implements Factory { if (!record) { return } + if (!await this.#assertDispatchLifecycleOwner(record)) { + this.#logger.warn?.('[factory] ignored agent exit after durable lifecycle ownership was lost', { + issue: record.issue.key, + name, + }) + return + } const exiting = record.agents.get(name) + if (this.#fleet.placementLocality === 'remote') { + const lifecycle = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)) + if (lifecycle?.phase === 'parking') { + this.#increment('clarificationParkingExitsSuppressed') + return + } + } + if (isCompletionReason(reason)) { + if (exiting?.spec.role === 'implementer' && await this.#issueHasCompletionPr(record)) { + if (this.#config.babysitter.enabled) await this.#ensureBabysitterForIssue(record) + else await this.#completeIssue(record) + return + } let publishedPr: GithubPublishPullRequestResult | undefined if ( exiting?.spec.role === 'implementer' && @@ -2653,10 +3264,13 @@ export class FactoryLoop implements Factory { (this.#mount.githubWrite || this.#mount.writebackTransport === 'relayfile-cloud') ) { try { + await this.#saveDispatchLifecycle(record, 'publishing') publishedPr = await this.#publishImplementerPullRequest(record, exiting) + if (publishedPr) await this.#saveDispatchLifecycle(record, 'published', publishedPr) } catch (error) { this.#increment('githubPullRequestPublishFailures') this.#error(error, record.issue) + this.#scheduleDispatchLifecycleRetry(record) return } } @@ -2706,8 +3320,10 @@ export class FactoryLoop implements Factory { // / human-review path. Best-effort: with no publishable branch (no commits // ahead of base, clone gone) it returns undefined and we fall through. if (tracked.spec.role === 'implementer') { + await this.#saveDispatchLifecycle(record, 'publishing') const publishedPr = await this.#tryPublishImplementerPr(record, tracked) if (publishedPr) { + await this.#saveDispatchLifecycle(record, 'published', publishedPr) if (this.#config.babysitter.enabled) { await this.#ensureBabysitter(record, { repo: publishedPr.repo, @@ -2719,6 +3335,10 @@ export class FactoryLoop implements Factory { } return } + if (tracked.result?.locality === 'remote' && tracked.spec.branch) { + this.#scheduleDispatchLifecycleRetry(record) + return + } } if (tracked.sessionRef) { @@ -2780,7 +3400,7 @@ export class FactoryLoop implements Factory { const result = await this.#fleet.spawn({ name: tracked.spec.name, capability: tracked.spec.capability, - node: tracked.spec.node ?? 'self', + node: tracked.result?.node ?? tracked.spec.node ?? 'self', repo: tracked.spec.repo, task: tracked.spec.task, model: tracked.spec.model, @@ -2826,7 +3446,7 @@ export class FactoryLoop implements Factory { record: InFlightIssue, implementer: TrackedAgent, ): Promise { - if (record.dryRun || !implementer.spec.clonePath || !this.#mount.githubWrite) { + if (record.dryRun || (!implementer.spec.clonePath && !implementer.spec.branch) || !this.#mount.githubWrite) { return undefined } try { @@ -2856,14 +3476,20 @@ export class FactoryLoop implements Factory { implementer: TrackedAgent, ): Promise { const key = `${issueKey(record.issue)}:${implementer.spec.repo}` - if (this.#publishedPullRequests.has(key)) return undefined + const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)) + if (durable?.pullRequest) return durable.pullRequest + const cached = this.#publishedPullRequests.get(key) + if (cached) return cached const githubWrite = this.#mount.githubWrite if (!githubWrite) { throw new Error('GitHub write path not available on this mount — connect GitHub to your workspace') } - if (!implementer.spec.clonePath) { - throw new Error(`GitHub PR publication requires a configured clone path for ${implementer.spec.repo}`) + const remoteBranch = implementer.result?.locality === 'remote' && implementer.spec.branch + ? implementer.spec.branch + : undefined + if (!remoteBranch && !implementer.spec.clonePath) { + throw new Error(`GitHub PR publication requires a pushed branch or configured clone path for ${implementer.spec.repo}`) } const issue = await this.#readIssue(record.issue.path) if (!issue) { @@ -2880,12 +3506,24 @@ export class FactoryLoop implements Factory { const baseRef = await this.#githubDefaultBranch(repo) const result = await githubWrite.publishPullRequest({ repo, - clonePath: implementer.spec.clonePath, + ...(remoteBranch ? { headRef: remoteBranch } : { clonePath: implementer.spec.clonePath }), baseRef, title: `${issue.key}: ${issue.title}`, body: githubPullRequestBody(issue), }) - this.#publishedPullRequests.add(key) + if ( + result.repo.toLowerCase() !== repo.toLowerCase() || + result.headRef !== (remoteBranch ?? result.headRef) || + !Number.isInteger(result.number) || + result.number <= 0 || + !result.url + ) { + throw new Error(`GitHub PR publication returned an unexpected receipt for ${repo}/${remoteBranch ?? 'local HEAD'}`) + } + if (remoteBranch && this.#mount.writebackTransport === 'relayfile-cloud') { + await this.#confirmPublishedRemotePullRequest(repo, result, remoteBranch) + } + this.#publishedPullRequests.set(key, result) this.#increment('githubPullRequestsPublished') this.#logger.info?.('[factory] published PR through workspace GitHub connection', { issue: issue.key, @@ -2896,6 +3534,58 @@ export class FactoryLoop implements Factory { return result } + async #confirmPublishedRemotePullRequest( + repo: string, + result: GithubPublishPullRequestResult, + expectedHeadRef: string, + ): Promise { + const parts = githubRepoParts(repo) + if (!parts) throw new Error(`GitHub repo must be owner/repo before confirming its pull request: ${repo}`) + const roots = [ + `/github/repos/${encodeURIComponent(parts.owner)}/${encodeURIComponent(parts.repo)}/pulls/`, + `/github/repos/${encodeURIComponent(parts.owner)}__${encodeURIComponent(parts.repo)}/pulls/`, + ] + let lastObserved = 'pull request metadata was not mounted' + for (let attempt = 0; attempt < PUBLISHED_PR_CONFIRM_ATTEMPTS; attempt += 1) { + const paths = (await Promise.all(roots.map(async (root) => { + try { + return await this.#mount.listTree(root) + } catch { + return [] + } + }))).flat() + for (const path of paths) { + const pathParts = githubPullPathParts(path) + if ( + !pathParts || + pathParts.number !== result.number || + pathParts.owner.toLowerCase() !== parts.owner.toLowerCase() || + pathParts.repo.toLowerCase() !== parts.repo.toLowerCase() + ) continue + try { + const snapshot = parsePullSnapshot((await this.#mount.readFile(path)).content, result.number) + if (!snapshot) { + lastObserved = `invalid metadata at ${path}` + continue + } + const state = snapshot.state?.trim().toUpperCase() + lastObserved = `head=${snapshot.headRef ?? 'unknown'} state=${state ?? 'unknown'} draft=${String(snapshot.draft)}` + if (snapshot.headRef === expectedHeadRef && state === 'OPEN' && snapshot.draft === false && snapshot.merged !== true) { + return + } + } catch (error) { + lastObserved = `${path}: ${describeError(error).errorMessage}` + } + } + if (attempt < PUBLISHED_PR_CONFIRM_ATTEMPTS - 1) { + await this.#clock.sleep(PUBLISHED_PR_CONFIRM_DELAY_MS) + } + } + throw new Error( + `Published GitHub PR ${repo}#${result.number} was not confirmed open, non-draft, and on ${expectedHeadRef}: ${lastObserved}`, + ) + } + async #githubDefaultBranch(repo: string): Promise { const parts = githubRepoParts(repo) if (!parts) { @@ -3053,10 +3743,16 @@ export class FactoryLoop implements Factory { const result = await this.#fleet.resume({ name, sessionRef: tracked.sessionRef, - node: tracked.spec.node ?? 'self', + node: tracked.result?.node ?? tracked.spec.node ?? 'self', capability: tracked.spec.capability, + repo: tracked.spec.repo, + clonePath: tracked.spec.clonePath, }) - tracked.result = result + tracked.result = { + ...result, + node: result.node ?? tracked.result?.node, + locality: result.locality ?? tracked.result?.locality, + } tracked.sessionRef = result.sessionRef ?? tracked.sessionRef record.agents.delete(name) record.agents.set(result.name, tracked) @@ -3122,6 +3818,11 @@ export class FactoryLoop implements Factory { this.#increment('babysitterCriticalSignalsIgnored') return } + if (!await this.#assertIssueDispatchLifecycleOwner(durableIssue)) { + this.#babysitterCriticalAgents.delete(babysitterCritical.agentName) + this.#increment('babysitterCriticalSignalsIgnoredNonOwner') + return + } if (babysitterCritical.action === 'begin') { // Durably install the fence before acknowledging it. A process crash // after the ACK can therefore restore both the exact owner and the @@ -3461,21 +4162,27 @@ export class FactoryLoop implements Factory { async #finishClarificationPark(waiting: WaitingClarification, recovered: boolean): Promise { const key = issueKey(waiting.issue) + const liveRecord = (await this.#batch()).getIssue(waiting.issue) + if (liveRecord && !await this.#saveDispatchLifecycle(liveRecord, 'parking')) { + throw new Error(`dispatch lifecycle ownership lost while parking ${waiting.issue.key}`) + } for (const { name } of waiting.agents) { this.#fleet.markAgentTerminal?.(name, 'waiting-for-human') } await this.#releaseAgentsForClarification(key, waiting.agents.map(({ name, tracked }) => [name, tracked])) - const batch = await this.#batch() - const next = batch.complete(waiting.issue) // parkedAtMs is the durable wake gate. It is written only after roster // confirmation proves every saved team member has relinquished its slot // and after the old local batch record can no longer race the wake. const parked = await this.#state.markClarificationParked(this.#workspaceId, key, this.#clock.now()) if (!parked) { - if (next) batch.queue(next.decision, next.dryRun) throw new Error(`durable clarification park refused for ${waiting.issue.key}`) } + if (liveRecord && !await this.#saveDispatchLifecycle(liveRecord, 'waiting-for-human')) { + throw new Error(`dispatch lifecycle ownership lost after parking ${waiting.issue.key}`) + } + const batch = await this.#batch() + const next = batch.complete(waiting.issue) await this.#clearDispatchInFlight(waiting.issue) await this.#writeInFlightRegistry() for (const { name } of waiting.agents) this.#clarificationIntents.delete(name) @@ -4025,6 +4732,7 @@ export class FactoryLoop implements Factory { slackDispatchThread: await this.#slackDispatchThreadFor(record), integrationsMountRoot: this.#integrationsMountRoot(), integrationInstructions, + branchName: implementer.spec.branch, }), from: 'factory', data: { issue: record.issue }, @@ -4112,6 +4820,10 @@ export class FactoryLoop implements Factory { this.#increment('babysitterOwnershipRestoreInvalid') continue } + if (!await this.#assertIssueDispatchLifecycleOwner(session.issue)) { + this.#increment('babysitterOwnershipRestoreSkippedNonOwner') + continue + } const record = batch.getIssue(session.issue) const tracked = record?.agents.get(session.agentName) ?? [...(record?.agents.values() ?? [])].find((agent) => agent.spec.role === 'babysitter') @@ -4152,6 +4864,9 @@ export class FactoryLoop implements Factory { } async #cancelBabysitterWake(issueIdentity: string): Promise { + const issue = this.#babysitterIssueRefs.get(issueIdentity) + const mayClearDurable = this.#fleet.placementLocality !== 'remote' + || Boolean(issue && await this.#assertIssueDispatchLifecycleOwner(issue)) for (const [key, state] of this.#babysitterWakeStates) { if (issueKey(state.issue) !== issueIdentity) continue state.cancelled = true @@ -4163,7 +4878,7 @@ export class FactoryLoop implements Factory { this.#babysitterPr.delete(issueIdentity) this.#babysitterIssueRefs.delete(issueIdentity) this.#babysitterSpawned.delete(issueIdentity) - await this.#state.clearBabysitterSession(this.#workspaceId, issueIdentity) + if (mayClearDurable) await this.#state.clearBabysitterSession(this.#workspaceId, issueIdentity) } async #routeBabysitterEvent(path: string, extraKinds: Iterable = []): Promise { @@ -4197,6 +4912,10 @@ export class FactoryLoop implements Factory { this.#logger.debug?.('[factory] ignored unowned PR event for babysitter routing', { ...event, prNumber: target.prNumber }) continue } + if (!await this.#assertIssueDispatchLifecycleOwner(owner.issue)) { + this.#increment('babysitterEventsIgnoredNonOwner') + continue + } const kinds = new Set([...target.kinds, ...extraKinds]) await this.#queueBabysitterWake(owner.issue, owner.ref, kinds, owner.tracked) } @@ -4258,6 +4977,10 @@ export class FactoryLoop implements Factory { kinds: Iterable, tracked: TrackedAgent, ): Promise { + if (!await this.#assertIssueDispatchLifecycleOwner(issue)) { + this.#increment('babysitterEventsIgnoredNonOwner') + return + } // Owner lookup and queueing straddle async mount/state reads. Revalidate // the exact composite owner so a concurrent close/merge cancellation can // never recreate durable state from a stale child event. @@ -4331,6 +5054,9 @@ export class FactoryLoop implements Factory { ref: BabysitterPrRef, tracked?: TrackedAgent, ): Promise { + if (!await this.#assertIssueDispatchLifecycleOwner(issue)) { + throw new Error(`Babysitter lifecycle ownership lost for ${issue.key}`) + } const pending = tracked?.spec.pendingPullRequestWake await this.#state.setBabysitterSession(this.#workspaceId, issueKey(issue), { issue: { ...issue }, @@ -4369,6 +5095,16 @@ export class FactoryLoop implements Factory { async #flushBabysitterWake(state: BabysitterWakeState): Promise { if (this.#stopping || state.cancelled || state.kinds.size === 0) return + if (!await this.#assertIssueDispatchLifecycleOwner(state.issue)) { + state.cancelled = true + this.#babysitterWakeStates.delete(babysitterWakeKey(state.issue, { + repo: state.repo, + prNumber: state.prNumber, + agentName: state.agentName, + })) + this.#increment('babysitterEventWakesCancelledNonOwner') + return + } if (this.#babysitterCriticalAgents.has(state.agentName)) { this.#increment('babysitterEventWakesDeferredCritical') return @@ -4746,6 +5482,10 @@ export class FactoryLoop implements Factory { async #ensureBabysitter(record: InFlightIssue, prRef: { repo: string; prNumber: number; url?: string; path?: string }): Promise { const babysitterKey = issueKey(record.issue) + if (!await this.#assertIssueDispatchLifecycleOwner(record.issue)) { + this.#increment('babysitterLifecycleOwnershipRejected') + return + } this.#babysitterIssueRefs.set(babysitterKey, { ...record.issue }) const existing = this.#babysitterPr.get(babysitterKey) if (existing && githubPrIdentity(existing.repo, existing.prNumber) !== githubPrIdentity(prRef.repo, prRef.prNumber)) { @@ -4839,6 +5579,7 @@ export class FactoryLoop implements Factory { }) await this.#persistBabysitterSession(record.issue, this.#babysitterPr.get(babysitterKey)!, tracked) await this.#writeInFlightRegistry() + if (!await this.#saveDispatchLifecycle(record, 'running')) return this.#increment('babysittersSpawned') this.#logger.info?.('[factory] babysitter spawned for open PR', { issue: record.issue.key, @@ -4862,7 +5603,9 @@ export class FactoryLoop implements Factory { this.#babysitterSpawned.delete(babysitterKey) this.#babysitterPr.delete(babysitterKey) this.#babysitterIssueRefs.delete(babysitterKey) - await this.#state.clearBabysitterSession(this.#workspaceId, babysitterKey) + if (await this.#assertIssueDispatchLifecycleOwner(record.issue)) { + await this.#state.clearBabysitterSession(this.#workspaceId, babysitterKey) + } this.#increment('babysitterSpawnFailures') this.#error(error, record.issue) } finally { @@ -4995,6 +5738,7 @@ export class FactoryLoop implements Factory { } this.#completionInFlight.add(completionKey) try { + if (!await this.#assertDispatchLifecycleOwner(record)) return const issue = await this.#readIssue(record.issue.path) // Resolve the terminal state for the issue's own team. Land in // `human-review` only when the operator opted into that terminal state AND @@ -5046,6 +5790,7 @@ export class FactoryLoop implements Factory { } this.#emit('writeback-verified', { issue: record.issue, path: issue.path }) } + if (!await this.#saveDispatchLifecycle(record, 'writeback-applied')) return if (this.#slack && this.#config.slack && !await this.#shouldSkipSlackWriteback('completion-thread')) { try { @@ -5079,21 +5824,22 @@ export class FactoryLoop implements Factory { await this.#runCompletionMergeGate(issue) } - await this.#releaseAndTerminateAgents([...record.agents], humanReview ? 'issue-human-review' : 'issue-done', 'completion') - - this.#increment(humanReview ? 'humanReview' : 'done') - this.#emit('issue-done', { issue: record.issue }) + const releaseReason = humanReview ? 'issue-human-review' : 'issue-done' + if (this.#fleet.placementLocality === 'remote') { + // Durable capacity is released as soon as terminal writeback is + // acknowledged. Agent cleanup remains fenced/retryable in `releasing`. + const batch = await this.#batch() + batch.complete(record.issue) + } + if (!await this.#saveDispatchLifecycle(record, 'releasing', undefined, releaseReason)) return await this.#stopSlackWatcher(record.issue) await this.#stopGithubIssueCommentWatcherForIssue(record.issue) await this.#recordDispatchTerminal(record.issue) - const next = (await this.#batch()).complete(record.issue) + await this.#finishDurableRelease(record, releaseReason) await this.#drainReadyClarificationWake() - if (next) { - await this.dispatch(next.decision, { dryRun: next.dryRun }) - } - await this.#writeInFlightRegistry() } catch (error) { this.#error(error, record.issue) + this.#scheduleDispatchLifecycleRetry(record) } finally { this.#completionInFlight.delete(completionKey) const stateKey = issueStateKey(record.issue) @@ -5102,8 +5848,11 @@ export class FactoryLoop implements Factory { this.#babysitterSpawned.delete(completionKey) this.#babysitterPr.delete(completionKey) await this.#cancelBabysitterWake(completionKey) - for (const publishedKey of this.#publishedPullRequests) { - if (publishedKey.startsWith(`${completionKey}:`)) this.#publishedPullRequests.delete(publishedKey) + const durable = await this.#state.getDispatchLifecycle(this.#workspaceId, issueKey(record.issue)).catch(() => undefined) + if (this.#fleet.placementLocality !== 'remote' || (durable && isTerminalDispatchLifecycle(durable))) { + for (const publishedKey of this.#publishedPullRequests.keys()) { + if (publishedKey.startsWith(`${completionKey}:`)) this.#publishedPullRequests.delete(publishedKey) + } } } } @@ -6057,17 +6806,57 @@ export class FactoryLoop implements Factory { return } - const record = batch.start(waiting.decision, waiting.dryRun) + let lifecycleDecision = waiting.decision + let promotedLifecycle: DispatchLifecycle | undefined + if (!waiting.dryRun && this.#fleet.placementLocality === 'remote') { + try { + const claim = await this.#claimDispatchLifecycle(waiting.decision, false) + const lifecycleKey = issueKey(waiting.issue) + const epoch = this.#dispatchLifecycleEpochs.get(lifecycleKey) + if ( + claim.lifecycle.phase === 'waiting-for-human' && + (epoch === undefined || !await this.#state.promoteDispatchLifecycle( + this.#workspaceId, + lifecycleKey, + this.#dispatchLifecycleOwner, + epoch, + this.#clock.now(), + )) + ) { + await this.#state.releaseClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner) + this.#increment('clarificationWakesQueuedForCapacity') + this.#scheduleClarificationWakeRetry(key) + return + } + const promoted = await this.#state.getDispatchLifecycle(this.#workspaceId, lifecycleKey) + promotedLifecycle = promoted + lifecycleDecision = promoted?.decision ?? claim.lifecycle.decision + } catch { + await this.#state.releaseClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner) + this.#increment('clarificationWakesQueuedForOwnership') + this.#scheduleClarificationWakeRetry(key) + return + } + } + const record = promotedLifecycle + ? batch.restore(inFlightRecordFromLifecycle(promotedLifecycle)) + : batch.start(lifecycleDecision, waiting.dryRun) if (!record) { await this.#state.releaseClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner) this.#increment('clarificationWakesQueuedForCapacity') return } + if (!await this.#saveDispatchLifecycle(record, 'dispatching')) { + batch.complete(waiting.issue) + await this.#state.releaseClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner) + this.#scheduleClarificationWakeRetry(key) + return + } const resumed: Array<[string, TrackedAgent]> = [] try { await renewLease() - const onlineNames = new Set((await this.#fleet.roster()).agents.map((agent) => agent.name)) + const onlineAgents = new Map((await this.#fleet.roster()).agents.map((agent) => [agent.name, agent])) this.#assertClarificationWakeRunning() for (const parked of waiting.agents) { this.#assertClarificationWakeRunning() @@ -6076,11 +6865,19 @@ export class FactoryLoop implements Factory { // A previous wake owner may have crashed after spawning but before // clearing the durable record. Adopt an already-online deterministic // name instead of duplicating the wake after the lease expires. - const result = onlineNames.has(parked.name) - ? { name: parked.name, sessionRef: tracked.sessionRef } + const online = onlineAgents.get(parked.name) + const result = online + ? { + ...tracked.result, + name: parked.name, + sessionRef: tracked.sessionRef ?? tracked.result?.sessionRef, + node: tracked.result?.node ?? online.node, + locality: tracked.result?.locality ?? this.#fleet.placementLocality, + } : await this.#resumeOrColdStartClarificationAgent(parked.name, tracked, waiting) const invocationId = batch.invocationIdFor(record.issue, tracked.spec) batch.recordSpawn(record, tracked.spec, invocationId, result) + await this.#saveDispatchLifecycle(record, 'dispatching') const live = record.agents.get(result.name) if (live) resumed.push([result.name, live]) this.#assertClarificationWakeRunning() @@ -6114,6 +6911,7 @@ export class FactoryLoop implements Factory { await renewLease() await this.#writeInFlightRegistry() + await this.#saveDispatchLifecycle(record, 'running') await renewLease() const completed = await this.#state.completeClarificationWake(this.#workspaceId, key, this.#clarificationWakeOwner) if (!completed) throw new ClarificationWakeLeaseLostError('clarification wake completion lost ownership') @@ -6228,15 +7026,22 @@ export class FactoryLoop implements Factory { name: string, tracked: TrackedAgent, waiting: WaitingClarification, - ): Promise<{ name: string; sessionRef?: string }> { + ): Promise { if (tracked.sessionRef) { try { - return await this.#fleet.resume({ + const resumed = await this.#fleet.resume({ name, sessionRef: tracked.sessionRef, - node: tracked.spec.node ?? 'self', + node: tracked.result?.node ?? tracked.spec.node ?? 'self', capability: tracked.spec.capability, + repo: tracked.spec.repo, + clonePath: tracked.spec.clonePath, }) + return { + ...resumed, + node: resumed.node ?? tracked.result?.node, + locality: resumed.locality ?? tracked.result?.locality ?? this.#fleet.placementLocality, + } } catch (error) { this.#assertClarificationWakeRunning() this.#increment('clarificationResumeFallbacks') @@ -6256,7 +7061,7 @@ export class FactoryLoop implements Factory { return await this.#fleet.spawn({ name, capability: tracked.spec.capability, - node: tracked.spec.node ?? 'self', + node: tracked.result?.node ?? tracked.spec.node ?? 'self', task: [ tracked.spec.task, '', @@ -7144,6 +7949,38 @@ function routeImplementerSpec( } } +function decisionWithLifecycleBranches(decision: TriageDecision, runId: string): TriageDecision { + const withBranch = (spec: AgentSpec): AgentSpec => { + const lifecycleSpec = { + ...spec, + // The same persisted lifecycle reuses this id after takeover, while a + // genuine reopen gets a new id and cannot replay an old placement ack. + invocationId: `factory:${decision.issue.key}:${runId}:${spec.role}:${sanitizeAgentSlug(spec.name)}`, + } + if (spec.role !== 'implementer') return lifecycleSpec + const runSuffix = `-${runId.slice(0, 8)}` + const stem = `${sanitizeAgentSlug(decision.issue.key)}-${sanitizeAgentSlug(spec.repo)}` + .slice(0, 120 - 'factory/'.length - runSuffix.length) + const branch = `factory/${stem}${runSuffix}` + return { + ...lifecycleSpec, + branch, + task: [ + spec.task, + '', + `Factory publication branch: ${branch}`, + 'Before editing, create or reset that exact branch from the repository default branch. Commit and push only that branch.', + ].join('\n'), + } + } + return { + ...structuredClone(decision), + implementers: decision.implementers.map(withBranch), + reviewer: withBranch(decision.reviewer), + ...(decision.workflow ? { workflow: withBranch(decision.workflow) } : {}), + } +} + function routeReviewerSpec( issue: LinearIssue, config: FactoryConfig, @@ -8525,6 +9362,46 @@ const durableBabysitterTrackedAgent = (session: BabysitterSessionState): Tracked result: { name: session.agentName }, }) +const isTerminalDispatchLifecycle = (lifecycle: DispatchLifecycle): boolean => + lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned' + +const lifecycleFromInFlightRecord = ( + record: InFlightIssue, + runId: string, + phase: DispatchLifecyclePhase, + updatedAtMs: number, + pullRequest?: GithubPublishPullRequestResult, + releaseReason?: string, +): DispatchLifecycle => ({ + runId, + issue: { ...record.issue }, + decision: structuredClone(record.decision), + dryRun: record.dryRun, + phase, + agents: [...record.agents].map(([name, tracked]) => ({ name, tracked: cloneTrackedAgent(tracked) })), + invocationIds: [...record.invocationIds], + result: record.result ? structuredClone(record.result) : undefined, + ...(pullRequest ? { pullRequest: { ...pullRequest } } : {}), + ...(releaseReason ? { releaseReason } : {}), + updatedAtMs, +}) + +const inFlightRecordFromLifecycle = (lifecycle: DispatchLifecycle): InFlightIssue => ({ + issue: { ...lifecycle.issue }, + decision: structuredClone(lifecycle.decision), + dryRun: lifecycle.dryRun, + agents: new Map(lifecycle.agents.map((agent) => [agent.name, cloneTrackedAgent(agent.tracked)])), + invocationIds: new Set(lifecycle.invocationIds), + result: lifecycle.result ? structuredClone(lifecycle.result) : undefined, +}) + +const dispatchResultFromLifecycle = (lifecycle: DispatchLifecycle): DispatchResult => + lifecycle.result ? structuredClone(lifecycle.result) : { + issue: { ...lifecycle.issue }, + agents: lifecycle.agents.map(({ name, tracked }) => ({ name, role: tracked.spec.role })), + dryRun: lifecycle.dryRun, + } + const parseSlackReply = (path: string, content: unknown, botUserId: string): SlackReply | undefined => { const raw = asRecord(parseJsonContent(content)) ?? {} const payload = wrappedPayload(raw) diff --git a/src/ports/fleet.ts b/src/ports/fleet.ts index cc91b71..29a1de2 100644 --- a/src/ports/fleet.ts +++ b/src/ports/fleet.ts @@ -26,10 +26,14 @@ export interface SpawnResult { sessionRef?: string pid?: number pids?: number[] + /** Placement node that owns this process. Local PID operations are unsafe when this is remote. */ + node?: string + /** Explicit process locality; never infer whether a PID is local from a node name. */ + locality?: 'local' | 'remote' } export interface RosterEntry { - agents: Array<{ name: string }> + agents: Array<{ name: string; node?: string }> nodes: Array<{ name: string; capabilities: Capability[]; live: boolean }> } @@ -43,8 +47,17 @@ export type AgentMessage = { from: string; target: string; body: string; threadI export type FleetTrackedAgent = { invocationId?: string; node?: string } export interface FleetClient { + /** Backend-wide placement locality, used when recovering a spawn ack crash gap. */ + readonly placementLocality?: 'local' | 'remote' spawn(input: SpawnInput): Promise - resume(input: { name?: string; sessionRef: string; node?: 'self' | string; capability?: Capability }): Promise + resume(input: { + name?: string + sessionRef: string + node?: 'self' | string + capability?: Capability + repo?: string + clonePath?: string + }): Promise release(name: string, reason?: string): Promise roster(): Promise resolveAgentPid?(name: string): Promise @@ -95,4 +108,6 @@ export type AgentSpec = { number: number kinds: string[] } + /** Deterministic pushed branch used by the durable cross-node PR publisher. */ + branch?: string } diff --git a/src/ports/mount.ts b/src/ports/mount.ts index a8f466e..bb32012 100644 --- a/src/ports/mount.ts +++ b/src/ports/mount.ts @@ -30,7 +30,11 @@ export interface ProviderSyncStatus { export interface GithubPublishPullRequestInput { repo: string - clonePath: string + /** Local checkout fallback for internal/local dispatches. */ + clonePath?: string + /** Exact branch already pushed by a remote implementer. Avoids reading its node-local clone. */ + headRef?: string + headSha?: string baseRef: string title: string body: string diff --git a/src/ports/state.ts b/src/ports/state.ts index 6bbda52..755eacf 100644 --- a/src/ports/state.ts +++ b/src/ports/state.ts @@ -71,6 +71,49 @@ export type DispatchAttemptState = { backoffUntilMs: number } +export type DispatchLifecyclePhase = + | 'queued' + | 'dispatching' + | 'retryable' + | 'running' + | 'parking' + | 'waiting-for-human' + | 'publishing' + | 'published' + | 'writeback-applied' + | 'releasing' + | 'complete' + | 'abandoned' + +export type DispatchLifecycleLease = { + owner: string + epoch: number + leaseUntilMs: number +} + +export type DispatchLifecycle = { + /** Stable for one dispatch attempt and reused by crash takeover; new after a true reopen. */ + runId: string + issue: IssueRef + decision: TriageDecision + dryRun: boolean + phase: DispatchLifecyclePhase + agents: Array<{ name: string; tracked: TrackedAgent; releasedAtMs?: number }> + invocationIds: string[] + result?: import('../types').DispatchResult + pullRequest?: import('./mount').GithubPublishPullRequestResult + releaseReason?: string + lease?: DispatchLifecycleLease + updatedAtMs: number +} + +export type DispatchLifecycleClaim = { + acquired: boolean + lifecycle: DispatchLifecycle + lease?: DispatchLifecycleLease + created: boolean +} + export type GithubIssueCommentWatchPending = { correlationId: string kind: 'triage' | 'agent-question' @@ -114,7 +157,9 @@ export interface BatchSnapshot { invocationId: string, result: SpawnResult, ): void + recordPlanned(record: InFlightIssue, spec: InFlightIssue['decision']['reviewer']): void recordDryRun(record: InFlightIssue, spec: InFlightIssue['decision']['reviewer'], invocationId: string): void + restore(record: InFlightIssue): InFlightIssue } export interface StateStore { @@ -123,6 +168,47 @@ export interface StateStore { getDispatchAttempts(workspaceId: string, issueKey: string): Promise releaseInFlight(workspaceId: string, issueKey: string): Promise + claimDispatchLifecycle( + workspaceId: string, + key: string, + seed: DispatchLifecycle, + owner: string, + nowMs: number, + leaseMs: number, + ): Promise + renewDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + leaseMs: number, + ): Promise + promoteDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + ): Promise + releaseDispatchLifecycleLease( + workspaceId: string, + key: string, + owner: string, + epoch: number, + ): Promise + saveDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + lifecycle: DispatchLifecycle, + ): Promise + getDispatchLifecycle(workspaceId: string, key: string): Promise + listDispatchLifecycles(workspaceId: string): Promise> + clearDispatchLifecycle(workspaceId: string, key: string): Promise + recordCritical(workspaceId: string, key: string, value: CriticalRecord): Promise consumeCritical(workspaceId: string, key: string): Promise isResumed(workspaceId: string, exitKey: string): Promise diff --git a/src/state/file-state-store.test.ts b/src/state/file-state-store.test.ts index bb1af86..f30c282 100644 --- a/src/state/file-state-store.test.ts +++ b/src/state/file-state-store.test.ts @@ -4,10 +4,54 @@ import { join } from 'node:path' import { setTimeout as delay } from 'node:timers/promises' import { describe, expect, it } from 'vitest' -import type { BabysitterSessionState, GithubIssueCommentWatchState, WaitingClarification } from '../ports/state' +import type { BabysitterSessionState, DispatchLifecycle, GithubIssueCommentWatchState, WaitingClarification } from '../ports/state' import { FileStateStore } from './file-state-store' describe('FileStateStore', () => { + it('persists and fences dispatch lifecycle ownership across processes and crash takeover', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-lifecycle-')) + try { + const watchStatePath = join(root, 'state.json') + const first = new FileStateStore({ batchSize: 2, watchStatePath }) + const second = new FileStateStore({ batchSize: 2, watchStatePath }) + const key = 'AR-85:uuid-85:/linear/issues/AR-85.json' + const seed = dispatchLifecycle(85) + + const initial = await first.claimDispatchLifecycle('workspace-1', key, seed, 'owner-a', 1_000, 5_000) + expect(initial).toMatchObject({ acquired: true, created: true, lease: { owner: 'owner-a', epoch: 1 } }) + + const duplicate = await second.claimDispatchLifecycle('workspace-1', key, seed, 'owner-b', 2_000, 5_000) + expect(duplicate).toMatchObject({ acquired: false, created: false, lifecycle: { lease: { owner: 'owner-a', epoch: 1 } } }) + + const takeover = await second.claimDispatchLifecycle('workspace-1', key, seed, 'owner-b', 6_001, 5_000) + expect(takeover).toMatchObject({ acquired: true, created: false, lease: { owner: 'owner-b', epoch: 2 } }) + + expect(await first.saveDispatchLifecycle('workspace-1', key, 'owner-a', 1, 6_002, { + ...seed, + phase: 'published', + })).toBe(false) + expect(await second.saveDispatchLifecycle('workspace-1', key, 'owner-b', 2, 6_002, { + ...takeover.lifecycle, + phase: 'published', + pullRequest: { + repo: 'AgentWorkforce/factory', + number: 85, + url: 'https://github.com/AgentWorkforce/factory/pull/85', + headRef: 'factory/ar-85-agentworkforce-factory', + }, + })).toBe(true) + + const restarted = new FileStateStore({ batchSize: 2, watchStatePath }) + expect(await restarted.getDispatchLifecycle('workspace-1', key)).toMatchObject({ + phase: 'published', + lease: { owner: 'owner-b', epoch: 2 }, + pullRequest: { number: 85 }, + }) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + it('restores and clears babysitter ownership plus pending wake state in a fresh process-equivalent store', async () => { const root = await mkdtemp(join(tmpdir(), 'factory-file-state-babysitter-')) try { @@ -37,6 +81,78 @@ describe('FileStateStore', () => { } }) + it('preserves babysitter sessions and dispatch lifecycles together across restart and independent updates', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-file-state-combined-')) + try { + const watchStatePath = join(root, 'factory-state.json') + const issueKey = 'AR-9387:uuid-9387:/linear/issues/AR-9387__uuid-9387.json' + const session: BabysitterSessionState = { + issue: { uuid: 'uuid-9387', key: 'AR-9387', path: '/linear/issues/AR-9387__uuid-9387.json' }, + repo: 'AgentWorkforce/factory', + prNumber: 93, + agentName: 'ar-9387-babysit-factory', + path: '/github/repos/AgentWorkforce/factory/pulls/93/metadata.json', + critical: true, + pendingKinds: ['review-comment'], + } + const first = new FileStateStore({ batchSize: 2, watchStatePath }) + const claim = await first.claimDispatchLifecycle( + 'workspace-1', issueKey, dispatchLifecycle(93), 'owner-a', 1_000, 5_000, + ) + expect(claim.acquired).toBe(true) + await first.setBabysitterSession('workspace-1', issueKey, session) + + const restarted = new FileStateStore({ batchSize: 2, watchStatePath }) + expect(await restarted.listBabysitterSessions('workspace-1')).toEqual([[issueKey, session]]) + expect(await restarted.getDispatchLifecycle('workspace-1', issueKey)).toMatchObject({ + phase: 'dispatching', + lease: { owner: 'owner-a', epoch: 1 }, + }) + + expect(await restarted.saveDispatchLifecycle('workspace-1', issueKey, 'owner-a', 1, 1_001, { + ...claim.lifecycle, + phase: 'published', + })).toBe(true) + expect(await restarted.listBabysitterSessions('workspace-1')).toEqual([[issueKey, session]]) + + await restarted.clearBabysitterSession('workspace-1', issueKey) + expect(await new FileStateStore({ batchSize: 2, watchStatePath }).getDispatchLifecycle('workspace-1', issueKey)) + .toMatchObject({ phase: 'published' }) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('enforces batch capacity across store instances and promotes queued work after a durable slot release', async () => { + const root = await mkdtemp(join(tmpdir(), 'factory-lifecycle-capacity-')) + try { + const watchStatePath = join(root, 'state.json') + const first = new FileStateStore({ batchSize: 1, watchStatePath }) + const second = new FileStateStore({ batchSize: 1, watchStatePath }) + const firstKey = 'AR-851:uuid-851:/linear/issues/AR-851.json' + const secondKey = 'AR-852:uuid-852:/linear/issues/AR-852.json' + const firstClaim = await first.claimDispatchLifecycle( + 'workspace-1', firstKey, dispatchLifecycle(851), 'owner-a', 1_000, 5_000, + ) + const secondClaim = await second.claimDispatchLifecycle( + 'workspace-1', secondKey, dispatchLifecycle(852), 'owner-b', 1_001, 5_000, + ) + + expect(firstClaim.lifecycle.phase).toBe('dispatching') + expect(secondClaim).toMatchObject({ acquired: true, lifecycle: { phase: 'queued' } }) + expect(await second.promoteDispatchLifecycle('workspace-1', secondKey, 'owner-b', 1, 1_002)).toBe(false) + + expect(await first.saveDispatchLifecycle('workspace-1', firstKey, 'owner-a', 1, 1_003, { + ...firstClaim.lifecycle, + phase: 'releasing', + })).toBe(true) + expect(await second.promoteDispatchLifecycle('workspace-1', secondKey, 'owner-b', 1, 1_004)).toBe(true) + expect(await second.getDispatchLifecycle('workspace-1', secondKey)).toMatchObject({ phase: 'dispatching' }) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + it('rejects a persisted babysitter session that omits the critical fence state', async () => { const root = await mkdtemp(join(tmpdir(), 'factory-file-state-invalid-babysitter-')) try { @@ -326,6 +442,40 @@ describe('FileStateStore', () => { }) }) +const dispatchLifecycle = (number: number): DispatchLifecycle => ({ + runId: `run-${number}`, + issue: { + key: `AR-${number}`, + uuid: `uuid-${number}`, + path: `/linear/issues/AR-${number}.json`, + }, + decision: { + issue: { + key: `AR-${number}`, + uuid: `uuid-${number}`, + path: `/linear/issues/AR-${number}.json`, + }, + routes: [{ repo: 'AgentWorkforce/factory', clonePath: '/work/factory', rationale: 'test' }], + scope: 'single', + implementers: [], + reviewer: { + name: `ar-${number}-review`, + role: 'reviewer', + capability: 'spawn:claude', + task: 'review', + repo: 'AgentWorkforce/factory', + }, + thin: false, + confidence: 'high', + rationale: 'test', + }, + dryRun: false, + phase: 'dispatching', + agents: [], + invocationIds: [], + updatedAtMs: 1_000, +}) + const waitingClarification = (number: number): WaitingClarification => { const issue = { uuid: `uuid-${number}`, key: `AR-${number}`, path: `path-${number}` } const implementer = { diff --git a/src/state/file-state-store.ts b/src/state/file-state-store.ts index 3e581be..1fc1f85 100644 --- a/src/state/file-state-store.ts +++ b/src/state/file-state-store.ts @@ -4,17 +4,25 @@ import { dirname, join } from 'node:path' import lockfile from 'proper-lockfile' -import type { BabysitterSessionState, ClarificationReply, GithubIssueCommentWatchState, WaitingClarification } from '../ports/state' +import type { + BabysitterSessionState, + ClarificationReply, + DispatchLifecycle, + DispatchLifecycleClaim, + GithubIssueCommentWatchState, + WaitingClarification, +} from '../ports/state' import { InMemoryStateStore, type InMemoryStateStoreOptions } from './in-memory-state-store' type PersistedWorkspaceState = { githubIssueCommentWatches: Record waitingClarifications: Record babysitterSessions: Record + dispatchLifecycles: Record } type WatchStateDocument = { - version: 2 + version: 3 workspaces: Record } @@ -41,11 +49,160 @@ const WATCH_STATE_LOCK_STALE_MS = 60_000 */ export class FileStateStore extends InMemoryStateStore { readonly #watchStatePath: string + readonly #batchSize: number #operation: Promise = Promise.resolve() constructor(options: FileStateStoreOptions) { super(options) this.#watchStatePath = options.watchStatePath + this.#batchSize = options.batchSize + } + + override async claimDispatchLifecycle( + workspaceId: string, + key: string, + seed: DispatchLifecycle, + owner: string, + nowMs: number, + leaseMs: number, + ): Promise { + return await this.#exclusive(async () => this.#withMutationLock(async () => { + const document = await this.#loadFromDisk() + const workspace = document.workspaces[workspaceId] ??= emptyWorkspaceState() + let lifecycle = workspace.dispatchLifecycles[key] + const created = !lifecycle + if (!lifecycle) { + lifecycle = cloneLifecycle(seed) + if (activeDispatchLifecycleCount(workspace.dispatchLifecycles) >= this.#batchSize) lifecycle.phase = 'queued' + workspace.dispatchLifecycles[key] = lifecycle + } + const terminal = lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned' + const activeOtherOwner = lifecycle.lease && lifecycle.lease.owner !== owner && lifecycle.lease.leaseUntilMs > nowMs + if (terminal || activeOtherOwner) { + return { acquired: false, lifecycle: cloneLifecycle(lifecycle), created } + } + const epoch = lifecycle.lease?.owner === owner + ? lifecycle.lease.epoch + : (lifecycle.lease?.epoch ?? 0) + 1 + lifecycle.lease = { owner, epoch, leaseUntilMs: nowMs + leaseMs } + lifecycle.updatedAtMs = nowMs + await this.#persist(document) + return { + acquired: true, + lifecycle: cloneLifecycle(lifecycle), + lease: { ...lifecycle.lease }, + created, + } + })) + } + + override async renewDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + leaseMs: number, + ): Promise { + return await this.#exclusive(async () => this.#withMutationLock(async () => { + const document = await this.#loadFromDisk() + const lifecycle = document.workspaces[workspaceId]?.dispatchLifecycles[key] + if (!lifecycle?.lease || lifecycle.lease.owner !== owner || lifecycle.lease.epoch !== epoch) return false + lifecycle.lease.leaseUntilMs = nowMs + leaseMs + lifecycle.updatedAtMs = nowMs + await this.#persist(document) + return true + })) + } + + override async promoteDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + ): Promise { + return await this.#exclusive(async () => this.#withMutationLock(async () => { + const document = await this.#loadFromDisk() + const workspace = document.workspaces[workspaceId] + const lifecycle = workspace?.dispatchLifecycles[key] + if ( + (lifecycle?.phase !== 'queued' && lifecycle?.phase !== 'waiting-for-human') || + !lifecycle.lease || + lifecycle.lease.owner !== owner || + lifecycle.lease.epoch !== epoch || + lifecycle.lease.leaseUntilMs <= nowMs || + activeDispatchLifecycleCount(workspace!.dispatchLifecycles, key) >= this.#batchSize + ) return false + lifecycle.phase = 'dispatching' + lifecycle.updatedAtMs = nowMs + await this.#persist(document) + return true + })) + } + + override async releaseDispatchLifecycleLease( + workspaceId: string, + key: string, + owner: string, + epoch: number, + ): Promise { + await this.#exclusive(async () => this.#withMutationLock(async () => { + const document = await this.#loadFromDisk() + const lease = document.workspaces[workspaceId]?.dispatchLifecycles[key]?.lease + if (lease?.owner !== owner || lease.epoch !== epoch) return + lease.leaseUntilMs = Number.MIN_SAFE_INTEGER + await this.#persist(document) + })) + } + + override async saveDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + lifecycle: DispatchLifecycle, + ): Promise { + return await this.#exclusive(async () => this.#withMutationLock(async () => { + const document = await this.#loadFromDisk() + const workspace = document.workspaces[workspaceId] + const current = workspace?.dispatchLifecycles[key] + if (!current?.lease || current.lease.owner !== owner || current.lease.epoch !== epoch || current.lease.leaseUntilMs <= nowMs) { + return false + } + const next = cloneLifecycle(lifecycle) + next.lease = { ...current.lease } + next.updatedAtMs = nowMs + workspace!.dispatchLifecycles[key] = next + await this.#persist(document) + return true + })) + } + + override async getDispatchLifecycle(workspaceId: string, key: string): Promise { + return await this.#exclusive(async () => { + const lifecycle = (await this.#loadFromDisk()).workspaces[workspaceId]?.dispatchLifecycles[key] + return lifecycle ? cloneLifecycle(lifecycle) : undefined + }) + } + + override async listDispatchLifecycles(workspaceId: string): Promise> { + return await this.#exclusive(async () => { + const lifecycles = (await this.#loadFromDisk()).workspaces[workspaceId]?.dispatchLifecycles ?? {} + return Object.entries(lifecycles).map(([key, lifecycle]) => [key, cloneLifecycle(lifecycle)]) + }) + } + + override async clearDispatchLifecycle(workspaceId: string, key: string): Promise { + await this.#exclusive(async () => this.#withMutationLock(async () => { + const document = await this.#loadFromDisk() + const workspace = document.workspaces[workspaceId] + if (!workspace || !(key in workspace.dispatchLifecycles)) return + delete workspace.dispatchLifecycles[key] + if (workspaceIsEmpty(workspace)) delete document.workspaces[workspaceId] + await this.#persist(document) + })) } override async setGithubIssueCommentWatch( @@ -449,7 +606,7 @@ export class FileStateStore extends InMemoryStateStore { return parseDocument(parsed) } catch (error) { if (!isMissingFileError(error)) throw error - return { version: 2, workspaces: {} } + return { version: 3, workspaces: {} } } } @@ -503,6 +660,31 @@ const parseDocument = (value: unknown): WatchStateDocument => { if (!isRecord(value) || !isRecord(value.workspaces)) { throw new Error('Factory GitHub watch state file is invalid') } + if (value.version === 3) { + const workspaces: Record = {} + for (const [workspaceId, rawWorkspace] of Object.entries(value.workspaces)) { + if (!isRecord(rawWorkspace)) throw new Error('Factory GitHub watch state file is invalid') + const watches = rawWorkspace.githubIssueCommentWatches + const clarifications = rawWorkspace.waitingClarifications + const babysitters = rawWorkspace.babysitterSessions + const lifecycles = rawWorkspace.dispatchLifecycles + if ( + !isRecord(watches) || + !isRecord(clarifications) || + (babysitters !== undefined && !isRecord(babysitters)) || + (lifecycles !== undefined && !isRecord(lifecycles)) + ) { + throw new Error('Factory GitHub watch state file is invalid') + } + workspaces[workspaceId] = { + githubIssueCommentWatches: watches as Record, + waitingClarifications: clarifications as Record, + babysitterSessions: parseBabysitterSessions(babysitters ?? {}), + dispatchLifecycles: (lifecycles ?? {}) as Record, + } + } + return { version: 3, workspaces } + } if (value.version === 2) { const workspaces: Record = {} for (const [workspaceId, rawWorkspace] of Object.entries(value.workspaces)) { @@ -517,9 +699,10 @@ const parseDocument = (value: unknown): WatchStateDocument => { githubIssueCommentWatches: watches as Record, waitingClarifications: clarifications as Record, babysitterSessions: parseBabysitterSessions(babysitters ?? {}), + dispatchLifecycles: {}, } } - return { version: 2, workspaces } + return { version: 3, workspaces } } if (value.version === 1) { const workspaces: Record = {} @@ -531,9 +714,10 @@ const parseDocument = (value: unknown): WatchStateDocument => { githubIssueCommentWatches: watches as Record, waitingClarifications: {}, babysitterSessions: {}, + dispatchLifecycles: {}, } } - return { version: 2, workspaces } + return { version: 3, workspaces } } throw new Error('Factory GitHub watch state file is invalid') } @@ -582,16 +766,30 @@ const parseBabysitterSessions = (value: Record): Record structuredClone(record) + +const activeDispatchLifecycleCount = (lifecycles: Record, exceptKey?: string): number => + Object.entries(lifecycles).filter(([key, lifecycle]) => key !== exceptKey && dispatchLifecycleOccupiesSlot(lifecycle)).length + +const dispatchLifecycleOccupiesSlot = (lifecycle: DispatchLifecycle): boolean => + lifecycle.phase !== 'queued' && + lifecycle.phase !== 'waiting-for-human' && + lifecycle.phase !== 'releasing' && + lifecycle.phase !== 'complete' && + lifecycle.phase !== 'abandoned' + const emptyWorkspaceState = (): PersistedWorkspaceState => ({ githubIssueCommentWatches: {}, waitingClarifications: {}, babysitterSessions: {}, + dispatchLifecycles: {}, }) const workspaceIsEmpty = (workspace: PersistedWorkspaceState): boolean => Object.keys(workspace.githubIssueCommentWatches).length === 0 && Object.keys(workspace.waitingClarifications).length === 0 && - Object.keys(workspace.babysitterSessions).length === 0 + Object.keys(workspace.babysitterSessions).length === 0 && + Object.keys(workspace.dispatchLifecycles).length === 0 const syncParentDirectory = async (filePath: string): Promise => { const handle = await open(dirname(filePath), 'r') diff --git a/src/state/in-memory-state-store.ts b/src/state/in-memory-state-store.ts index 98f88ea..61082fa 100644 --- a/src/state/in-memory-state-store.ts +++ b/src/state/in-memory-state-store.ts @@ -3,6 +3,8 @@ import type { BatchSnapshot, BabysitterSessionState, CriticalRecord, + DispatchLifecycle, + DispatchLifecycleClaim, DispatchAttemptState, GithubIssueCommentWatchState, RegistryHandoffAgent, @@ -24,6 +26,7 @@ type WorkspaceState = { canonicalIssueStates: Map dispatchFailureReaperHandoffs: Map babysitterSessions: Map + dispatchLifecycles: Map } export type InMemoryStateStoreOptions = { @@ -61,6 +64,116 @@ export class InMemoryStateStore implements StateStore { } } + async claimDispatchLifecycle( + workspaceId: string, + key: string, + seed: DispatchLifecycle, + owner: string, + nowMs: number, + leaseMs: number, + ): Promise { + const lifecycles = this.#workspace(workspaceId).dispatchLifecycles + let lifecycle = lifecycles.get(key) + const created = !lifecycle + if (!lifecycle) { + lifecycle = cloneDispatchLifecycle(seed) + if (activeDispatchLifecycleCount(lifecycles) >= this.#batchSize) lifecycle.phase = 'queued' + lifecycles.set(key, lifecycle) + } + const terminal = lifecycle.phase === 'complete' || lifecycle.phase === 'abandoned' + const activeOtherOwner = lifecycle.lease && lifecycle.lease.owner !== owner && lifecycle.lease.leaseUntilMs > nowMs + if (terminal || activeOtherOwner) { + return { acquired: false, lifecycle: cloneDispatchLifecycle(lifecycle), created } + } + const epoch = lifecycle.lease?.owner === owner + ? lifecycle.lease.epoch + : (lifecycle.lease?.epoch ?? 0) + 1 + lifecycle.lease = { owner, epoch, leaseUntilMs: nowMs + leaseMs } + lifecycle.updatedAtMs = nowMs + return { + acquired: true, + lifecycle: cloneDispatchLifecycle(lifecycle), + lease: { ...lifecycle.lease }, + created, + } + } + + async renewDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + leaseMs: number, + ): Promise { + const lifecycle = this.#workspace(workspaceId).dispatchLifecycles.get(key) + if (!lifecycle?.lease || lifecycle.lease.owner !== owner || lifecycle.lease.epoch !== epoch) return false + lifecycle.lease.leaseUntilMs = nowMs + leaseMs + lifecycle.updatedAtMs = nowMs + return true + } + + async promoteDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + ): Promise { + const lifecycles = this.#workspace(workspaceId).dispatchLifecycles + const lifecycle = lifecycles.get(key) + if ( + (lifecycle?.phase !== 'queued' && lifecycle?.phase !== 'waiting-for-human') || + !lifecycle.lease || + lifecycle.lease.owner !== owner || + lifecycle.lease.epoch !== epoch || + lifecycle.lease.leaseUntilMs <= nowMs || + activeDispatchLifecycleCount(lifecycles, key) >= this.#batchSize + ) return false + lifecycle.phase = 'dispatching' + lifecycle.updatedAtMs = nowMs + return true + } + + async releaseDispatchLifecycleLease(workspaceId: string, key: string, owner: string, epoch: number): Promise { + const lease = this.#workspace(workspaceId).dispatchLifecycles.get(key)?.lease + if (lease?.owner !== owner || lease.epoch !== epoch) return + lease.leaseUntilMs = Number.MIN_SAFE_INTEGER + } + + async saveDispatchLifecycle( + workspaceId: string, + key: string, + owner: string, + epoch: number, + nowMs: number, + lifecycle: DispatchLifecycle, + ): Promise { + const current = this.#workspace(workspaceId).dispatchLifecycles.get(key) + if (!current?.lease || current.lease.owner !== owner || current.lease.epoch !== epoch || current.lease.leaseUntilMs <= nowMs) { + return false + } + const next = cloneDispatchLifecycle(lifecycle) + next.lease = { ...current.lease } + next.updatedAtMs = nowMs + this.#workspace(workspaceId).dispatchLifecycles.set(key, next) + return true + } + + async getDispatchLifecycle(workspaceId: string, key: string): Promise { + const lifecycle = this.#workspace(workspaceId).dispatchLifecycles.get(key) + return lifecycle ? cloneDispatchLifecycle(lifecycle) : undefined + } + + async listDispatchLifecycles(workspaceId: string): Promise> { + return [...this.#workspace(workspaceId).dispatchLifecycles] + .map(([key, lifecycle]) => [key, cloneDispatchLifecycle(lifecycle)]) + } + + async clearDispatchLifecycle(workspaceId: string, key: string): Promise { + this.#workspace(workspaceId).dispatchLifecycles.delete(key) + } + async recordCritical(workspaceId: string, key: string, value: CriticalRecord): Promise { this.#workspace(workspaceId).criticalMessages.set(key, value) } @@ -378,6 +491,7 @@ export class InMemoryStateStore implements StateStore { canonicalIssueStates: new Map(), dispatchFailureReaperHandoffs: new Map(), babysitterSessions: new Map(), + dispatchLifecycles: new Map(), } this.#workspaces.set(workspaceId, state) } @@ -385,6 +499,18 @@ export class InMemoryStateStore implements StateStore { } } +const cloneDispatchLifecycle = (lifecycle: DispatchLifecycle): DispatchLifecycle => structuredClone(lifecycle) + +const activeDispatchLifecycleCount = (lifecycles: Map, exceptKey?: string): number => + [...lifecycles].filter(([key, lifecycle]) => key !== exceptKey && dispatchLifecycleOccupiesSlot(lifecycle)).length + +const dispatchLifecycleOccupiesSlot = (lifecycle: DispatchLifecycle): boolean => + lifecycle.phase !== 'queued' && + lifecycle.phase !== 'waiting-for-human' && + lifecycle.phase !== 'releasing' && + lifecycle.phase !== 'complete' && + lifecycle.phase !== 'abandoned' + const cloneWaitingClarification = (record: WaitingClarification): WaitingClarification => structuredClone(record) diff --git a/src/testing/fakes.ts b/src/testing/fakes.ts index dca798c..eab7d64 100644 --- a/src/testing/fakes.ts +++ b/src/testing/fakes.ts @@ -171,8 +171,16 @@ const mergedLinearIssueContent = (existing: unknown, content: unknown): unknown } export class FakeFleetClient implements FleetClient { + readonly placementLocality: 'local' | 'remote' = 'local' readonly spawns: SpawnInput[] = [] - readonly resumes: Array<{ name?: string; sessionRef: string; node?: 'self' | string; capability?: Capability }> = [] + readonly resumes: Array<{ + name?: string + sessionRef: string + node?: 'self' | string + capability?: Capability + repo?: string + clonePath?: string + }> = [] readonly releases: Array<{ name: string; reason?: string }> = [] readonly messages: SendInput[] = [] readonly inputs: Array<{ name: string; data: string }> = [] @@ -201,7 +209,14 @@ export class FakeFleetClient implements FleetClient { return { name: input.name, sessionRef: this.#sessionRefs.get(input.name) ?? input.sessionRef } } - async resume(input: { name?: string; sessionRef: string; node?: 'self' | string; capability?: Capability }): Promise { + async resume(input: { + name?: string + sessionRef: string + node?: 'self' | string + capability?: Capability + repo?: string + clonePath?: string + }): Promise { this.resumes.push(input) const name = input.name ?? input.sessionRef this.#agents.add(name) diff --git a/src/types.ts b/src/types.ts index bc73a7e..1c489cb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -48,6 +48,7 @@ export interface Factory { runLoop(opts?: FactoryLoopRunOptions): Promise triageIssue(issue: LinearIssue): Promise dispatch(decision: TriageDecision, opts?: { dryRun?: boolean }): Promise + waitForDispatchTerminal(issue: IssueRef): Promise status(): FactoryStatus on( event: 'issue-queued' | 'dispatched' | 'issue-done' | 'writeback-verified' | 'error', @@ -56,7 +57,7 @@ export interface Factory { } export interface FactoryStartOptions { - mode?: 'backfill-and-subscribe' | 'live' + mode?: 'backfill-and-subscribe' | 'live' | 'dispatch-owner' liveSubscription?: Partial }