From c95c1e2ec8fab9d60a46d3591ea709d80d2f15fd Mon Sep 17 00:00:00 2001 From: Miya Date: Fri, 17 Jul 2026 00:58:30 +0200 Subject: [PATCH] feat(runtime): add reserved no-reply marker --- packages/runtime/CHANGELOG.md | 5 + packages/runtime/src/cloud-defaults.ts | 5 +- packages/runtime/src/ctx.test.ts | 145 +++++++++++++++++++++++++ packages/runtime/src/ctx.ts | 33 +++++- packages/runtime/src/no-reply.test.ts | 80 ++++++++++++++ packages/runtime/src/no-reply.ts | 31 ++++++ packages/runtime/src/runner.test.ts | 54 ++++++++- packages/runtime/src/types.ts | 11 ++ 8 files changed, 360 insertions(+), 4 deletions(-) create mode 100644 packages/runtime/src/no-reply.test.ts create mode 100644 packages/runtime/src/no-reply.ts diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index 9ffa0ea3..859680ae 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add the reserved `[[NO_REPLY]]` harness contract for successful silent runs, + including marker leak sanitization and observability. + ## [4.1.26] - 2026-07-16 ### Fixed diff --git a/packages/runtime/src/cloud-defaults.ts b/packages/runtime/src/cloud-defaults.ts index db222c19..3e509aef 100644 --- a/packages/runtime/src/cloud-defaults.ts +++ b/packages/runtime/src/cloud-defaults.ts @@ -25,6 +25,7 @@ import { import { createDefaultLlm } from './cloud-llm.js'; import { SandboxNotAvailableError } from './errors.js'; import { spawnAndCapture, spawnNonInteractiveAndCapture } from './harness-process.js'; +import { appendNoReplyPromptContract } from './no-reply.js'; import type { FilesContext, HarnessRunArgs, @@ -497,7 +498,9 @@ function createProcessHarnessRunner(args: CloudDefaultOptions & { args.log('warn', 'harness.config.dropped', { warning }); } - const renderedSystemPrompt = renderPersonaInputs(personaSystemPrompt, inputResolution.values); + const renderedSystemPrompt = appendNoReplyPromptContract( + renderPersonaInputs(personaSystemPrompt, inputResolution.values) + ); const cwd = resolveWorkspacePath(args.workspaceRoot, run.cwd ?? args.workspaceRoot); await assertDirectory(cwd); const task = run.prompt; diff --git a/packages/runtime/src/ctx.test.ts b/packages/runtime/src/ctx.test.ts index e73c89b0..93489123 100644 --- a/packages/runtime/src/ctx.test.ts +++ b/packages/runtime/src/ctx.test.ts @@ -121,6 +121,151 @@ test('buildCtx exposes agent and deployment metadata', () => { }); }); +test('ctx.harness.run turns an exact no-reply marker into observable silent success', async () => { + const logs: Array<{ level: string; message: string; attrs?: Record }> = []; + const ctx = buildCtx({ + persona: basePersona, + workspaceId: 'ws-test', + sandbox: stubSandbox, + harnessRunner: async () => ({ output: ' [[NO_REPLY]]\n', exitCode: 0, durationMs: 12 }), + agent: { + id: 'agent_123', + deployedName: 'docs-demo', + spawnedByAgentId: null + }, + deployment: { + id: 'deployment_456', + triggerKind: 'inbox', + parentDeploymentId: null + }, + log: (level, message, attrs) => logs.push({ level, message, attrs }) + }); + + const result = await ctx.harness.run({ prompt: 'decide whether to reply' }); + assert.deepEqual(result, { + output: '', + exitCode: 0, + durationMs: 12, + containsMarker: true, + suppressed: true + }); + assert.deepEqual(logs, [ + { + level: 'info', + message: 'harness.no_reply.suppressed', + attrs: { containsMarker: true } + } + ]); +}); + +test('ctx.harness.run strips a mixed no-reply marker and logs the leak', async () => { + const logs: Array<{ level: string; message: string; attrs?: Record }> = []; + const ctx = buildCtx({ + persona: basePersona, + workspaceId: 'ws-test', + sandbox: stubSandbox, + harnessRunner: async () => ({ + output: 'Visible before. [[NO_REPLY]] Visible after.', + exitCode: 0, + durationMs: 8 + }), + agent: { + id: 'agent_123', + deployedName: 'docs-demo', + spawnedByAgentId: null + }, + deployment: { + id: 'deployment_456', + triggerKind: 'inbox', + parentDeploymentId: null + }, + log: (level, message, attrs) => logs.push({ level, message, attrs }) + }); + + const result = await ctx.harness.run({ prompt: 'decide whether to reply' }); + assert.deepEqual(result, { + output: 'Visible before. Visible after.', + exitCode: 0, + durationMs: 8, + containsMarker: true, + suppressed: false + }); + assert.deepEqual(logs, [ + { + level: 'warn', + message: 'harness.no_reply.marker_leak', + attrs: { containsMarker: true } + } + ]); +}); + +test('ctx.harness.run strips the marker from stderr as well as output', async () => { + const logs: Array<{ level: string; message: string; attrs?: Record }> = []; + const ctx = buildCtx({ + persona: basePersona, + workspaceId: 'ws-test', + sandbox: stubSandbox, + harnessRunner: async () => ({ + output: 'Visible response.', + stderr: 'diagnostic [[NO_REPLY]]', + exitCode: 0, + durationMs: 5 + }), + agent: { + id: 'agent_123', + deployedName: 'docs-demo', + spawnedByAgentId: null + }, + deployment: { + id: 'deployment_456', + triggerKind: 'inbox', + parentDeploymentId: null + }, + log: (level, message, attrs) => logs.push({ level, message, attrs }) + }); + + const result = await ctx.harness.run({ prompt: 'decide whether to reply' }); + assert.equal(result.output, 'Visible response.'); + assert.equal(result.stderr, 'diagnostic'); + assert.equal(result.containsMarker, true); + assert.equal(result.suppressed, false); + assert.deepEqual(logs, [ + { + level: 'warn', + message: 'harness.no_reply.marker_leak', + attrs: { containsMarker: true } + } + ]); +}); + +test('ctx.harness.run does not turn a failed marker result into success', async () => { + const logs: Array<{ level: string; message: string }> = []; + const ctx = buildCtx({ + persona: basePersona, + workspaceId: 'ws-test', + sandbox: stubSandbox, + harnessRunner: async () => ({ output: '[[NO_REPLY]]', exitCode: 1, durationMs: 4 }), + agent: { + id: 'agent_123', + deployedName: 'docs-demo', + spawnedByAgentId: null + }, + deployment: { + id: 'deployment_456', + triggerKind: 'inbox', + parentDeploymentId: null + }, + log: (level, message) => logs.push({ level, message }) + }); + + const result = await ctx.harness.run({ prompt: 'decide whether to reply' }); + assert.equal(result.output, ''); + assert.equal(result.exitCode, 1); + assert.equal(result.containsMarker, true); + assert.equal(result.suppressed, true); + assert.deepEqual(logs, [{ level: 'warn', message: 'harness.no_reply.marker_leak' }]); +}); + test('buildCtx exposes ctx.files as a sandbox file helper', async () => { const reads: string[] = []; const writes: Array<{ path: string; contents: string }> = []; diff --git a/packages/runtime/src/ctx.ts b/packages/runtime/src/ctx.ts index c1868900..5c33f128 100644 --- a/packages/runtime/src/ctx.ts +++ b/packages/runtime/src/ctx.ts @@ -16,6 +16,7 @@ import type { } from './types.js'; import { attachTrajectoryRecorder, createTrajectoryRecorder } from './trajectory.js'; import { buildRelayContext } from './relay.js'; +import { NO_REPLY_MARKER, sanitizeNoReplyOutput } from './no-reply.js'; type AgentInputValue = string | number | boolean | null | undefined; @@ -162,7 +163,37 @@ export function buildCtx(options: CtxBuildOptions): WorkforceCtx { workspaceId: options.workspaceId, agentName, llm: options.llm ?? UNAVAILABLE_LLM, - harness: { run: options.harnessRunner }, + harness: { + async run(args) { + const result = await options.harnessRunner(args); + const sanitizedOutput = sanitizeNoReplyOutput(result.output); + const sanitizedStderr = result.stderr === undefined + ? undefined + : sanitizeNoReplyOutput(result.stderr); + const containsMarker = + sanitizedOutput.containsMarker || (sanitizedStderr?.containsMarker ?? false); + + if (containsMarker) { + const exactMarker = + result.exitCode === 0 && + result.output.trim() === NO_REPLY_MARKER && + !(result.stderr?.includes(NO_REPLY_MARKER) ?? false); + log( + exactMarker ? 'info' : 'warn', + exactMarker ? 'harness.no_reply.suppressed' : 'harness.no_reply.marker_leak', + { containsMarker: true } + ); + } + + return { + ...result, + output: sanitizedOutput.output, + ...(sanitizedStderr ? { stderr: sanitizedStderr.output } : {}), + containsMarker, + suppressed: sanitizedOutput.suppressed + }; + } + }, sandbox: options.sandbox, files, credentials: credentialsFromEnv(), diff --git a/packages/runtime/src/no-reply.test.ts b/packages/runtime/src/no-reply.test.ts new file mode 100644 index 00000000..1b1b8865 --- /dev/null +++ b/packages/runtime/src/no-reply.test.ts @@ -0,0 +1,80 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { buildNonInteractiveSpec, type Harness } from '@agentworkforce/persona-kit'; +import { + NO_REPLY_MARKER, + NO_REPLY_PROMPT_CONTRACT, + appendNoReplyPromptContract, + sanitizeNoReplyOutput +} from './no-reply.js'; + +test('sanitizeNoReplyOutput suppresses an exact marker with surrounding transport whitespace', () => { + assert.deepEqual(sanitizeNoReplyOutput(` ${NO_REPLY_MARKER}\n`), { + output: '', + containsMarker: true, + suppressed: true + }); +}); + +test('sanitizeNoReplyOutput strips a leaked marker while preserving visible prose', () => { + assert.deepEqual( + sanitizeNoReplyOutput(`Keep this paragraph.\n\n${NO_REPLY_MARKER}\n\nAnd keep this one.`), + { + output: 'Keep this paragraph.\n\n\n\nAnd keep this one.', + containsMarker: true, + suppressed: false + } + ); +}); + +test('sanitizeNoReplyOutput leaves ordinary harness output unchanged', () => { + assert.deepEqual(sanitizeNoReplyOutput(' visible response\n'), { + output: ' visible response\n', + containsMarker: false, + suppressed: false + }); +}); + +test('the no-reply prompt contract is appended once', () => { + const appended = appendNoReplyPromptContract('Persona instructions.'); + assert.equal(appended, `Persona instructions.\n\n${NO_REPLY_PROMPT_CONTRACT}`); + assert.equal(appendNoReplyPromptContract(appended), appended); +}); + +for (const harness of ['claude', 'codex', 'opencode', 'grok'] as const satisfies readonly Harness[]) { + test(`${harness} receives the no-reply contract on its harness-specific prompt surface`, () => { + const spec = buildNonInteractiveSpec({ + harness, + personaId: 'no-reply-test', + model: modelFor(harness), + systemPrompt: appendNoReplyPromptContract('Persona instructions.'), + task: 'Handle the event.', + name: 'no-reply-test', + workingDirectory: '/workspace' + }); + const harnessVisiblePrompt = [ + ...spec.args, + spec.prompt.contents, + ...spec.configFiles.map((file) => file.contents) + ].join('\n'); + + assert.match(harnessVisiblePrompt, /Persona instructions\./); + assert.match( + harnessVisiblePrompt, + /When no visible reply is useful, make the final message exactly \[\[NO_REPLY\]\]\./ + ); + }); +} + +function modelFor(harness: Harness): string { + switch (harness) { + case 'claude': + return 'anthropic/claude-sonnet-4-6'; + case 'codex': + return 'gpt-5.4'; + case 'opencode': + return 'opencode/minimax-m2.5'; + case 'grok': + return 'grok-build-0.1'; + } +} diff --git a/packages/runtime/src/no-reply.ts b/packages/runtime/src/no-reply.ts new file mode 100644 index 00000000..389720c8 --- /dev/null +++ b/packages/runtime/src/no-reply.ts @@ -0,0 +1,31 @@ +export const NO_REPLY_MARKER = '[[NO_REPLY]]'; + +export const NO_REPLY_PROMPT_CONTRACT = + `When no visible reply is useful, make the final message exactly ${NO_REPLY_MARKER}.`; + +export interface SanitizedNoReplyOutput { + output: string; + containsMarker: boolean; + suppressed: boolean; +} + +/** Append the reserved silent-success instruction without duplicating it. */ +export function appendNoReplyPromptContract(systemPrompt: string): string { + if (systemPrompt.includes(NO_REPLY_PROMPT_CONTRACT)) return systemPrompt; + return systemPrompt + ? `${systemPrompt}\n\n${NO_REPLY_PROMPT_CONTRACT}` + : NO_REPLY_PROMPT_CONTRACT; +} + +/** Remove the reserved marker before harness output can reach a user-visible sink. */ +export function sanitizeNoReplyOutput(output: string): SanitizedNoReplyOutput { + const containsMarker = output.includes(NO_REPLY_MARKER); + if (!containsMarker) return { output, containsMarker: false, suppressed: false }; + + const visibleOutput = output.replaceAll(NO_REPLY_MARKER, '').trim(); + return { + output: visibleOutput, + containsMarker: true, + suppressed: visibleOutput.length === 0 + }; +} diff --git a/packages/runtime/src/runner.test.ts b/packages/runtime/src/runner.test.ts index 9e83dbca..f9be1a2a 100644 --- a/packages/runtime/src/runner.test.ts +++ b/packages/runtime/src/runner.test.ts @@ -255,7 +255,10 @@ test('cloud harness runner materializes AGENTS.md for grok personas', async () = ]); assert.equal(parsed.args.at(-2), '--prompt-file'); assert.equal(parsed.args.at(-1), parsed.promptPath); - assert.equal(parsed.prompt, 'Grok system prompt\n\nUser task:\nsay hello'); + assert.equal( + parsed.prompt, + 'Grok system prompt\n\nWhen no visible reply is useful, make the final message exactly [[NO_REPLY]].\n\nUser task:\nsay hello' + ); await assert.rejects(() => readFile(parsed.promptPath, 'utf8'), { code: 'ENOENT' }); assert.equal(parsed.agents, 'Grok agents sidecar\n'); assert.ok(logs.find((l) => l.message === 'harness.sidecar.materialized')); @@ -265,6 +268,44 @@ test('cloud harness runner materializes AGENTS.md for grok personas', async () = } }); +test('cloud harness runner injects the no-reply contract into opencode config', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'workforce-opencode-cloud-')); + const binDir = path.join(root, 'bin'); + const capturePath = path.join(root, 'opencode-capture.json'); + await writeArgCaptureHarness(binDir, 'opencode', capturePath); + + const defaults = createCloudRuntimeDefaults({ + persona: { + ...persona, + harness: 'opencode', + model: 'opencode/minimax-m2.5', + systemPrompt: 'OpenCode system prompt' + }, + agent: runtimeAgent, + deployment: runtimeDeployment, + workspaceId: 'ws-test', + log: () => {}, + env: { + PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ''}`, + WORKFORCE_SANDBOX_ROOT: root + } + }); + + try { + const result = await defaults.harnessRunner({ prompt: 'say hello' }); + assert.equal(result.exitCode, 0); + const config = JSON.parse(await readFile(path.join(root, 'opencode.json'), 'utf8')) as { + agent: Record; + }; + assert.equal( + config.agent.demo.prompt, + 'OpenCode system prompt\n\nWhen no visible reply is useful, make the final message exactly [[NO_REPLY]].' + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + async function writeFakeHarness(binDir: string, name: string, stdout: string): Promise { await mkdir(binDir, { recursive: true }); await writeFile( @@ -470,7 +511,10 @@ test('cloud default codex harness injects agent-relay MCP args from broker helpe captured.argv.slice(11, -1) ); assert.equal(captured.argv.at(-1), '-'); - assert.equal(captured.stdin, 'coordinate with the team\n\nUser task:\ndo the work'); + assert.equal( + captured.stdin, + 'coordinate with the team\n\nWhen no visible reply is useful, make the final message exactly [[NO_REPLY]].\n\nUser task:\ndo the work' + ); assert.equal( brokerCaptured.argv[brokerCaptured.argv.indexOf('--workspaces-json') + 1], '{"workspaces":[{"id":"ws-relay"}]}' @@ -667,6 +711,12 @@ test('cloud default claude harness merges agent-relay MCP config from broker hel assert.ok(captured.argv.includes('--output-format')); assert.notEqual(captured.argv.at(-1), 'do the work'); assert.equal(captured.stdin, 'do the work'); + const systemPromptIdx = captured.argv.indexOf('--append-system-prompt'); + assert.notEqual(systemPromptIdx, -1); + assert.equal( + captured.argv[systemPromptIdx + 1], + 'coordinate with the team\n\nWhen no visible reply is useful, make the final message exactly [[NO_REPLY]].' + ); const mcpConfigIdx = captured.argv.indexOf('--mcp-config'); const payload = JSON.parse(captured.argv[mcpConfigIdx + 1]) as { diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 0bc32e0f..b8649a08 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -155,6 +155,17 @@ export interface HarnessRunResult { * `spawn ... ENOENT`, etc.) is visible to callers that only read `output`. */ output: string; + /** + * True when the raw harness output contained the reserved `[[NO_REPLY]]` + * marker. The runtime always strips the marker before returning `output`; + * mixed-content occurrences are also logged as leaks. + */ + containsMarker?: boolean; + /** + * True when stripping `[[NO_REPLY]]` leaves no user-visible output. This is + * a successful silent result when {@link exitCode} is 0, not a failure. + */ + suppressed?: boolean; /** Process exit code; 0 on success. */ exitCode: number; /**