diff --git a/CHANGELOG.md b/CHANGELOG.md index 2abfd55..24d36bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Expose per-model thinking tiers and forward the selected level as `reasoning_effort`. Each model now registers a `thinkingLevelMap` built from its supported efforts (extracted from the official command-code CLI catalog into `MODEL_EFFORTS`), so pi/OMP expose exactly the tiers each model supports and send the selected level to `/alpha/generate` as `params.reasoning_effort`. Previously all Command Code models registered `reasoning: true` with no `thinkingLevelMap`, so pi hid the `xhigh` and `max` tiers and silently clamped them to `high`. The `off` level and unsupported tiers are omitted from the request. Models absent from `MODEL_EFFORTS` fall back to pi's default `off` → `high` tiers. +- Resolve `COMMANDCODE_MODELS_URL` and `COMMANDCODE_MODELS_CACHE` inside the entry function so overrides are honored at call time (were fixed at module load). - Allow pi to start when model discovery is unavailable. The provider now caches the last successfully fetched model catalog so previously discovered Command Code models remain selectable offline; a first offline start without a cache keeps Command Code unavailable until `/reload` succeeds. ### Contributors diff --git a/README.md b/README.md index 865c6f5..b52bc1d 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,14 @@ You can list the current Command Code models with: pi -e index.ts --list-models ``` +## Thinking levels + +Each model registers a `thinkingLevelMap` built from its supported reasoning efforts, so pi/OMP expose exactly the tiers the model supports — and forward the selected level as `reasoning_effort` in the `/alpha/generate` request body. + +**Per-model efforts.** The supported levels vary by model (e.g. DeepSeek V4 Flash supports only `high` and `max`; Claude Sonnet 5 supports `low` through `max`). These are extracted from the official command-code CLI catalog (`MODEL_EFFORTS` in `src/models.ts`), because the Provider API exposes no effort metadata. Selecting an unsupported tier is hidden from the UI. Models absent from the table get no map, preserving pi's default `off` → `high` tiers. + +**Real passthrough.** The selected thinking level is sent to Command Code as `params.reasoning_effort` (e.g. `"max"`), matching the official CLI. The `off` level has no upstream equivalent and is omitted, so the model's default reasoning applies. + ## Install ```sh diff --git a/index.ts b/index.ts index 4b67b15..b8fc6b1 100644 --- a/index.ts +++ b/index.ts @@ -18,13 +18,17 @@ import { join } from "node:path" import { COMMAND_CODE_CLI_VERSION, createStreamCommandCode, DEFAULT_API_BASE } from "./src/core.ts" import { calculateCommandCodeCost } from "./src/cost.ts" -import { DEFAULT_MODELS_URL, loadCommandCodeModels } from "./src/models.ts" +import { + DEFAULT_MODELS_URL, + loadCommandCodeModels, + thinkingMetadataForModel, +} from "./src/models.ts" import { getApiKey, login, refreshToken } from "./src/oauth.ts" const API_BASE = process.env.COMMANDCODE_API_BASE ?? DEFAULT_API_BASE -const MODELS_URL = process.env.COMMANDCODE_MODELS_URL ?? DEFAULT_MODELS_URL -const MODELS_CACHE_PATH = - process.env.COMMANDCODE_MODELS_CACHE ?? join(getAgentDir(), "commandcode-models.json") +// NOTE: COMMANDCODE_MODELS_URL / COMMANDCODE_MODELS_CACHE are resolved inside the +// entry function so tests (and callers) can override them per-invocation via the +// process environment. type CommandCodeModelCost = { input: number @@ -71,6 +75,17 @@ const MODEL_COSTS: Record = { "xiaomi/mimo-v2.5": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, } +// --------------------------------------------------------------------------- +// Thinking level metadata +// --------------------------------------------------------------------------- +// +// Command Code's /alpha/generate accepts a reasoning_effort parameter whose +// supported values vary per model (extracted from the official command-code CLI +// catalog into MODEL_EFFORTS). We build a per-model reasoningEffortMap from that +// list so pi exposes exactly the tiers the upstream supports — and forwards the +// selected level as reasoning_effort in the request body (see core.ts). +// Models absent from MODEL_EFFORTS get no map, preserving pi's default tiers. + const streamCommandCode = createStreamCommandCode({ createStream: () => new AssistantMessageEventStream(), calculateCost: calculateCommandCodeCost, @@ -82,9 +97,12 @@ const streamCommandCode = createStreamCommandCode({ // --------------------------------------------------------------------------- export default async function (pi: ExtensionAPI) { + const modelsUrl = process.env.COMMANDCODE_MODELS_URL ?? DEFAULT_MODELS_URL + const modelsCachePath = + process.env.COMMANDCODE_MODELS_CACHE ?? join(getAgentDir(), "commandcode-models.json") const { models, warning } = await loadCommandCodeModels({ - url: MODELS_URL, - cachePath: MODELS_CACHE_PATH, + url: modelsUrl, + cachePath: modelsCachePath, }) if (warning) console.warn(`[commandcode] ${warning}`) @@ -106,14 +124,20 @@ export default async function (pi: ExtensionAPI) { refreshToken, getApiKey, }, - models: models.map((model) => ({ - id: model.id, - name: model.name, - reasoning: model.reasoning, - input: ["text"] as const, - cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST, - contextWindow: model.contextWindow, - maxTokens: model.maxTokens, - })), + models: models.map((model) => { + const thinking = thinkingMetadataForModel(model.id) + return { + id: model.id, + name: model.name, + reasoning: model.reasoning, + // OMP 17.2.x reads model.thinking.effortMap; pi-ai <=0.75.5 reads + // model.thinkingLevelMap. Emit both so the plugin works across versions. + ...(thinking ?? {}), + input: ["text"] as const, + cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST, + contextWindow: model.contextWindow, + maxTokens: model.maxTokens, + } + }), }) } diff --git a/package.json b/package.json index 34f19bd..0894d54 100644 --- a/package.json +++ b/package.json @@ -28,12 +28,13 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-thinking-levels.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "test:unit": "tsx tests/test-pure-functions.ts", "test:models": "tsx tests/test-models.ts", + "test:thinking": "tsx tests/test-thinking-levels.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:oauth": "tsx tests/test-oauth.ts", "test:abort": "tsx tests/test-abort.ts", diff --git a/src/core.ts b/src/core.ts index 301ff64..966bc48 100644 --- a/src/core.ts +++ b/src/core.ts @@ -421,6 +421,16 @@ export function createStreamCommandCode(deps: CoreDependencies) { const workingDir = cwd() const threadId = uuid() + // pi passes the resolved thinking level as options.reasoning ("off".."max"). + // Resolve the effort map from whichever field the host version uses: + // pi-ai <=0.75.5 attaches `thinkingLevelMap`; OMP 17.2.x attaches + // `thinking.effortMap`. Then forward as reasoning_effort. Omit when the + // level is "off" or maps to null (unsupported) so the upstream default applies. + const piLevel = options?.reasoning + const effortMap = model.thinking?.effortMap ?? model.thinkingLevelMap + const entry = piLevel && piLevel !== "off" ? effortMap?.[piLevel] : undefined + const reasoningEffort = entry && entry !== "off" ? entry : undefined + let body: unknown = { config: { workingDir, @@ -444,6 +454,7 @@ export function createStreamCommandCode(deps: CoreDependencies) { max_tokens: generateMaxTokens(model, options), temperature: 0.3, stream: true, + ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), }, threadId, } diff --git a/src/models.ts b/src/models.ts index 604cf7c..70f9226 100644 --- a/src/models.ts +++ b/src/models.ts @@ -20,6 +20,93 @@ export interface CommandCodeModel { maxTokens: number } +/** + * Per-model reasoning effort levels supported by Command Code's /alpha/generate. + * + * Extracted from the official command-code CLI (1.7.0) catalog. The Provider API + * (/provider/v1/models) exposes no effort metadata, so this static table is the + * only source. Unknown model ids fall back to no thinkingLevelMap, preserving + * pi's default off..high tiers. + */ +export const MODEL_EFFORTS: Readonly> = { + "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], + "claude-haiku-4-5-20251001": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-4-8": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-5": ["low", "medium", "high", "xhigh", "max"], + "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], + "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], + "deepseek/deepseek-v4-flash": ["high", "max"], + "deepseek/deepseek-v4-pro": ["high", "max"], + "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], + "gpt-5.4": ["low", "medium", "high", "xhigh"], + "gpt-5.4-mini": ["low", "medium", "high"], + "gpt-5.5": ["low", "medium", "high", "xhigh"], + "gpt-5.6-luna": ["low", "medium", "high", "xhigh", "max"], + "gpt-5.6-sol": ["low", "medium", "high", "xhigh", "max"], + "gpt-5.6-terra": ["low", "medium", "high", "xhigh", "max"], + "google/gemini-3.1-flash-lite": ["low", "medium", "high"], + "google/gemini-3.5-flash": ["low", "medium", "high"], + "google/gemini-3.5-flash-lite": ["low", "medium", "high"], + "google/gemini-3.6-flash": ["low", "medium", "high"], + "meta/muse-spark-1.1": ["low", "medium", "high"], + "moonshotai/Kimi-K2.5": ["high", "max"], + "moonshotai/Kimi-K2.6": ["high", "max"], + "sakana/fugu-ultra": ["high", "xhigh"], + "tencent/hy3-paid": ["low", "medium", "high"], + "xai/grok-4.5": ["low", "medium", "high"], + "zai-org/GLM-5.2": ["high", "max"], +} + +/** pi thinking levels in increasing order (mirrors pi-ai ThinkingLevel). */ +const PI_THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const + +/** + * Build a pi ThinkingLevelMap from a Command Code effort list. Supported levels + * map to their upstream effort string; unsupported levels are explicitly `null` + * so pi-ai's getSupportedThinkingLevels hides them and clampThinkingLevel snaps + * to the nearest supported level. `off` always maps to itself. + */ +export function thinkingLevelMapForEfforts( + efforts: readonly string[], +): Record { + const map: Record = { off: "off" } + for (const level of PI_THINKING_LEVELS) { + if (level === "off") continue + map[level] = efforts.includes(level) ? level : null + } + return map +} + +export type ThinkingMetadata = { + thinkingLevelMap: Record + thinking: { + effortMap: Record + efforts: string[] + defaultLevel: string + } +} + +/** + * Resolve per-model thinking metadata for provider registration. Models absent + * from MODEL_EFFORTS get no map, preserving pi's default off..high tiers. + * Emits both `thinkingLevelMap` (pi-ai <=0.75.5) and `thinking.effortMap` + * (OMP 17.2.x) so the plugin works across host versions. + */ +export function thinkingMetadataForModel(modelId: string): ThinkingMetadata | undefined { + const efforts = MODEL_EFFORTS[modelId] + if (!efforts) return undefined + const effortMap = thinkingLevelMapForEfforts(efforts) + return { + thinkingLevelMap: effortMap, + thinking: { + effortMap, + efforts: [...efforts], + defaultLevel: efforts[efforts.length - 2] ?? efforts[0], + }, + } +} + interface FetchCommandCodeModelsOptions { url?: string fetchImpl?: typeof fetch diff --git a/src/types.ts b/src/types.ts index d2a9abe..5c091e2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -63,6 +63,13 @@ export interface ModelLike { provider: string maxTokens: number cost: ModelCost + reasoning?: boolean + thinkingLevelMap?: Record + thinking?: { + effortMap?: Record + efforts?: readonly string[] + defaultLevel?: string + } } export interface MessageLike { @@ -95,6 +102,8 @@ export interface StreamOptions { signal?: AbortSignal headers?: Record maxTokens?: number + /** Resolved pi thinking level ("off".."max"); mapped to reasoning_effort upstream. */ + reasoning?: string onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise onResponse?: (response: ProviderResponseInfo, model: ModelLike) => void | Promise /** diff --git a/tests/test-thinking-levels.ts b/tests/test-thinking-levels.ts new file mode 100644 index 0000000..1f8caab --- /dev/null +++ b/tests/test-thinking-levels.ts @@ -0,0 +1,260 @@ +/** + * Contract tests for thinking-level metadata and reasoning_effort passthrough. + * + * Command Code's /alpha/generate accepts a reasoning_effort parameter whose + * supported values vary per model (extracted from the official command-code CLI + * into MODEL_EFFORTS). These tests verify: + * 1. Each known model exposes exactly the tiers its MODEL_EFFORTS entry lists + * (validated offline for every entry, not just the flash model). + * 2. Unknown models get no map (pi's default off..high applies). + * 3. reasoning_effort appears in the request body only when a supported level + * is selected, and is omitted for "off" and for unsupported (null-mapped) levels. + */ + +import assert from "node:assert/strict" +import { createServer, type Server } from "node:http" +import { after, before, beforeEach, describe, it } from "node:test" + +import { + loadCommandCodeModels, + MODEL_EFFORTS, + thinkingLevelMapForEfforts, + thinkingMetadataForModel, +} from "../src/models.ts" +import { + createTestDeps, + makeContext, + makeModel, + type MockCommandCodeServer, + startMockCommandCodeServer, +} from "./helpers.ts" + +// Mirrors pi-ai's getSupportedThinkingLevels (pi-ai/dist/models.js): a level is +// available when reasoning is true; xhigh/max additionally require an explicit +// non-null thinkingLevelMap entry. Duplicated here so the test catches +// regressions that a presence check would miss. +const ALL_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const + +function supportedLevels(model: Record): string[] { + if (!model.reasoning) return ["off"] + const map = model.thinkingLevelMap as Record | undefined + return ALL_LEVELS.filter((level) => { + const mapped = map?.[level] + if (mapped === null) return false + if (level === "xhigh" || level === "max") return mapped !== undefined + return true + }) +} + +function paramsOf(server: MockCommandCodeServer): Record { + const body = server.lastRequestBody() as { params?: Record } | undefined + return body?.params ?? {} +} + +// ──────────────────────────────────────────────────────────────────────────── +// Pure function tests (no network) +// ──────────────────────────────────────────────────────────────────────────── + +describe("MODEL_EFFORTS + thinkingLevelMapForEfforts", () => { + it("maps supported efforts to themselves and nulls the rest", () => { + const map = thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]) + assert.deepEqual(map, { + off: "off", + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + max: "max", + }) + }) + + it("every MODEL_EFFORTS entry yields a valid map with correct supported tiers", () => { + // Offline structural validation for every entry in the static table. Each + // effort list must be a non-empty subset of valid effort levels, and + // thinkingLevelMapForEfforts must produce a map whose visible tiers match + // exactly. Catches transcription errors without network access. + const VALID_EFFORTS = ["minimal", "low", "medium", "high", "xhigh", "max"] + for (const [id, efforts] of Object.entries(MODEL_EFFORTS)) { + assert.ok(efforts.length > 0, `MODEL_EFFORTS["${id}"] must be non-empty`) + for (const e of efforts) { + assert.ok(VALID_EFFORTS.includes(e), `MODEL_EFFORTS["${id}"] has invalid effort "${e}"`) + } + const map = thinkingLevelMapForEfforts(efforts) + const levels = supportedLevels({ reasoning: true, thinkingLevelMap: map }) + const expected = ALL_LEVELS.filter((l) => l === "off" || efforts.includes(l)) + assert.deepEqual(levels, expected, `model "${id}" should expose tiers matching its efforts`) + } + }) +}) + +// ──────────────────────────────────────────────────────────────────────────── +// Extension registration tests (mock model endpoint) +// ──────────────────────────────────────────────────────────────────────────── + +describe("thinkingMetadataForModel — per-model thinking tiers", () => { + it("exposes only supported tiers per model, and omits the map for unknowns", () => { + const flashMeta = thinkingMetadataForModel("deepseek/deepseek-v4-flash") + assert.ok(flashMeta, "known models must yield thinking metadata") + const flash = { + reasoning: true, + thinkingLevelMap: flashMeta.thinkingLevelMap, + thinking: flashMeta.thinking, + } + assert.deepEqual(supportedLevels(flash), ["off", "high", "max"]) + assert.deepEqual(flashMeta.thinking.efforts, MODEL_EFFORTS["deepseek/deepseek-v4-flash"]) + assert.equal(flashMeta.thinking.effortMap, flashMeta.thinkingLevelMap) + + // unknown model: no thinking metadata → pi default off..high + assert.equal(thinkingMetadataForModel("some-unknown/new-model"), undefined) + const unknownLevels = supportedLevels({ reasoning: true }) + assert.deepEqual(unknownLevels, ["off", "minimal", "low", "medium", "high"]) + }) + + it("keeps the offline-loaded catalog compatible with per-model thinking maps", async () => { + // Smoke-check that thinking metadata remains available for models discovered + // through the post-#27 offline-capable loadCommandCodeModels path. + const sampleBody = { + object: "list", + data: [ + { + id: "deepseek/deepseek-v4-flash", + object: "model", + created: 1, + owned_by: "command-code", + name: "DeepSeek V4 Flash", + context_length: 1_000_000, + }, + { + id: "some-unknown/new-model", + object: "model", + created: 1, + owned_by: "command-code", + name: "Unknown", + context_length: 200_000, + }, + ], + } + let hits = 0 + const server = createServer((_req, res) => { + hits += 1 + res.writeHead(200, { "Content-Type": "application/json" }) + res.end(JSON.stringify(sampleBody)) + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + try { + const address = server.address() + const port = typeof address === "object" && address ? address.port : 0 + const cachePath = `/tmp/commandcode-models-thinking-levels-test.json` + const { models } = await loadCommandCodeModels({ + url: `http://127.0.0.1:${port}/provider/v1/models`, + cachePath, + }) + assert.ok(hits > 0, "model fetch should hit the mock server, not the network") + const registered = models.map((model) => ({ + id: model.id, + reasoning: model.reasoning, + ...(thinkingMetadataForModel(model.id) ?? {}), + })) + const flash = registered.find((m) => m.id === "deepseek/deepseek-v4-flash")! + const unknown = registered.find((m) => m.id === "some-unknown/new-model")! + assert.deepEqual(supportedLevels(flash), ["off", "high", "max"]) + assert.equal(unknown.thinkingLevelMap, undefined) + assert.deepEqual(supportedLevels(unknown), ["off", "minimal", "low", "medium", "high"]) + } finally { + await new Promise((resolve) => server.close(() => resolve())) + } + }) +}) + +// ──────────────────────────────────────────────────────────────────────────── +// reasoning_effort passthrough tests (mock generate endpoint) +// ──────────────────────────────────────────────────────────────────────────── + +describe("streamCommandCode — reasoning_effort passthrough", () => { + let server: MockCommandCodeServer + + before(async () => { + server = await startMockCommandCodeServer() + }) + + after(async () => { + await server.close() + }) + + beforeEach(() => { + server.reset() + }) + + it("forwards the selected effort as reasoning_effort in params", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const flash = makeModel({ + reasoning: true, + thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), + }) + + await collectEventsSafe( + streamCommandCode(flash, makeContext(), { apiKey: "mock-key", reasoning: "max" }), + ) + + assert.ok(server.lastRequestBody(), "a request must have been sent") + assert.equal(paramsOf(server).reasoning_effort, "max") + }) + + it("omits reasoning_effort when the level is off", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const flash = makeModel({ + reasoning: true, + thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), + }) + + await collectEventsSafe( + streamCommandCode(flash, makeContext(), { apiKey: "mock-key", reasoning: "off" }), + ) + + assert.ok(server.lastRequestBody(), "a request must have been sent") + assert.equal( + "reasoning_effort" in paramsOf(server), + false, + "reasoning_effort must be absent for off", + ) + }) + + it("omits reasoning_effort when the level maps to null (unsupported)", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + // flash thinkingLevelMap has low: null — selecting low must not send it. + const flash = makeModel({ + reasoning: true, + thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), + }) + + await collectEventsSafe( + streamCommandCode(flash, makeContext(), { apiKey: "mock-key", reasoning: "low" }), + ) + + assert.ok(server.lastRequestBody(), "a request must have been sent") + assert.equal( + paramsOf(server).reasoning_effort, + undefined, + "reasoning_effort must be omitted when the level maps to null", + ) + }) +}) + +async function collectEventsSafe(stream: AsyncIterable): Promise { + for await (const _event of stream) { + void _event + } +}