docs(pi): record parity and pinned behavior evidence

This commit is contained in:
SaladDay
2026-08-02 22:52:40 +00:00
parent a88a58b516
commit 816a8b3928
3 changed files with 585 additions and 4 deletions
+404 -4
View File
@@ -21,9 +21,16 @@
import { createServer } from "node:http";
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { createRequire } from "node:module";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import {
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { join, resolve } from "node:path";
import { pathToFileURL } from "node:url";
const PI = process.env.PI_CHECKOUT;
@@ -45,6 +52,17 @@ if (piCommit !== EXPECTED_PI_COMMIT) {
const requireFromPi = createRequire(join(PI, "package.json"));
const { buildSync, version: esbuildVersion } = requireFromPi("esbuild");
const codingAgentPackagePath = join(PI, "packages/coding-agent/package.json");
const codingAgentPackageBytes = readFileSync(codingAgentPackagePath);
const codingAgentPackage = JSON.parse(codingAgentPackageBytes.toString("utf8"));
const distributionMetadata = {
source: "packages/coding-agent/package.json",
sha256: createHash("sha256").update(codingAgentPackageBytes).digest("hex"),
name: codingAgentPackage.name,
version: codingAgentPackage.version,
bin: codingAgentPackage.bin,
piConfig: codingAgentPackage.piConfig,
};
const ANTHROPIC_SSE =
'event: message_start\ndata: {"type":"message_start","message":{"id":"m","type":"message","role":"assistant","model":"m","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":1}}}\n\n' +
@@ -77,11 +95,40 @@ const aiShimPath = join(harnessDirectory, "pi-ai-shim.mjs");
const compatShimPath = join(harnessDirectory, "pi-ai-compat-shim.mjs");
writeFileSync(
aiShimPath,
'export function lazyStream() { throw new Error("transport shim must not execute during compat capture"); }\n',
[
'export function lazyStream() { throw new Error("transport shim must not execute during compat capture"); }',
"let uuidSequence = 0;",
'export function uuidv7() { uuidSequence += 1; return `00000000-0000-7000-8000-${String(uuidSequence).padStart(12, "0")}`; }',
"export class EventStream {}",
"export class ModelsError extends Error {}",
"export function validateToolArguments() { return undefined; }",
'export function contentText(value) { return typeof value === "string" ? value : ""; }',
'export function retryAssistantCall() { throw new Error("resource capture must not call AI"); }',
"export function parseStreamingJson() { return undefined; }",
"export function modelsAreEqual(left, right) { return left === right; }",
"export function createModels() { return {}; }",
"export function getBuiltinModelDataGeneratedAt() { return undefined; }",
"export function builtinProviders() { return []; }",
"export function radiusProvider() { return undefined; }",
"",
].join("\n"),
);
writeFileSync(
compatShimPath,
'export function getApiProvider() { throw new Error("transport shim must not execute during compat capture"); }\n',
[
'export function getApiProvider() { throw new Error("transport shim must not execute during compat capture"); }',
"export function clampThinkingLevel(value) { return value; }",
"export async function cleanupSessionResources() {}",
"export function getSupportedThinkingLevels() { return []; }",
"export function isContextOverflow() { return false; }",
"export function isRetryableAssistantError() { return false; }",
"export function modelsAreEqual(left, right) { return left === right; }",
"export function resetApiProviders() {}",
'export function streamSimple() { throw new Error("resource capture must not call AI"); }',
'export function stream() { throw new Error("resource capture must not call AI"); }',
'export function completeSimple() { throw new Error("resource capture must not call AI"); }',
"",
].join("\n"),
);
const entryPoint = join(harnessDirectory, "entry.mjs");
writeFileSync(
@@ -90,8 +137,14 @@ writeFileSync(
`export { streamSimple as anthropicMessages } from "${PI}/packages/ai/src/api/anthropic-messages.ts";`,
`export { streamSimple as openaiResponses } from "${PI}/packages/ai/src/api/openai-responses.ts";`,
`export { streamSimple as openaiCompletions } from "${PI}/packages/ai/src/api/openai-completions.ts";`,
`export { streamSimple as googleGenerativeAi } from "${PI}/packages/ai/src/api/google-generative-ai.ts";`,
`export { composeModelProvider } from "${PI}/packages/coding-agent/src/core/provider-composer.ts";`,
`export { resolveConfigValueOrThrow } from "${PI}/packages/coding-agent/src/core/resolve-config-value.ts";`,
`export { loadSkills } from "${PI}/packages/coding-agent/src/core/skills.ts";`,
`export { loadPromptTemplates } from "${PI}/packages/coding-agent/src/core/prompt-templates.ts";`,
`export { SessionManager } from "${PI}/packages/coding-agent/src/core/session-manager.ts";`,
`export { parseArgs } from "${PI}/packages/coding-agent/src/cli/args.ts";`,
`export { createAllToolDefinitions } from "${PI}/packages/coding-agent/src/core/tools/index.ts";`,
].join("\n"),
);
const bundlePath = join(harnessDirectory, "bundle.mjs");
@@ -111,16 +164,57 @@ buildSync({
});
const adapters = await import(pathToFileURL(bundlePath).href);
// ResourceLoader pulls the complete coding-agent resource graph. Bundle the
// real pinned ResourceLoader separately with broad AI stubs; the captured
// instruction behavior remains real while unrelated generated model data is
// kept outside this resource-only probe.
const resourceEntryPoint = join(harnessDirectory, "resource-entry.mjs");
const resourceBundlePath = join(harnessDirectory, "resource-bundle.mjs");
writeFileSync(
resourceEntryPoint,
`export { DefaultResourceLoader } from "${PI}/packages/coding-agent/src/core/resource-loader.ts";\n`,
);
buildSync({
entryPoints: [resourceEntryPoint],
bundle: true,
platform: "node",
format: "esm",
outfile: resourceBundlePath,
external: ["node:*"],
packages: "external",
alias: {
"@earendil-works/pi-ai/providers/all": aiShimPath,
"@earendil-works/pi-ai/oauth": aiShimPath,
"@earendil-works/pi-ai": aiShimPath,
"@earendil-works/pi-ai/compat": compatShimPath,
},
logLevel: "silent",
});
const resourceAdapters = await import(pathToFileURL(resourceBundlePath).href);
const API_BY_ADAPTER = {
anthropicMessages: "anthropic-messages",
openaiResponses: "openai-responses",
openaiCompletions: "openai-completions",
googleGenerativeAi: "google-generative-ai",
};
/** 每个用例只改变凭证与显式 header,其余保持最小合法模型。 */
const CASES = [
["anthropicMessages", "plain-key", "sk-ant-api03-plain", {}],
["anthropicMessages", "oauth-token", "sk-ant-oat01-token", {}],
[
"anthropicMessages",
"oauth-with-explicit-x-api-key",
"sk-ant-oat01-token",
{ "x-api-key": "explicit-secret" },
],
[
"anthropicMessages",
"oauth-with-explicit-authorization",
"sk-ant-oat01-token",
{ authorization: "Bearer configured" },
],
[
"anthropicMessages",
"explicit-x-api-key",
@@ -153,6 +247,13 @@ const CASES = [
"synthesized-secret",
{ "x-api-key": "explicit-secret" },
],
["googleGenerativeAi", "plain-key", "google-plain", {}],
[
"googleGenerativeAi",
"explicit-x-goog-api-key",
"google-synthesized",
{ "x-goog-api-key": "google-explicit" },
],
];
const results = [];
@@ -193,6 +294,7 @@ for (const [adapter, label, apiKey, headers] of CASES) {
adapter: API_BY_ADAPTER[adapter],
case: label,
requestSent: Boolean(request),
requestUrl: request?.url,
error: request ? undefined : error,
authHeaders: request
? Object.fromEntries(
@@ -243,6 +345,38 @@ const compatProvider = adapters.composeModelProvider(
);
const compatSpread = compatProvider.getModels()[0].compat;
const minimalProvider = adapters.composeModelProvider(
"minimal-provider",
undefined,
{
getProvider(providerId) {
return providerId === "minimal-provider"
? {
name: "Minimal provider",
api: "openai-responses",
baseUrl: "https://minimal.example/v1",
apiKey: "literal",
models: [{ id: "minimal-model", name: "Minimal model" }],
}
: undefined;
},
},
undefined,
);
const minimalModel = minimalProvider.getModels()[0];
const minimalProviderComposition = {
id: minimalModel.id,
name: minimalModel.name,
provider: minimalModel.provider,
api: minimalModel.api,
baseUrl: minimalModel.baseUrl,
reasoning: minimalModel.reasoning,
input: minimalModel.input,
cost: minimalModel.cost,
contextWindow: minimalModel.contextWindow,
maxTokens: minimalModel.maxTokens,
};
function jsonSafeJavaScriptValue(value) {
if (typeof value === "string") {
const codeUnits = Array.from({ length: value.length }, (_, index) =>
@@ -357,6 +491,262 @@ const resolverCases = resolverInputs.map((input) => {
}
});
// Execute Pi's real discovery entry point. The directories are deliberately
// created in reverse order: a deterministic winner must come from Pi's
// discovery ordering, not filesystem insertion order.
const skillProbeRoot = join(harnessDirectory, "skill-discovery");
const skillAgentDir = join(skillProbeRoot, "agent");
const skillProjectDir = join(skillProbeRoot, "project");
for (const directory of [
skillProjectDir,
join(skillAgentDir, "skills", "b-second"),
join(skillAgentDir, "skills", "a-first"),
]) {
mkdirSync(directory, { recursive: true });
}
writeFileSync(
join(skillAgentDir, "skills", "b-second", "SKILL.md"),
"---\nname: duplicate\ndescription: second\n---\nsecond\n",
);
writeFileSync(
join(skillAgentDir, "skills", "a-first", "SKILL.md"),
"---\nname: duplicate\ndescription: first\n---\nfirst\n",
);
const skillDiscovery = jsonSafeJavaScriptValue(
await adapters.loadSkills({
cwd: skillProjectDir,
agentDir: skillAgentDir,
skillPaths: [],
includeDefaults: true,
}),
);
const promptAgentDir = join(harnessDirectory, "prompt-agent");
const promptProjectDir = join(harnessDirectory, "prompt-project");
mkdirSync(join(promptAgentDir, "prompts", "nested"), { recursive: true });
mkdirSync(promptProjectDir, { recursive: true });
writeFileSync(
join(promptAgentDir, "prompts", "review.md"),
"---\ndescription: Review captured changes\nargument-hint: <range>\n---\nReview $1\n",
);
writeFileSync(join(promptAgentDir, "prompts", "empty.md"), "");
writeFileSync(
join(promptAgentDir, "prompts", "nested", "ignored.md"),
"nested",
);
const promptTemplateDiscovery = jsonSafeJavaScriptValue(
adapters
.loadPromptTemplates({
cwd: promptProjectDir,
agentDir: promptAgentDir,
promptPaths: [],
includeDefaults: true,
})
.map((template) => ({
name: template.name,
description: template.description,
argumentHint: template.argumentHint,
content: template.content,
source: template.sourceInfo?.source,
scope: template.sourceInfo?.scope,
relativeFile:
template.filePath ===
join(promptAgentDir, "prompts", `${template.name}.md`)
? `prompts/${template.name}.md`
: template.filePath,
})),
);
// File presence, including a zero-byte file, is the native activation state
// for Pi's global instruction resources. Execute the real resource loader so
// cc-switch does not infer that rule from a parser implementation.
const instructionAgentDir = join(harnessDirectory, "instruction-agent");
const instructionProjectDir = join(harnessDirectory, "instruction-project");
mkdirSync(instructionAgentDir, { recursive: true });
mkdirSync(instructionProjectDir, { recursive: true });
for (const filename of ["AGENTS.md", "SYSTEM.md", "APPEND_SYSTEM.md"]) {
writeFileSync(join(instructionAgentDir, filename), "");
}
const instructionLoader = new resourceAdapters.DefaultResourceLoader({
cwd: instructionProjectDir,
agentDir: instructionAgentDir,
noExtensions: true,
noSkills: true,
noPromptTemplates: true,
noThemes: true,
});
await instructionLoader.reload();
const emptyInstructionFiles = {
agentsFiles: instructionLoader.getAgentsFiles().agentsFiles.map((entry) => ({
relativeFile: entry.path.startsWith(instructionAgentDir)
? entry.path.slice(instructionAgentDir.length + 1)
: entry.path,
content: entry.content,
})),
systemPrompt: instructionLoader.getSystemPrompt(),
systemPromptSource: instructionLoader.getSystemPromptSource()?.path,
appendSystemPrompt: instructionLoader.getAppendSystemPrompt(),
appendSystemPromptSources: instructionLoader
.getAppendSystemPromptSources()
.map((entry) => entry.path),
};
// Exercise Pi's real SessionManager instead of inferring sessionDir or JSONL
// shape from its TypeScript source. Relative sessionDir is resolved against the
// launching process cwd, while the header keeps the explicit project cwd.
const sessionProjectDir = join(harnessDirectory, "session project");
mkdirSync(sessionProjectDir, { recursive: true });
const originalCwd = process.cwd();
process.chdir(sessionProjectDir);
const capturedSession = adapters.SessionManager.create(
sessionProjectDir,
".pi/sessions",
{ id: "cc-switch-capture-session" },
);
capturedSession.appendSessionInfo("Captured session");
capturedSession.appendMessage({
role: "user",
content: [{ type: "text", text: "captured question" }],
timestamp: 1_700_000_000_000,
});
capturedSession.appendMessage({
role: "assistant",
content: [{ type: "text", text: "captured answer" }],
api: "openai-responses",
provider: "capture",
model: "capture-model",
usage: {
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: 1_700_000_001_000,
});
const capturedSessionFile = capturedSession.getSessionFile();
if (!capturedSessionFile) {
throw new Error("pinned SessionManager did not persist the capture session");
}
const capturedSessionPath = resolve(capturedSessionFile);
const parsedSessionArgs = adapters.parseArgs([
"--session",
capturedSessionPath,
]);
const parsedVersionArgs = adapters.parseArgs(["--version"]);
const sessionCliSemantics = {
argv: ["--session", capturedSessionPath],
parsedSession: parsedSessionArgs.session,
diagnostics: parsedSessionArgs.diagnostics,
versionArgv: ["--version"],
parsedVersion: parsedVersionArgs.version,
versionDiagnostics: parsedVersionArgs.diagnostics,
};
const capturedSessionLines = capturedSessionFile
? readFileSync(capturedSessionFile, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line))
: [];
const sessionDirectorySemantics = {
processCwd: sessionProjectDir,
suppliedProjectCwd: sessionProjectDir,
suppliedSessionDir: ".pi/sessions",
resolvedSessionDir: capturedSession.getSessionDir(),
headerKeys: Object.keys(capturedSession.getHeader() ?? {}).sort(),
entryShapes: capturedSession.getEntries().map((entry) => ({
type: entry.type,
keys: Object.keys(entry).sort(),
messageRole: entry.type === "message" ? entry.message.role : undefined,
messageKeys:
entry.type === "message" ? Object.keys(entry.message).sort() : undefined,
})),
persistedLineTypes: capturedSessionLines.map((entry) => entry.type),
listAllCount: (
await adapters.SessionManager.listAll(capturedSession.getSessionDir())
).length,
};
const branchedSession = adapters.SessionManager.create(
sessionProjectDir,
".pi/sessions",
{ id: "cc-switch-capture-branch" },
);
const branchRootId = branchedSession.appendMessage({
role: "user",
content: [{ type: "text", text: "branch root" }],
timestamp: 1_700_000_002_000,
});
branchedSession.appendSessionInfo("Abandoned branch name");
branchedSession.appendMessage({
role: "assistant",
content: [{ type: "text", text: "abandoned answer" }],
api: "openai-responses",
provider: "capture",
model: "capture-model",
usage: {
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: 1_700_000_003_000,
});
branchedSession.branch(branchRootId);
branchedSession.appendMessage({
role: "user",
content: [{ type: "text", text: "active branch" }],
timestamp: 1_700_000_004_000,
});
const sessionBranchSemantics = {
sessionName: branchedSession.getSessionName(),
activeEntryTypes: branchedSession.getBranch().map((entry) => entry.type),
};
const malformedSessionFile = join(
capturedSession.getSessionDir(),
"cc-switch-capture-malformed.jsonl",
);
writeFileSync(
malformedSessionFile,
[
JSON.stringify(capturedSession.getHeader()),
"{not valid json",
...capturedSession
.getEntries()
.map((entry) => JSON.stringify(entry)),
"",
].join("\n"),
);
let malformedSessionSemantics;
try {
const malformedSession = adapters.SessionManager.open(malformedSessionFile);
malformedSessionSemantics = {
status: "accepted",
entryTypes: malformedSession.getEntries().map((entry) => entry.type),
};
} catch (error) {
malformedSessionSemantics = {
status: "rejected",
error: error instanceof Error ? error.message : String(error),
};
}
process.chdir(originalCwd);
// The executed built-in tool factory is the authoritative core inventory.
// Extension-provided tools remain possible, but Pi exposes no native MCP
// registry for cc-switch to mirror as an app toggle.
const nativeToolInventory = Object.values(
adapters.createAllToolDefinitions(sessionProjectDir),
)
.map((tool) => tool.name)
.sort();
server.close();
console.log(
JSON.stringify(
@@ -364,11 +754,21 @@ console.log(
bundler: `esbuild@${esbuildVersion}`,
piCheckout: PI,
piCommit,
distributionMetadata,
baseUrl,
results,
minimalProviderComposition,
compatSpread,
compatEdgeCases,
resolverCases,
skillDiscovery,
promptTemplateDiscovery,
emptyInstructionFiles,
sessionDirectorySemantics,
sessionCliSemantics,
sessionBranchSemantics,
malformedSessionSemantics,
nativeToolInventory,
},
null,
2,