fix(transform): relocate volatile env text behind ALL cache breakpoints

Volatile env/context text (git status, cwd, date) rode in req.system —
BEFORE the slab anchor in Anthropic's prefix order (tools → system →
messages) — so any git-state change cold-restarted the entire anchored
prefix. Telemetry attribution (events.jsonl 2026-06-26..07-02): 48.8%
of cold-create waste, ~2.6k tokens/session vs ~200 saved by imaging it.

Now appended as a trailing text block on the LAST user message — the
per-turn live tail that re-caches incrementally anyway — placed AFTER
history collapse (never baked into frozen chunks) and after tool_result
compression. Session-stable billingLine/sysRemainder stay in system;
fallback keeps env in system when no user message exists to carry it.
New telemetry field: envRelocatedChars.
This commit is contained in:
teamchong
2026-07-02 23:17:13 -04:00
parent e49bb11878
commit 06a8b70c01
2 changed files with 66 additions and 19 deletions
+45 -10
View File
@@ -511,6 +511,9 @@ export interface TransformInfo {
staticChars: number;
/** Length of the dynamic (per-turn) slab kept as plain text. */
dynamicChars: number;
/** Chars of volatile env/context text relocated from system to the tail of
* the last user message (absent when kept in system fallback). */
envRelocatedChars?: number;
dynamicBlockCount: number;
/** Tag-shaped blocks in the static slab not in DYNAMIC_BLOCK_TAGS.
* Canary: a new per-turn Claude Code tag would appear here before cache rate collapses. */
@@ -1628,21 +1631,33 @@ export async function transformRequest(
info.imageSourceText = combinedWithHeader.slice(0, 65_536);
}
// 4. Splice images back into the request. OCR framing is baked into the image;
// tail text ("[End of rendered context.] + dynamic + billing") sits after.
const tailParts: string[] = ['[End of rendered context.]'];
if (dynamicText) tailParts.push(dynamicText);
if (billingLine) tailParts.push(billingLine);
const tailText = tailParts.join('\n\n');
// 4. Splice images back into the request. OCR framing is baked into the image.
//
// Volatile env/context text (git status, cwd, date) must NOT ride in
// req.system: Anthropic's cache prefix order is tools → system → messages,
// so system bytes sit BEFORE the slab anchor and any git-state change
// cold-restarted the whole anchored prefix (48.8% of cold-create waste,
// events.jsonl 2026-06-26..07-02). It is carried instead at the END of the
// last user message — the per-turn live tail that re-caches incrementally
// anyway — appended late in this function, AFTER history collapse, so it can
// never be baked into a frozen history chunk. Fallback: if no user message
// exists to carry it, keep it in system rather than drop content.
const hasUserMsg = (req.messages ?? []).some((m) => m.role === 'user');
const volatileEnvParts: string[] = [];
if (dynamicText) volatileEnvParts.push(dynamicText);
if (envMarkdown) volatileEnvParts.push(envMarkdown);
const volatileEnvText = hasUserMsg ? volatileEnvParts.join('\n\n') : '';
// Images go into first user message — system field rejects images (400 system.N.type).
{
const sysTail: SystemField = [];
// billingLine is session-stable (warm reads through the anchored prefix
// confirm it; a per-turn value here would zero every cache read).
if (billingLine) sysTail.push({ type: 'text', text: billingLine });
if (dynamicText) sysTail.push({ type: 'text', text: dynamicText });
// Volatile env section rides after the cache anchor as plain text — the
// model still sees it; the cached slab image no longer depends on it.
if (envMarkdown) sysTail.push({ type: 'text', text: envMarkdown });
if (!hasUserMsg) {
if (dynamicText) sysTail.push({ type: 'text', text: dynamicText });
if (envMarkdown) sysTail.push({ type: 'text', text: envMarkdown });
}
if (Array.isArray(sysRemainder)) sysTail.push(...sysRemainder);
// Tool Reference now rides INSIDE the imaged slab (combinedRaw above) — no
// text splice here. Stubbed tools[] descriptions cite the "## Tool: <name>"
@@ -1962,6 +1977,26 @@ export async function transformRequest(
}
}
// Volatile env/context text lands at the END of the last user message (see
// the block above image splice for why). Runs AFTER history collapse so the
// env bytes stay in the live tail — never imaged into a frozen chunk — and
// AFTER 5b so they are never run through tool_result compression. Note
// tool_result blocks legally precede trailing text blocks in a user message
// (Claude Code appends its own system-reminders the same way).
if (volatileEnvText) {
const msgs = req.messages ?? [];
for (let i = msgs.length - 1; i >= 0; i--) {
const m = msgs[i]!;
if (m.role !== 'user') continue;
const content = Array.isArray(m.content)
? m.content
: [{ type: 'text' as const, text: m.content }];
msgs[i] = { ...m, content: [...content, { type: 'text' as const, text: volatileEnvText }] };
info.envRelocatedChars = volatileEnvText.length;
break;
}
}
info.compressed = true;
// Attribution signal for prompt-cache busts (#11): digest the exact pinned
// prefix we send (history/slab boundary; live tail excluded) AFTER all marker
+21 -9
View File
@@ -373,19 +373,31 @@ describe('e2e cache alignment — Anthropic /v1/messages through the real proxy'
expect(a.length).toBeGreaterThan(0);
expect(b).toEqual(a);
// Not dropped: the volatile section re-enters as plain system text, so the
// model still sees the current git state.
// Not dropped: the volatile section re-enters as trailing TEXT on the LAST
// user message (per-turn live tail), so the model still sees the current
// git state. It must NOT ride in system: system bytes sit BEFORE the slab
// anchor in Anthropic's prefix order (tools → system → messages), so any
// env change there cold-restarts the entire anchored prefix (48.8% of
// telemetry-era cold-create waste).
const sysText = (bodyText: string): string => {
const sys = JSON.parse(bodyText).system;
return Array.isArray(sys) ? sys.map((s: any) => s?.text ?? '').join('\n') : String(sys ?? '');
};
expect(sysText(cap2.main[0]!.body)).toContain('modified: src/pricing.ts');
expect(sysText(cap1.main[0]!.body)).toContain('Git status:\nclean');
// And the section left the imaged (cache-marked) region: the marked block is
// an image, and no remaining system TEXT block still carries the heading
// upstream of the anchor. (Byte-equality above is the load-bearing check;
// this pins the mechanism.)
expect(sysText(cap2.main[0]!.body)).toContain('# Environment');
const lastUserText = (bodyText: string): string => {
const msgs = JSON.parse(bodyText).messages as Array<{ role: string; content: unknown }>;
const m = [...msgs].reverse().find((x) => x.role === 'user')!;
return Array.isArray(m.content)
? m.content.map((c: any) => (c?.type === 'text' ? c.text : '')).join('\n')
: String(m.content ?? '');
};
expect(lastUserText(cap2.main[0]!.body)).toContain('modified: src/pricing.ts');
expect(lastUserText(cap1.main[0]!.body)).toContain('Git status:\nclean');
// And the section left both the imaged region AND system entirely — nothing
// upstream of the anchor may depend on git state. (Byte-equality above is
// the load-bearing check; this pins the mechanism.)
expect(lastUserText(cap2.main[0]!.body)).toContain('# Environment');
expect(sysText(cap2.main[0]!.body)).not.toContain('modified: src/pricing.ts');
expect(sysText(cap2.main[0]!.body)).not.toContain('# Environment');
});
it('FIRST COLLAPSE (turn-2 rewrite): no frozen chunk yet → anchor stays on the SLAB image', async () => {