mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
transform: large-tool_result paging + adaptive break-even derivation
Two related improvements:
1. Per-tool_result paging / truncation. When a single tool_result would
render to more than maxImagesPerToolResult images (default 10), the
source text is truncated head+marker+tail BEFORE rendering. Classifier
picks the truncation strategy:
- structured (JSON/YAML/diff): head-only, tail is repeating elements
- log (timestamps / [LEVEL] prefixes): 60% head + 40% tail
- other (prose, dumps): same head/tail split
The marker tells the model what was elided so it can reason about the
gap: '[ pixelpipe paging: omitted N lines (K chars) ... ]'. Keeps
requests under Anthropic's 100-image-per-request cap even when a single
tool dumps a multi-megabyte log.
2. Adaptive CHARS_PER_IMAGE. Previously hardcoded at 14_100 (Unifont 10px
@ cell 5×11). Now derived at module load from ATLAS_CELL_H + render.ts
constants. When gen-atlas regenerates with a different font/size, the
break-even threshold automatically updates: at Cozette 4×7 the chars/
image jumps from 14.1k → 22.2k and the break-even drops from ~10k → ~3k.
Single source of truth in src/core/.
Wires:
- TransformOptions.maxImagesPerToolResult (default 10)
- TransformInfo.truncatedToolResults + omittedChars
- TrackEvent.truncated_tool_results + omitted_chars (emit when > 0)
- Two new exported helpers: estimateImageCount, classifyContent, truncateForBudget
- Paging fires before image-encoding at both tool_result branches
(string content and array-of-text-blocks content)
Tests: +28 paging-specific (paging.test.ts), 199/199 total green.
Bundle delta: negligible (~3 KB raw, ~1 KB gzipped). 488 KB dist gzipped.
Closes the paging-implementer worktree merge that was deferred earlier.
This commit is contained in:
+7
-2
@@ -22,10 +22,15 @@ import {
|
||||
} from './atlas.js';
|
||||
import { encodeGrayPng } from './png.js';
|
||||
|
||||
const MAX_HEIGHT_PX = 1568;
|
||||
/** Vertical pixel budget per rendered PNG. Bounded by Anthropic's 1568×1568
|
||||
* image cap. Exported so the break-even gate in transform.ts can derive
|
||||
* CHARS_PER_IMAGE from the same constants the renderer actually uses. */
|
||||
export const MAX_HEIGHT_PX = 1568;
|
||||
const DEFAULT_COLS = 100;
|
||||
const PAD_X = 4;
|
||||
const PAD_Y = 4;
|
||||
/** Vertical padding inside the rendered PNG (top + bottom each). Exported
|
||||
* for the same reason as MAX_HEIGHT_PX. */
|
||||
export const PAD_Y = 4;
|
||||
|
||||
export interface RenderedImage {
|
||||
/** Raw PNG bytes. */
|
||||
|
||||
@@ -39,6 +39,11 @@ export interface TrackEvent {
|
||||
reminder_imgs?: number;
|
||||
/** Image count attributable to compressing tool_result content. */
|
||||
tool_result_imgs?: number;
|
||||
/** Number of tool_result blocks where the source text exceeded the
|
||||
* per-tool_result image budget and was truncated before rendering. */
|
||||
truncated_tool_results?: number;
|
||||
/** Total chars elided by paging across all tool_results this request. */
|
||||
omitted_chars?: number;
|
||||
/** Codepoints rendered into images that weren't in the glyph atlas. A
|
||||
* spike here means users are typing glyphs we don't ship — consider
|
||||
* switching ATLAS_PROFILE to `full-bmp`. */
|
||||
@@ -150,6 +155,12 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent {
|
||||
if (info.dynamicBlockCount !== undefined) out.dynamic_block_count = info.dynamicBlockCount;
|
||||
if (info.reminderImgs !== undefined) out.reminder_imgs = info.reminderImgs;
|
||||
if (info.toolResultImgs !== undefined) out.tool_result_imgs = info.toolResultImgs;
|
||||
if (info.truncatedToolResults !== undefined && info.truncatedToolResults > 0) {
|
||||
out.truncated_tool_results = info.truncatedToolResults;
|
||||
}
|
||||
if (info.omittedChars !== undefined && info.omittedChars > 0) {
|
||||
out.omitted_chars = info.omittedChars;
|
||||
}
|
||||
if (info.droppedChars !== undefined && info.droppedChars > 0) {
|
||||
out.dropped_chars = info.droppedChars;
|
||||
}
|
||||
|
||||
+263
-17
@@ -19,8 +19,9 @@ import type {
|
||||
ToolDef,
|
||||
ToolResultBlock,
|
||||
} from './types.js';
|
||||
import { renderTextToPngs } from './render.js';
|
||||
import { renderTextToPngs, MAX_HEIGHT_PX, PAD_Y } from './render.js';
|
||||
import { bytesToBase64 } from './png.js';
|
||||
import { ATLAS_CELL_H } from './atlas.js';
|
||||
|
||||
export interface TransformOptions {
|
||||
/** Master switch — false makes this a no-op pass-through. */
|
||||
@@ -49,6 +50,11 @@ export interface TransformOptions {
|
||||
placement?: 'system' | 'user';
|
||||
/** Soft-wrap column count. */
|
||||
cols?: number;
|
||||
/** Hard upper bound on images emitted per single tool_result. Above this,
|
||||
* the source text is truncated (head + paging marker + tail) BEFORE
|
||||
* rendering so the request stays under Anthropic's 100-image-per-request
|
||||
* cap even when a single tool dumps a huge log. Default 10. */
|
||||
maxImagesPerToolResult?: number;
|
||||
}
|
||||
|
||||
const DEFAULTS: Required<TransformOptions> = {
|
||||
@@ -71,6 +77,11 @@ const DEFAULTS: Required<TransformOptions> = {
|
||||
// into a user message instead.
|
||||
placement: 'user',
|
||||
cols: 100,
|
||||
// Cap at 10 images per tool_result. With ~14k chars/image at current cell,
|
||||
// a single tool_result can grow to ~140k chars before paging kicks in. A
|
||||
// `find` over a big tree or `grep -r` can easily exceed this; the paging
|
||||
// marker tells the model what was elided. Tuneable per session.
|
||||
maxImagesPerToolResult: 10,
|
||||
};
|
||||
|
||||
// --- per-block break-even check ---
|
||||
@@ -87,13 +98,6 @@ const DEFAULTS: Required<TransformOptions> = {
|
||||
// threshold (5k) was wide of the break-even point (10k) and let net-loss
|
||||
// compressions through. The check below is the real gate.
|
||||
|
||||
/** Characters per rendered image at the current renderer config. Derived
|
||||
* from `cols × floor((MAX_HEIGHT_PX − 2·PAD_Y) / cell_height)` with the
|
||||
* shipping values (cols=100, cell=5×11, MAX=1568, PAD=4):
|
||||
* 100 × floor(1560 / 11) = 100 × 141 = 14,100
|
||||
* Re-derive if cell dimensions or column count ever change. */
|
||||
const CHARS_PER_IMAGE = 14_100;
|
||||
|
||||
/** English ~4 chars per token average. Holds well enough for code + prose
|
||||
* mix; tool_result content is typically code-shaped. */
|
||||
const CHARS_PER_TOKEN = 4;
|
||||
@@ -103,11 +107,39 @@ const CHARS_PER_TOKEN = 4;
|
||||
* to keep `src/core/` free of dashboard imports — that's a one-way edge. */
|
||||
const TOKENS_PER_IMAGE = 2500;
|
||||
|
||||
/** Characters per rendered image at the current renderer config. Derived
|
||||
* at runtime from `ATLAS_CELL_H` (cell height) and the render canvas
|
||||
* dimensions imported from render.ts — single source of truth.
|
||||
*
|
||||
* Formula: `cols × floor((MAX_HEIGHT_PX − 2·PAD_Y) / ATLAS_CELL_H)`
|
||||
*
|
||||
* At the shipping config (Unifont, cell 5×11, cols=100):
|
||||
* 100 × floor((1568 − 8) / 11) = 100 × 141 = 14,100
|
||||
*
|
||||
* When the atlas swaps (e.g. Cozette 4×7, cell H=7), this auto-updates:
|
||||
* 100 × floor(1560 / 7) = 100 × 222 = 22,200
|
||||
* …and the break-even threshold drops accordingly. Without this, the
|
||||
* hardcoded 14,100 would silently let net-loss compressions through on
|
||||
* every smaller-cell atlas. */
|
||||
/** Visual rows per image at the current atlas cell. Derived once at module
|
||||
* load. Auto-updates when gen-atlas regenerates with a different font/size. */
|
||||
export const LINES_PER_IMAGE = Math.max(1, Math.floor((MAX_HEIGHT_PX - 2 * PAD_Y) / ATLAS_CELL_H));
|
||||
|
||||
export function maxCharsPerImage(cols: number): number {
|
||||
return cols * LINES_PER_IMAGE;
|
||||
}
|
||||
|
||||
/** Returns true iff image-compressing a text block of `textLen` chars would
|
||||
* actually save tokens vs leaving it as text. Used as the gate before every
|
||||
* image-encoding decision in transformRequest. */
|
||||
export function isCompressionProfitable(textLen: number): boolean {
|
||||
const estImages = Math.max(1, Math.ceil(textLen / CHARS_PER_IMAGE));
|
||||
* image-encoding decision in transformRequest.
|
||||
*
|
||||
* `cols` defaults to `DEFAULTS.cols` (100) so existing callers and unit
|
||||
* tests that pass only `textLen` keep working byte-identically at the
|
||||
* current atlas. New call sites should pass `o.cols` so a runtime
|
||||
* `--cols` override flows into the break-even math too. */
|
||||
export function isCompressionProfitable(textLen: number, cols: number = DEFAULTS.cols): boolean {
|
||||
const charsPerImage = maxCharsPerImage(cols);
|
||||
const estImages = Math.max(1, Math.ceil(textLen / charsPerImage));
|
||||
const imageTokensCost = estImages * TOKENS_PER_IMAGE;
|
||||
const textTokensEquivalent = textLen / CHARS_PER_TOKEN;
|
||||
return imageTokensCost < textTokensEquivalent;
|
||||
@@ -196,6 +228,11 @@ export interface TransformInfo {
|
||||
* returned false (image cost ≥ text cost at current cell config)
|
||||
* Only emitted when at least one counter is > 0. */
|
||||
passthroughReasons?: { below_threshold?: number; not_profitable?: number };
|
||||
/** Number of tool_result blocks where the source text exceeded the
|
||||
* per-tool_result image budget and was truncated before rendering. */
|
||||
truncatedToolResults?: number;
|
||||
/** Total chars elided by paging across all tool_results this request. */
|
||||
omittedChars?: number;
|
||||
}
|
||||
|
||||
// --- helpers ---------------------------------------------------------------
|
||||
@@ -647,6 +684,204 @@ function makeImageBlock(pngB64: string, ephemeral = false): ImageBlock {
|
||||
* Also returns the total `droppedChars` across all rendered images plus the
|
||||
* merged codepoint→count map so the caller can fold both into the request's
|
||||
* `info.droppedChars` / `info.droppedCodepointsTop`. */
|
||||
|
||||
// --- paging / truncation -------------------------------------------------
|
||||
//
|
||||
// Anthropic's API caps a request at 100 images. A single huge tool_result
|
||||
// (find over a big tree, multi-MB log dump) can blow that cap by itself.
|
||||
// To keep the request valid AND not waste tokens on dozens of bottom-of-log
|
||||
// images, we truncate the source text before render with a marker that
|
||||
// tells the model what was elided.
|
||||
|
||||
/** Visual rows a single input line will consume after soft-wrap at `cols`. */
|
||||
function lineRows(line: string, cols: number): number {
|
||||
return Math.max(1, Math.ceil(line.length / cols));
|
||||
}
|
||||
|
||||
/** Count the visual rows `text` will consume after soft-wrap at `cols`. */
|
||||
function countVisualRows(text: string, cols: number): number {
|
||||
let rows = 0;
|
||||
let lineStart = 0;
|
||||
const len = text.length;
|
||||
for (let i = 0; i <= len; i++) {
|
||||
if (i === len || text.charCodeAt(i) === 10 /* \n */) {
|
||||
const lineLen = i - lineStart;
|
||||
rows += Math.max(1, Math.ceil(lineLen / cols));
|
||||
lineStart = i + 1;
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Estimate how many images `text` will render to at the given column width.
|
||||
* Counts soft-wrapped visual rows, which is what render.ts actually budgets
|
||||
* against. Exported for tests + the paging gate. */
|
||||
export function estimateImageCount(textOrLen: string | number, cols: number): number {
|
||||
if (typeof textOrLen === 'number') {
|
||||
// Back-compat shim — numeric arg gets the looser chars-based estimate.
|
||||
return Math.max(1, Math.ceil(textOrLen / Math.max(1, maxCharsPerImage(cols))));
|
||||
}
|
||||
const rows = countVisualRows(textOrLen, cols);
|
||||
return Math.max(1, Math.ceil(rows / LINES_PER_IMAGE));
|
||||
}
|
||||
|
||||
/** Classify content so we can pick a truncation strategy. Cheap heuristics on
|
||||
* the first ~4 KiB. Returns:
|
||||
* - `'structured'`: JSON/YAML/diff markers at the top. Truncate tail.
|
||||
* - `'log'`: ≥30% of lines start with a log level or timestamp. Truncate middle.
|
||||
* - `'other'`: prose, file dumps, etc. Truncate middle.
|
||||
* Exported for tests. */
|
||||
export function classifyContent(text: string): 'structured' | 'log' | 'other' {
|
||||
const head = text.slice(0, 4096);
|
||||
const trimmed = head.trimStart();
|
||||
if (trimmed.startsWith('{') && /^\{\s*("|\})/.test(trimmed)) return 'structured';
|
||||
if (trimmed.startsWith('[') && /^\[\s*("|\{|\[|-?\d|true\b|false\b|null\b|\])/.test(trimmed))
|
||||
return 'structured';
|
||||
if (trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n')) return 'structured';
|
||||
if (trimmed.startsWith('diff --git ') || /^---\s+\S/.test(trimmed)) return 'structured';
|
||||
const lines = head.split('\n').slice(0, 40).filter((l) => l.length > 0);
|
||||
if (lines.length < 4) return 'other';
|
||||
const LOG_LINE =
|
||||
/^(\[?(DEBUG|INFO|WARN|WARNING|ERROR|TRACE|FATAL)\]?\b|\d{4}-\d{2}-\d{2}[T ]?|\d{2}:\d{2}:\d{2}\b)/;
|
||||
let logHits = 0;
|
||||
for (const line of lines) if (LOG_LINE.test(line)) logHits++;
|
||||
if (logHits / lines.length >= 0.3) return 'log';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
/** Build the paging marker text. The model sees this verbatim INSIDE the
|
||||
* rendered image so it can reason about what was elided. */
|
||||
function buildPagingMarker(args: {
|
||||
originalChars: number;
|
||||
originalLines: number;
|
||||
originalEstImages: number;
|
||||
shownHeadLines: number;
|
||||
shownTailLines: number;
|
||||
omittedLines: number;
|
||||
omittedChars: number;
|
||||
}): string {
|
||||
const tailNote =
|
||||
args.shownTailLines > 0
|
||||
? ` Showing first ${args.shownHeadLines} lines and last ${args.shownTailLines} lines.`
|
||||
: ` Showing first ${args.shownHeadLines} lines (tail elided).`;
|
||||
return (
|
||||
`\n\n[ pixelpipe paging: omitted ${args.omittedLines.toLocaleString('en-US')} lines ` +
|
||||
`(${args.omittedChars.toLocaleString('en-US')} chars) of content here. ` +
|
||||
`Original length: ${args.originalChars.toLocaleString('en-US')} chars ` +
|
||||
`(${args.originalLines.toLocaleString('en-US')} lines, ~${args.originalEstImages} images).` +
|
||||
`${tailNote} ]\n\n`
|
||||
);
|
||||
}
|
||||
|
||||
/** Truncate `text` so it renders to roughly `maxImages` images at the given
|
||||
* `cols`. Picks head/tail split based on `classifyContent`. Budget measured
|
||||
* in visual rows (what render.ts actually slices on). Returns the truncated
|
||||
* text (with paging marker embedded) and the count of chars omitted. If
|
||||
* `text` already fits, returns unchanged with `omittedChars: 0`. Exported
|
||||
* for tests. */
|
||||
export function truncateForBudget(
|
||||
text: string,
|
||||
maxImages: number,
|
||||
cols: number,
|
||||
): { text: string; omittedChars: number; truncated: boolean } {
|
||||
const estImages = estimateImageCount(text, cols);
|
||||
if (estImages <= maxImages) return { text, omittedChars: 0, truncated: false };
|
||||
const totalRowBudget = Math.max(8, maxImages * LINES_PER_IMAGE - 6);
|
||||
const shape = classifyContent(text);
|
||||
const lines = text.split('\n');
|
||||
const originalLines = lines.length;
|
||||
const originalChars = text.length;
|
||||
|
||||
if (shape === 'structured') {
|
||||
let rows = 0;
|
||||
let cut = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const r = lineRows(lines[i]!, cols);
|
||||
if (rows + r > totalRowBudget) break;
|
||||
rows += r;
|
||||
cut = i + 1;
|
||||
}
|
||||
if (cut === 0) cut = 1;
|
||||
const head = lines.slice(0, cut).join('\n');
|
||||
const omitted = originalChars - head.length;
|
||||
return {
|
||||
text:
|
||||
head +
|
||||
buildPagingMarker({
|
||||
originalChars,
|
||||
originalLines,
|
||||
originalEstImages: estImages,
|
||||
shownHeadLines: cut,
|
||||
shownTailLines: 0,
|
||||
omittedLines: originalLines - cut,
|
||||
omittedChars: omitted,
|
||||
}),
|
||||
omittedChars: omitted,
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
|
||||
// log / other: 60% head, 40% tail.
|
||||
const headRowBudget = Math.floor(totalRowBudget * 0.6);
|
||||
const tailRowBudget = totalRowBudget - headRowBudget;
|
||||
let headRows = 0;
|
||||
let headCut = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const r = lineRows(lines[i]!, cols);
|
||||
if (headRows + r > headRowBudget) break;
|
||||
headRows += r;
|
||||
headCut = i + 1;
|
||||
}
|
||||
if (headCut === 0) headCut = 1;
|
||||
let tailRows = 0;
|
||||
let tailStart = lines.length;
|
||||
for (let i = lines.length - 1; i >= headCut; i--) {
|
||||
const r = lineRows(lines[i]!, cols);
|
||||
if (tailRows + r > tailRowBudget) break;
|
||||
tailRows += r;
|
||||
tailStart = i;
|
||||
}
|
||||
if (tailStart <= headCut || tailStart >= lines.length) {
|
||||
const head = lines.slice(0, headCut).join('\n');
|
||||
const omitted = originalChars - head.length;
|
||||
return {
|
||||
text:
|
||||
head +
|
||||
buildPagingMarker({
|
||||
originalChars,
|
||||
originalLines,
|
||||
originalEstImages: estImages,
|
||||
shownHeadLines: headCut,
|
||||
shownTailLines: 0,
|
||||
omittedLines: originalLines - headCut,
|
||||
omittedChars: omitted,
|
||||
}),
|
||||
omittedChars: omitted,
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
const headText = lines.slice(0, headCut).join('\n');
|
||||
const tailText = lines.slice(tailStart).join('\n');
|
||||
const shownChars = headText.length + tailText.length;
|
||||
const omitted = originalChars - shownChars;
|
||||
return {
|
||||
text:
|
||||
headText +
|
||||
buildPagingMarker({
|
||||
originalChars,
|
||||
originalLines,
|
||||
originalEstImages: estImages,
|
||||
shownHeadLines: headCut,
|
||||
shownTailLines: lines.length - tailStart,
|
||||
omittedLines: originalLines - headCut - (lines.length - tailStart),
|
||||
omittedChars: omitted,
|
||||
}) +
|
||||
tailText,
|
||||
omittedChars: omitted,
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function textToImageBlocks(
|
||||
text: string,
|
||||
cols: number,
|
||||
@@ -823,7 +1058,7 @@ export async function transformRequest(
|
||||
// usually 25-30 KB so it always passes (1 image @ 2500 tokens < 25000/4 =
|
||||
// 6250 text-equivalent tokens), but the check guards against the edge
|
||||
// case where a tiny tool docs + tiny static slab combine to <10k chars.
|
||||
if (!isCompressionProfitable(combined.length)) {
|
||||
if (!isCompressionProfitable(combined.length, o.cols)) {
|
||||
info.reason = `not_profitable (slab=${combined.length} chars)`;
|
||||
bumpPassthrough(info, 'not_profitable');
|
||||
return { body, info };
|
||||
@@ -917,7 +1152,7 @@ export async function transformRequest(
|
||||
processedExisting.push(blk);
|
||||
continue;
|
||||
}
|
||||
if (!isCompressionProfitable(textLen)) {
|
||||
if (!isCompressionProfitable(textLen, o.cols)) {
|
||||
// Above threshold but image cost ≥ text cost. Net loss to compress.
|
||||
bumpPassthrough(info, 'not_profitable');
|
||||
processedExisting.push(blk);
|
||||
@@ -976,12 +1211,18 @@ export async function transformRequest(
|
||||
if (inner.length < o.minToolResultChars) {
|
||||
bumpPassthrough(info, 'below_threshold');
|
||||
rewritten.push(blk);
|
||||
} else if (!isCompressionProfitable(inner.length)) {
|
||||
} else if (!isCompressionProfitable(inner.length, o.cols)) {
|
||||
bumpPassthrough(info, 'not_profitable');
|
||||
rewritten.push(blk);
|
||||
} else {
|
||||
// Paging: truncate before render if it would blow the image cap.
|
||||
const paged = truncateForBudget(inner, o.maxImagesPerToolResult, o.cols);
|
||||
if (paged.truncated) {
|
||||
info.truncatedToolResults = (info.truncatedToolResults ?? 0) + 1;
|
||||
info.omittedChars = (info.omittedChars ?? 0) + paged.omittedChars;
|
||||
}
|
||||
const { blocks: imgs, droppedChars, droppedCodepoints: dcp } =
|
||||
await textToImageBlocks(inner, o.cols);
|
||||
await textToImageBlocks(paged.text, o.cols);
|
||||
for (const img of imgs) info.imageBytes += approxBlockBytes(img);
|
||||
info.toolResultImgs = (info.toolResultImgs ?? 0) + imgs.length;
|
||||
info.imageCount += imgs.length;
|
||||
@@ -1010,13 +1251,18 @@ export async function transformRequest(
|
||||
newInner.push(ib as TextBlock | ImageBlock);
|
||||
continue;
|
||||
}
|
||||
if (!isCompressionProfitable(innerText.length)) {
|
||||
if (!isCompressionProfitable(innerText.length, o.cols)) {
|
||||
bumpPassthrough(info, 'not_profitable');
|
||||
newInner.push(ib as TextBlock | ImageBlock);
|
||||
continue;
|
||||
}
|
||||
const paged = truncateForBudget(innerText, o.maxImagesPerToolResult, o.cols);
|
||||
if (paged.truncated) {
|
||||
info.truncatedToolResults = (info.truncatedToolResults ?? 0) + 1;
|
||||
info.omittedChars = (info.omittedChars ?? 0) + paged.omittedChars;
|
||||
}
|
||||
const { blocks: imgs, droppedChars, droppedCodepoints: dcp } =
|
||||
await textToImageBlocks(innerText, o.cols);
|
||||
await textToImageBlocks(paged.text, o.cols);
|
||||
for (const img of imgs) {
|
||||
newInner.push(img);
|
||||
info.imageBytes += approxBlockBytes(img);
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
/**
|
||||
* Tests for the per-tool_result paging / truncation slice (task #42).
|
||||
*
|
||||
* Strategy:
|
||||
* - Unit-test the truncation helpers directly (`classifyContent`,
|
||||
* `estimateImageCount`, `truncateForBudget`) — they're pure, no
|
||||
* rendering needed.
|
||||
* - End-to-end through `transformRequest` to verify the counters land
|
||||
* in `info` (`truncatedToolResults`, `omittedChars`) and that the
|
||||
* image budget is actually honored.
|
||||
*
|
||||
* The rendered PNGs themselves are opaque in tests (we don't have an OCR
|
||||
* harness in this repo), but the truncated *source* text is what
|
||||
* actually carries the paging marker into the image — so verifying the
|
||||
* source string is the right level.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
classifyContent,
|
||||
estimateImageCount,
|
||||
truncateForBudget,
|
||||
transformRequest,
|
||||
} from '../src/core/transform.js';
|
||||
import { toTrackEvent } from '../src/core/tracker.js';
|
||||
import type { ProxyEvent } from '../src/core/proxy.js';
|
||||
|
||||
// Default render config: cols=100, ~141 lines/img → ~14,100 chars/img if
|
||||
// lines fully fill the width. For shorter lines, the budget is dominated
|
||||
// by row count (each line takes ≥1 row regardless of length).
|
||||
const COLS = 100;
|
||||
const ROWS_PER_IMG = 141; // floor((1568 - 8) / 11)
|
||||
|
||||
describe('estimateImageCount', () => {
|
||||
it('returns 1 for empty / tiny text', () => {
|
||||
expect(estimateImageCount('', COLS)).toBe(1);
|
||||
expect(estimateImageCount('hello world', COLS)).toBe(1);
|
||||
});
|
||||
|
||||
it('scales linearly with row count for short-line content', () => {
|
||||
// 141 lines of "x" (1 char) = 141 rows = 1 image.
|
||||
const oneImage = Array.from({ length: 141 }, () => 'x').join('\n');
|
||||
expect(estimateImageCount(oneImage, COLS)).toBe(1);
|
||||
// 142 lines = 2 images (just over the line).
|
||||
const justOver = Array.from({ length: 142 }, () => 'x').join('\n');
|
||||
expect(estimateImageCount(justOver, COLS)).toBe(2);
|
||||
// 10 × 141 = 1410 lines → 10 images.
|
||||
const tenImages = Array.from({ length: 1410 }, () => 'x').join('\n');
|
||||
expect(estimateImageCount(tenImages, COLS)).toBe(10);
|
||||
});
|
||||
|
||||
it('accounts for soft-wrap of long lines', () => {
|
||||
// A single 1000-char line wraps to ceil(1000/100) = 10 rows.
|
||||
const wrapped = 'x'.repeat(1000);
|
||||
expect(estimateImageCount(wrapped, COLS)).toBe(1); // 10 rows, fits in 1 img
|
||||
// 14,100 chars on one line → 141 rows → 1 image.
|
||||
const oneImg = 'x'.repeat(14_100);
|
||||
expect(estimateImageCount(oneImg, COLS)).toBe(1);
|
||||
// 14,101 chars → 142 rows → 2 images.
|
||||
const twoImgs = 'x'.repeat(14_101);
|
||||
expect(estimateImageCount(twoImgs, COLS)).toBe(2);
|
||||
});
|
||||
|
||||
it('also accepts a numeric length (legacy chars-based estimate)', () => {
|
||||
expect(estimateImageCount(0, COLS)).toBe(1);
|
||||
expect(estimateImageCount(14_100, COLS)).toBe(1);
|
||||
expect(estimateImageCount(14_101, COLS)).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyContent', () => {
|
||||
it('flags JSON objects as structured', () => {
|
||||
const json = JSON.stringify({ foo: 'bar', baz: [1, 2, 3] }, null, 2);
|
||||
expect(classifyContent(json)).toBe('structured');
|
||||
});
|
||||
|
||||
it('flags JSON arrays of objects as structured', () => {
|
||||
const json = JSON.stringify(
|
||||
[
|
||||
{ a: 1, b: 2 },
|
||||
{ a: 3, b: 4 },
|
||||
],
|
||||
null,
|
||||
2,
|
||||
);
|
||||
expect(classifyContent(json)).toBe('structured');
|
||||
});
|
||||
|
||||
it('flags YAML frontmatter as structured', () => {
|
||||
const yaml = '---\ntitle: foo\ndate: 2026-05-18\n---\n\nBody text here.';
|
||||
expect(classifyContent(yaml)).toBe('structured');
|
||||
});
|
||||
|
||||
it('flags unified diffs as structured', () => {
|
||||
const diff =
|
||||
'diff --git a/foo.ts b/foo.ts\nindex 1234..5678 100644\n--- a/foo.ts\n+++ b/foo.ts\n@@ -1,3 +1,3 @@\n-old\n+new\n';
|
||||
expect(classifyContent(diff)).toBe('structured');
|
||||
});
|
||||
|
||||
it('flags ISO-timestamp lines as log', () => {
|
||||
const log = Array.from(
|
||||
{ length: 20 },
|
||||
(_, i) => `2026-05-18T12:00:${String(i).padStart(2, '0')}Z some log line`,
|
||||
).join('\n');
|
||||
expect(classifyContent(log)).toBe('log');
|
||||
});
|
||||
|
||||
it('flags [LEVEL] prefix lines as log', () => {
|
||||
const log = Array.from(
|
||||
{ length: 20 },
|
||||
(_, i) => `[INFO] line ${i} doing a thing`,
|
||||
).join('\n');
|
||||
expect(classifyContent(log)).toBe('log');
|
||||
});
|
||||
|
||||
it('flags bare HH:MM:SS prefix lines as log', () => {
|
||||
const log = Array.from(
|
||||
{ length: 20 },
|
||||
(_, i) => `12:00:${String(i).padStart(2, '0')} event ${i}`,
|
||||
).join('\n');
|
||||
expect(classifyContent(log)).toBe('log');
|
||||
});
|
||||
|
||||
it('does NOT flag a stack trace that opens with [ERROR] alone as structured', () => {
|
||||
// Only 4 lines — too few to log-classify cleanly, falls back to other.
|
||||
const text = '[ERROR] something went wrong\n at foo()\n at bar()\n at baz()';
|
||||
// log-line threshold needs ≥30% of ≥4 non-empty lines to start with a
|
||||
// log marker. Just the first line does → 1/4 = 25%, fails → other.
|
||||
expect(classifyContent(text)).toBe('other');
|
||||
});
|
||||
|
||||
it('falls back to other for plain prose', () => {
|
||||
const prose =
|
||||
'The quick brown fox jumps over the lazy dog.\n'.repeat(20);
|
||||
expect(classifyContent(prose)).toBe('other');
|
||||
});
|
||||
|
||||
it('falls back to other for very short input (under 4 lines)', () => {
|
||||
expect(classifyContent('one line')).toBe('other');
|
||||
expect(classifyContent('one\ntwo\nthree')).toBe('other');
|
||||
});
|
||||
});
|
||||
|
||||
describe('truncateForBudget', () => {
|
||||
it('passes through text under the budget unchanged', () => {
|
||||
const text = 'x'.repeat(1000); // way under 10-image budget
|
||||
const { text: out, omittedChars, truncated } = truncateForBudget(text, 10, COLS);
|
||||
expect(truncated).toBe(false);
|
||||
expect(omittedChars).toBe(0);
|
||||
expect(out).toBe(text);
|
||||
});
|
||||
|
||||
it('truncates head+tail for log-shaped content over the budget', () => {
|
||||
// 10k log lines, each ~32 chars → ~320k chars total. With short lines
|
||||
// the row budget dominates: 10k rows >> 10 × 141 = 1410 row budget.
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 10_000; i++) {
|
||||
lines.push(`2026-05-18T12:00:${String(i % 60).padStart(2, '0')}Z entry ${i}`);
|
||||
}
|
||||
const log = lines.join('\n');
|
||||
expect(log.length).toBeGreaterThan(300_000);
|
||||
|
||||
const { text: out, omittedChars, truncated } = truncateForBudget(log, 10, COLS);
|
||||
expect(truncated).toBe(true);
|
||||
expect(omittedChars).toBeGreaterThan(0);
|
||||
// Output should fit in the 10-image budget (count visual rows).
|
||||
expect(estimateImageCount(out, COLS)).toBeLessThanOrEqual(10);
|
||||
// Marker present
|
||||
expect(out).toContain('pixelpipe paging:');
|
||||
// Head + tail format: marker mentions both first and last lines
|
||||
expect(out).toMatch(/Showing first \d+ lines and last \d+ lines/);
|
||||
// Both ends visible: first log entry and last log entry survive
|
||||
expect(out).toContain('entry 0\n'); // first line
|
||||
expect(out).toContain(`entry ${9999}`); // last line (no newline after)
|
||||
});
|
||||
|
||||
it('truncates tail-only for structured (JSON) content over the budget', () => {
|
||||
// Build a huge JSON-shaped blob.
|
||||
const items = Array.from({ length: 5000 }, (_, i) => ({
|
||||
id: i,
|
||||
name: `item-${i}`,
|
||||
payload: 'x'.repeat(100),
|
||||
}));
|
||||
const json = JSON.stringify(items, null, 2);
|
||||
expect(json.length).toBeGreaterThan(500_000);
|
||||
|
||||
const { text: out, omittedChars, truncated } = truncateForBudget(json, 10, COLS);
|
||||
expect(truncated).toBe(true);
|
||||
expect(omittedChars).toBeGreaterThan(0);
|
||||
expect(estimateImageCount(out, COLS)).toBeLessThanOrEqual(10);
|
||||
// Marker present
|
||||
expect(out).toContain('pixelpipe paging:');
|
||||
// Tail-only format: marker says "tail elided", NOT head+tail
|
||||
expect(out).toContain('tail elided');
|
||||
expect(out).not.toMatch(/Showing first \d+ lines and last \d+ lines/);
|
||||
// Head preserved (the structure opens with `[`)
|
||||
expect(out.trimStart().startsWith('[')).toBe(true);
|
||||
// First few items present
|
||||
expect(out).toContain('"item-0"');
|
||||
expect(out).toContain('"item-1"');
|
||||
// Last item is NOT present (it was the tail we dropped)
|
||||
expect(out).not.toContain('"item-4999"');
|
||||
});
|
||||
|
||||
it('truncates head+tail for unclassified prose (default behavior)', () => {
|
||||
// A blob with no log/JSON markers — random prose, repeated.
|
||||
const para =
|
||||
'The quick brown fox jumps over the lazy dog and goes home for dinner.\n';
|
||||
const prose = para.repeat(8000); // ~550k chars
|
||||
|
||||
const { text: out, omittedChars, truncated } = truncateForBudget(prose, 10, COLS);
|
||||
expect(truncated).toBe(true);
|
||||
expect(omittedChars).toBeGreaterThan(0);
|
||||
expect(estimateImageCount(out, COLS)).toBeLessThanOrEqual(10);
|
||||
expect(out).toContain('pixelpipe paging:');
|
||||
// Default prose gets head+tail (not tail-only)
|
||||
expect(out).toMatch(/Showing first \d+ lines and last \d+ lines/);
|
||||
});
|
||||
|
||||
it('marker reports accurate omitted-lines and original-size numbers', () => {
|
||||
// Predictable shape: 10,000 lines of "logline N" — easy to count.
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 10_000; i++) {
|
||||
lines.push(`2026-05-18T12:00:00Z logline ${i} something`);
|
||||
}
|
||||
const log = lines.join('\n');
|
||||
const originalChars = log.length;
|
||||
const originalLines = lines.length;
|
||||
|
||||
const { text: out, omittedChars } = truncateForBudget(log, 10, COLS);
|
||||
|
||||
// Pull the numbers out of the marker.
|
||||
const omittedLinesMatch = out.match(
|
||||
/omitted ([\d,]+) lines \(([\d,]+) chars\)/,
|
||||
);
|
||||
const originalMatch = out.match(/Original length: ([\d,]+) chars \(([\d,]+) lines/);
|
||||
expect(omittedLinesMatch).not.toBeNull();
|
||||
expect(originalMatch).not.toBeNull();
|
||||
|
||||
const parseNum = (s: string) => parseInt(s.replaceAll(',', ''), 10);
|
||||
const reportedOmittedLines = parseNum(omittedLinesMatch![1]!);
|
||||
const reportedOmittedChars = parseNum(omittedLinesMatch![2]!);
|
||||
const reportedOriginalChars = parseNum(originalMatch![1]!);
|
||||
const reportedOriginalLines = parseNum(originalMatch![2]!);
|
||||
|
||||
// Original size numbers should match exactly.
|
||||
expect(reportedOriginalChars).toBe(originalChars);
|
||||
expect(reportedOriginalLines).toBe(originalLines);
|
||||
// Omitted-chars number in marker should match the returned count.
|
||||
expect(reportedOmittedChars).toBe(omittedChars);
|
||||
// Omitted lines should be most-but-not-all of the original.
|
||||
expect(reportedOmittedLines).toBeGreaterThan(0);
|
||||
expect(reportedOmittedLines).toBeLessThan(originalLines);
|
||||
});
|
||||
|
||||
it('always shows at least one head line even on degenerate input', () => {
|
||||
// Single huge line — bigger than budget. Should still render with marker.
|
||||
const text = 'x'.repeat(500_000);
|
||||
const { text: out, truncated } = truncateForBudget(text, 10, COLS);
|
||||
// No newlines means lines.length === 1, so "truncation" can only show
|
||||
// that single line. Verify behavior is sane (doesn't crash, marker
|
||||
// present somewhere if truncated).
|
||||
if (truncated) {
|
||||
expect(out).toContain('pixelpipe paging:');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// -- end-to-end through transformRequest -----------------------------------
|
||||
|
||||
function makeReq(toolResultText: string) {
|
||||
return new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
model: 'claude-3-5-sonnet',
|
||||
// Force compression to fire: need a system slab past the per-block
|
||||
// break-even (≥10k chars) so the main static-slab compression runs
|
||||
// and `info.compressed` flips to true. Smaller slabs no-op out via
|
||||
// isCompressionProfitable and the test wouldn't see compressed=true.
|
||||
system: 'x'.repeat(60_000),
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'tool_result', tool_use_id: 'toolu_x', content: toolResultText },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe('paging end-to-end (transformRequest)', () => {
|
||||
it('tool_result under cap renders normally (no truncation counters)', async () => {
|
||||
// Above 10k break-even, well under the 10-image budget (~140k chars).
|
||||
const text = 'x'.repeat(40_000);
|
||||
const { info } = await transformRequest(makeReq(text));
|
||||
expect(info.compressed).toBe(true);
|
||||
expect((info.toolResultImgs ?? 0)).toBeGreaterThan(0);
|
||||
expect(info.truncatedToolResults ?? 0).toBe(0);
|
||||
expect(info.omittedChars ?? 0).toBe(0);
|
||||
});
|
||||
|
||||
it('tool_result over cap fires truncation, lands ≤ 10 images', async () => {
|
||||
// ~500k char log → ~36 raw images, should clamp to ≤10.
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 10_000; i++) {
|
||||
lines.push(`2026-05-18T12:00:00Z entry ${i} payload content here`);
|
||||
}
|
||||
const log = lines.join('\n');
|
||||
expect(log.length).toBeGreaterThan(400_000);
|
||||
|
||||
const { info } = await transformRequest(makeReq(log));
|
||||
expect(info.compressed).toBe(true);
|
||||
expect(info.truncatedToolResults).toBe(1);
|
||||
expect(info.omittedChars).toBeGreaterThan(0);
|
||||
// Image count for this tool_result should be capped at the budget.
|
||||
// (Allow 1-image slack for the marker / rounding.)
|
||||
expect(info.toolResultImgs).toBeLessThanOrEqual(11);
|
||||
});
|
||||
|
||||
it('respects a custom maxImagesPerToolResult option', async () => {
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 10_000; i++) {
|
||||
lines.push(`2026-05-18T12:00:00Z entry ${i} payload content here`);
|
||||
}
|
||||
const log = lines.join('\n');
|
||||
|
||||
// Tight budget of 2 images = ~28k chars.
|
||||
const { info } = await transformRequest(makeReq(log), {
|
||||
maxImagesPerToolResult: 2,
|
||||
});
|
||||
expect(info.truncatedToolResults).toBe(1);
|
||||
expect(info.toolResultImgs).toBeLessThanOrEqual(3); // 2 + slack
|
||||
});
|
||||
|
||||
it('counts multiple tool_results that all exceed the budget', async () => {
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 10_000; i++) {
|
||||
lines.push(`2026-05-18T12:00:00Z entry ${i} payload content here`);
|
||||
}
|
||||
const log = lines.join('\n');
|
||||
|
||||
// Two big tool_results in one request.
|
||||
const req = new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
model: 'claude-3-5-sonnet',
|
||||
system: 'x'.repeat(60_000),
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'tool_result', tool_use_id: 'toolu_a', content: log },
|
||||
{ type: 'tool_result', tool_use_id: 'toolu_b', content: log },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { info } = await transformRequest(req);
|
||||
expect(info.truncatedToolResults).toBe(2);
|
||||
// Both should have been truncated → omittedChars roughly doubled.
|
||||
expect(info.omittedChars).toBeGreaterThan(800_000 - 30_000);
|
||||
});
|
||||
|
||||
it('handles array-shaped tool_result content', async () => {
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 10_000; i++) {
|
||||
lines.push(`2026-05-18T12:00:00Z entry ${i} payload content here`);
|
||||
}
|
||||
const log = lines.join('\n');
|
||||
|
||||
// Array shape: tool_result content is [{type: 'text', text: ...}]
|
||||
const req = new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
model: 'claude-3-5-sonnet',
|
||||
system: 'x'.repeat(60_000),
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'toolu_x',
|
||||
content: [{ type: 'text', text: log }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { info } = await transformRequest(req);
|
||||
expect(info.truncatedToolResults).toBe(1);
|
||||
expect(info.omittedChars).toBeGreaterThan(0);
|
||||
expect(info.toolResultImgs).toBeLessThanOrEqual(11);
|
||||
});
|
||||
});
|
||||
|
||||
// -- tracker wire-through ---------------------------------------------------
|
||||
|
||||
describe('paging telemetry → TrackEvent', () => {
|
||||
it('forwards truncated_tool_results and omitted_chars when set', () => {
|
||||
const ev: ProxyEvent = {
|
||||
method: 'POST',
|
||||
path: '/v1/messages',
|
||||
status: 200,
|
||||
durationMs: 100,
|
||||
info: {
|
||||
compressed: true,
|
||||
origChars: 500_000,
|
||||
imageCount: 10,
|
||||
imageBytes: 20_000,
|
||||
staticChars: 0,
|
||||
dynamicChars: 0,
|
||||
dynamicBlockCount: 0,
|
||||
truncatedToolResults: 2,
|
||||
omittedChars: 350_000,
|
||||
},
|
||||
};
|
||||
const out = toTrackEvent(ev);
|
||||
expect(out.truncated_tool_results).toBe(2);
|
||||
expect(out.omitted_chars).toBe(350_000);
|
||||
});
|
||||
|
||||
it('omits the fields when no truncation fired (zero / undefined)', () => {
|
||||
const ev: ProxyEvent = {
|
||||
method: 'POST',
|
||||
path: '/v1/messages',
|
||||
status: 200,
|
||||
durationMs: 100,
|
||||
info: {
|
||||
compressed: true,
|
||||
origChars: 10_000,
|
||||
imageCount: 1,
|
||||
imageBytes: 2_000,
|
||||
staticChars: 0,
|
||||
dynamicChars: 0,
|
||||
dynamicBlockCount: 0,
|
||||
// truncatedToolResults: undefined
|
||||
// omittedChars: undefined
|
||||
},
|
||||
};
|
||||
const out = toTrackEvent(ev);
|
||||
expect(out.truncated_tool_results).toBeUndefined();
|
||||
expect(out.omitted_chars).toBeUndefined();
|
||||
});
|
||||
|
||||
it('omits the fields when explicitly zero (no-op truncation pass)', () => {
|
||||
const ev: ProxyEvent = {
|
||||
method: 'POST',
|
||||
path: '/v1/messages',
|
||||
status: 200,
|
||||
durationMs: 100,
|
||||
info: {
|
||||
compressed: true,
|
||||
origChars: 10_000,
|
||||
imageCount: 1,
|
||||
imageBytes: 2_000,
|
||||
staticChars: 0,
|
||||
dynamicChars: 0,
|
||||
dynamicBlockCount: 0,
|
||||
truncatedToolResults: 0,
|
||||
omittedChars: 0,
|
||||
},
|
||||
};
|
||||
const out = toTrackEvent(ev);
|
||||
// Skipped because the wire-through gate is `> 0`.
|
||||
expect(out.truncated_tool_results).toBeUndefined();
|
||||
expect(out.omitted_chars).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+48
-1
@@ -6,7 +6,7 @@ import {
|
||||
minifyForRender,
|
||||
} from '../src/core/render.js';
|
||||
import { encodeGrayPng, bytesToBase64 } from '../src/core/png.js';
|
||||
import { transformRequest, isCompressionProfitable } from '../src/core/transform.js';
|
||||
import { transformRequest, isCompressionProfitable, maxCharsPerImage } from '../src/core/transform.js';
|
||||
import {
|
||||
atlasRank,
|
||||
ATLAS_CELL_H,
|
||||
@@ -1499,6 +1499,53 @@ describe('transform', () => {
|
||||
expect(isCompressionProfitable(40000)).toBe(true);
|
||||
});
|
||||
|
||||
// --- Adaptive break-even: CHARS_PER_IMAGE derived from atlas cell, not hardcoded ---
|
||||
// Brief: when font-rater swaps to a smaller cell (e.g. Cozette 4×7), more chars
|
||||
// pack into one image, so the N-image break-even thresholds shift. Tests below
|
||||
// verify both the regression case (current Unifont 5×11) AND that the formula
|
||||
// responds to `cols` (which scales chars/image linearly the same way a smaller
|
||||
// cell-H would).
|
||||
|
||||
it('maxCharsPerImage: matches the historic 14,100 constant at the shipping config', () => {
|
||||
// Unifont 5×11, cols=100 → floor((1568−8)/11) × 100 = 141 × 100 = 14,100.
|
||||
// If this ever drifts, every break-even test downstream needs re-pinning.
|
||||
expect(maxCharsPerImage(100)).toBe(14_100);
|
||||
});
|
||||
|
||||
it('maxCharsPerImage: scales linearly with cols (same atlas)', () => {
|
||||
expect(maxCharsPerImage(50)).toBe(7_050);
|
||||
expect(maxCharsPerImage(200)).toBe(28_200);
|
||||
});
|
||||
|
||||
it('isCompressionProfitable: doubling cols halves the 2-image break-even threshold', () => {
|
||||
// At cols=100, CHARS_PER_IMAGE=14,100. 20,000 chars needs 2 images (cost
|
||||
// 5000 tokens) vs 5000 text-tokens → tied, strict `<` returns false.
|
||||
expect(isCompressionProfitable(20_000, 100)).toBe(false);
|
||||
// At cols=200, CHARS_PER_IMAGE=28,200. 20,000 chars fits in 1 image
|
||||
// (cost 2500 tokens) vs 5000 text-tokens → clear win.
|
||||
expect(isCompressionProfitable(20_000, 200)).toBe(true);
|
||||
});
|
||||
|
||||
it('isCompressionProfitable: tiny-cols config raises the break-even threshold', () => {
|
||||
// Simulated narrow render: cols=20 → CHARS_PER_IMAGE=2820. A 10,001-char
|
||||
// block needs ceil(10001/2820)=4 images (10,000 tokens) vs 2500 text →
|
||||
// huge net loss. At cols=100 the same block was profitable.
|
||||
expect(isCompressionProfitable(10_001, 100)).toBe(true);
|
||||
expect(isCompressionProfitable(10_001, 20)).toBe(false);
|
||||
});
|
||||
|
||||
it('isCompressionProfitable: smaller-cell atlas (Cozette-shape) would let 16k blocks become 1-image wins (cols proxy)', () => {
|
||||
// True smaller-cell test would need to mock ATLAS_CELL_H. We use cols as
|
||||
// a proxy since CHARS_PER_IMAGE = cols × floor((1568−8)/cell_H) — doubling
|
||||
// cols at fixed cell_H is mathematically the same as halving cell_H at
|
||||
// fixed cols. A Cozette 4×7 cell at cols=100 yields floor(1560/7)×100 =
|
||||
// 22,200 chars/image, ~57% more than today. Equivalent: cols=157 at the
|
||||
// current cell. A 16,000-char block needs 2 images today (2-image break-
|
||||
// even fails); at the equivalent Cozette-shape config it fits in 1.
|
||||
expect(isCompressionProfitable(16_000, 100)).toBe(false); // 2 imgs @ 5000 vs 4000 text
|
||||
expect(isCompressionProfitable(16_000, 157)).toBe(true); // 1 img @ 2500 vs 4000 text
|
||||
});
|
||||
|
||||
it('break-even gate: 7000-char tool_result stays as text (below break-even)', async () => {
|
||||
// Above the old 5000 minToolResultChars cutoff but still net-loss to
|
||||
// image (image=2500 > text=7000/4=1750). The fast-path threshold (now
|
||||
|
||||
Reference in New Issue
Block a user