mirror of
https://github.com/sdwolf4103/opencode-working-memory.git
synced 2026-06-02 06:19:36 +02:00
feat(explainability): add diagnostics JSON, per-memory explain, lifecycle trace
Phase 4 Tasks 4.1-4.3: - memory-diag health --json: machine-readable MemoryDiagJSON output - memory-diag explain: per-memory render status with strength, reasons, evidence event IDs - memory-diag trace --memory <id>: lifecycle history from evidence events and relations (superseded_by, reinforced_by) - MemoryRenderStatus type with 9 statuses - All diagnostics are read-only, no storage mutations - Privacy-safe: redacted text previews, no raw secrets - 270 tests pass, typecheck pass
This commit is contained in:
Ralph Chang
+370
-5
@@ -12,7 +12,7 @@ import { dataHome, extractionRejectionLogPath, migrationLogPath, workspaceKey, w
|
||||
import { assessMemoryQuality, HARD_QUALITY_REASONS } from "../src/memory-quality.ts";
|
||||
import { redactCredentials } from "../src/redaction.ts";
|
||||
import { scanWorkspaceResidues } from "../src/workspace-cleanup.ts";
|
||||
import { renderWorkspaceMemory } from "../src/workspace-memory.ts";
|
||||
import { accountWorkspaceMemoryRender, renderWorkspaceMemory } from "../src/workspace-memory.ts";
|
||||
import {
|
||||
DORMANT_DECAY_MULTIPLIER,
|
||||
RETENTION_TYPE_MAX,
|
||||
@@ -21,18 +21,70 @@ import {
|
||||
} from "../src/retention.ts";
|
||||
import type { LongTermMemoryEntry, LongTermSource, LongTermType, PendingMemoryJournalStore, WorkspaceMemoryStore } from "../src/types.ts";
|
||||
import { LONG_TERM_LIMITS, PROMOTION_RETRY_LIMITS } from "../src/types.ts";
|
||||
import {
|
||||
queryEvidenceEvents,
|
||||
summarizeMemoryEvidence,
|
||||
traceMemoryLifecycle,
|
||||
type EvidenceEventType,
|
||||
type EvidenceEventV1,
|
||||
type EvidenceOutcome,
|
||||
} from "../src/evidence-log.ts";
|
||||
|
||||
type Command = "health" | "rejections" | "audit";
|
||||
export type MemoryRenderStatus =
|
||||
| "rendered"
|
||||
| "omitted_superseded"
|
||||
| "omitted_type_cap"
|
||||
| "omitted_global_cap"
|
||||
| "omitted_char_budget"
|
||||
| "omitted_absorbed_duplicate"
|
||||
| "pending_retry"
|
||||
| "pending_rejected_capacity"
|
||||
| "quarantined_corrupt_store";
|
||||
|
||||
export type MemoryDiagJSON = {
|
||||
version: 1;
|
||||
workspace: { rootHash: string; key: string };
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
storedActive: number;
|
||||
rendered: number;
|
||||
pending: number;
|
||||
rejectedLast7Days: number;
|
||||
corruptStoresQuarantinedLast30Days: number;
|
||||
};
|
||||
memories: Array<{
|
||||
id: string;
|
||||
type: "feedback" | "project" | "decision" | "reference";
|
||||
source: "explicit" | "compaction" | "manual";
|
||||
status: MemoryRenderStatus;
|
||||
strength?: number;
|
||||
reasonCodes: string[];
|
||||
textPreview?: string;
|
||||
evidenceEventIds: string[];
|
||||
}>;
|
||||
recentEvents: Array<{
|
||||
eventId: string;
|
||||
type: EvidenceEventType;
|
||||
outcome: EvidenceOutcome;
|
||||
createdAt: string;
|
||||
memoryId?: string;
|
||||
reasonCodes: string[];
|
||||
}>;
|
||||
};
|
||||
|
||||
type Command = "health" | "rejections" | "audit" | "explain" | "trace";
|
||||
type Origin = "explicit_trigger" | "compaction_candidate" | "manual" | "migration_check" | "unknown";
|
||||
|
||||
type CliOptions = {
|
||||
raw: boolean;
|
||||
json?: boolean;
|
||||
workspace?: string;
|
||||
all?: boolean;
|
||||
softOnly?: boolean;
|
||||
triggerOnly?: boolean;
|
||||
since?: string;
|
||||
migration?: string;
|
||||
memory?: string;
|
||||
};
|
||||
|
||||
type RejectionLogRecord = {
|
||||
@@ -89,7 +141,9 @@ const ALLOWED_ORIGINS = new Set<Origin>([
|
||||
|
||||
function usage(): string {
|
||||
return `Usage:
|
||||
bun scripts/memory-diag.ts health [--workspace <path>] [--all] [--raw]
|
||||
bun scripts/memory-diag.ts health [--workspace <path>] [--all] [--raw] [--json]
|
||||
bun scripts/memory-diag.ts explain [--workspace <path>] [--raw]
|
||||
bun scripts/memory-diag.ts trace --memory <id> [--workspace <path>] [--raw]
|
||||
bun scripts/memory-diag.ts rejections [--soft-only] [--trigger-only] [--since 14d] [--raw]
|
||||
bun scripts/memory-diag.ts audit [--migration <id>] [--raw]
|
||||
`;
|
||||
@@ -107,7 +161,7 @@ function parseArgs(argv: string[]): { command: Command; options: CliOptions } {
|
||||
console.log(usage());
|
||||
process.exit(0);
|
||||
}
|
||||
if (command !== "health" && command !== "rejections" && command !== "audit") {
|
||||
if (command !== "health" && command !== "rejections" && command !== "audit" && command !== "explain" && command !== "trace") {
|
||||
die(`Unknown subcommand: ${command}`);
|
||||
}
|
||||
|
||||
@@ -115,6 +169,7 @@ function parseArgs(argv: string[]): { command: Command; options: CliOptions } {
|
||||
for (let i = 0; i < rest.length; i += 1) {
|
||||
const arg = rest[i];
|
||||
if (arg === "--raw") options.raw = true;
|
||||
else if (arg === "--json") options.json = true;
|
||||
else if (arg === "--all") options.all = true;
|
||||
else if (arg === "--soft-only") options.softOnly = true;
|
||||
else if (arg === "--trigger-only") options.triggerOnly = true;
|
||||
@@ -130,6 +185,10 @@ function parseArgs(argv: string[]): { command: Command; options: CliOptions } {
|
||||
const value = rest[++i];
|
||||
if (!value) die("--migration requires an id");
|
||||
options.migration = value;
|
||||
} else if (arg === "--memory") {
|
||||
const value = rest[++i];
|
||||
if (!value) die("--memory requires an id");
|
||||
options.memory = value;
|
||||
} else {
|
||||
die(`Unknown option: ${arg}`);
|
||||
}
|
||||
@@ -137,15 +196,25 @@ function parseArgs(argv: string[]): { command: Command; options: CliOptions } {
|
||||
|
||||
if (command === "health") {
|
||||
if (options.all && options.workspace) die("Use either --all or --workspace, not both");
|
||||
if (options.json && options.all) die("health --json does not support --all");
|
||||
} else if (command === "explain" || command === "trace") {
|
||||
if (options.all) die(`${command} does not accept --all`);
|
||||
} else {
|
||||
if (options.all || options.workspace) die(`${command} does not accept --all or --workspace`);
|
||||
}
|
||||
if (command !== "health" && options.json) die(`${command} does not accept --json`);
|
||||
if (command !== "rejections" && (options.softOnly || options.triggerOnly || options.since)) {
|
||||
die(`${command} does not accept rejection filters`);
|
||||
}
|
||||
if (command !== "audit" && options.migration) {
|
||||
die(`${command} does not accept --migration`);
|
||||
}
|
||||
if (command !== "trace" && options.memory) {
|
||||
die(`${command} does not accept --memory`);
|
||||
}
|
||||
if (command === "trace" && !options.memory) {
|
||||
die("--memory requires an id");
|
||||
}
|
||||
|
||||
return { command, options };
|
||||
}
|
||||
@@ -335,7 +404,186 @@ function normalizedJournal(journal: PendingMemoryJournalStore | null): PendingMe
|
||||
};
|
||||
}
|
||||
|
||||
type WorkspaceDiagSnapshot = {
|
||||
store: WorkspaceMemoryStore;
|
||||
journal: PendingMemoryJournalStore;
|
||||
retention: ReturnType<typeof retentionCandidatesForDiag>;
|
||||
memories: MemoryDiagJSON["memories"];
|
||||
recentEvents: MemoryDiagJSON["recentEvents"];
|
||||
allEvents: EvidenceEventV1[];
|
||||
summary: MemoryDiagJSON["summary"];
|
||||
};
|
||||
|
||||
function uniqueStrings(values: string[]): string[] {
|
||||
return [...new Set(values.filter(Boolean))];
|
||||
}
|
||||
|
||||
function eventMemoryId(event: EvidenceEventV1): string | undefined {
|
||||
return event.memory?.memoryId
|
||||
?? event.relations?.map(relation => relation.memory?.memoryId).find((id): id is string => Boolean(id));
|
||||
}
|
||||
|
||||
function isWithinDays(iso: string, days: number): boolean {
|
||||
const ms = new Date(iso).getTime();
|
||||
return Number.isFinite(ms) && ms >= Date.now() - days * 86_400_000;
|
||||
}
|
||||
|
||||
function renderStatusReason(status: MemoryRenderStatus, fallback?: string): string[] {
|
||||
switch (status) {
|
||||
case "rendered": return ["within_caps", "within_char_budget"];
|
||||
case "omitted_superseded": return ["superseded"];
|
||||
case "omitted_type_cap": return ["type_cap"];
|
||||
case "omitted_global_cap": return ["global_cap"];
|
||||
case "omitted_char_budget": return [fallback === "empty_render_budget" ? "empty_render_budget" : "char_budget"];
|
||||
case "omitted_absorbed_duplicate": return ["absorbed_duplicate"];
|
||||
case "pending_retry": return ["retryable_capacity_rejection"];
|
||||
case "pending_rejected_capacity": return ["capacity_rejected", "max_attempts_reached"];
|
||||
case "quarantined_corrupt_store": return ["invalid_json"];
|
||||
}
|
||||
}
|
||||
|
||||
function statusFromOmissionReason(reason: string | undefined): MemoryRenderStatus {
|
||||
if (reason === "superseded") return "omitted_superseded";
|
||||
if (reason === "type_cap") return "omitted_type_cap";
|
||||
if (reason === "global_cap") return "omitted_global_cap";
|
||||
return "omitted_char_budget";
|
||||
}
|
||||
|
||||
function pendingStatus(entry: LongTermMemoryEntry): MemoryRenderStatus {
|
||||
const attempts = entry.promotionAttempts ?? 0;
|
||||
return attempts >= promotionLimit(entry.source) ? "pending_rejected_capacity" : "pending_retry";
|
||||
}
|
||||
|
||||
function safeTextPreview(text: string): string {
|
||||
return truncate(cleanText(text, false), 120);
|
||||
}
|
||||
|
||||
async function buildWorkspaceDiagSnapshot(input: {
|
||||
root: string;
|
||||
key: string;
|
||||
memoryPath: string;
|
||||
pendingPath: string;
|
||||
}): Promise<WorkspaceDiagSnapshot> {
|
||||
const rawStore = await readJSONFile<WorkspaceMemoryStore>(input.memoryPath);
|
||||
const storeRoot = rawStore?.workspace?.root ?? input.root;
|
||||
const storeKey = rawStore?.workspace?.key ?? input.key;
|
||||
const store = normalizedStore(rawStore, storeRoot, storeKey);
|
||||
const journal = normalizedJournal(await readJSONFile<PendingMemoryJournalStore>(input.pendingPath));
|
||||
const retention = retentionCandidatesForDiag(store);
|
||||
const renderAccounting = accountWorkspaceMemoryRender(store);
|
||||
const renderedIds = new Set(renderAccounting.rendered.map(memory => memory.id));
|
||||
const omittedById = new Map(renderAccounting.omitted.map(item => [item.memory.id, item.reason]));
|
||||
const allEvents = await queryEvidenceEvents(input.root);
|
||||
const recentEvidence = await queryEvidenceEvents(input.root, { newestFirst: true, limit: 50 });
|
||||
const memoryRows: MemoryDiagJSON["memories"] = [];
|
||||
const seenIds = new Set<string>();
|
||||
|
||||
for (const entry of store.entries) {
|
||||
const omissionReason = omittedById.get(entry.id);
|
||||
const status: MemoryRenderStatus = renderedIds.has(entry.id)
|
||||
? "rendered"
|
||||
: statusFromOmissionReason(omissionReason ?? (entry.status === "superseded" ? "superseded" : undefined));
|
||||
const summary = await summarizeMemoryEvidence(input.root, { memoryId: entry.id });
|
||||
memoryRows.push({
|
||||
id: entry.id,
|
||||
type: entry.type,
|
||||
source: entry.source,
|
||||
status,
|
||||
strength: calculateRetentionStrength(entry, Date.now(), store.lastActivityAt),
|
||||
reasonCodes: uniqueStrings([...renderStatusReason(status, omissionReason), ...summary.reasonCodes]),
|
||||
textPreview: safeTextPreview(entry.text),
|
||||
evidenceEventIds: summary.eventIds,
|
||||
});
|
||||
seenIds.add(entry.id);
|
||||
}
|
||||
|
||||
for (const entry of journal.entries) {
|
||||
const status = pendingStatus(entry);
|
||||
const summary = await summarizeMemoryEvidence(input.root, { memoryId: entry.id });
|
||||
memoryRows.push({
|
||||
id: entry.id,
|
||||
type: entry.type,
|
||||
source: entry.source,
|
||||
status,
|
||||
strength: calculateRetentionStrength(entry, Date.now(), store.lastActivityAt),
|
||||
reasonCodes: uniqueStrings([...renderStatusReason(status), entry.lastPromotionFailureReason ?? "", ...summary.reasonCodes]),
|
||||
textPreview: safeTextPreview(entry.text),
|
||||
evidenceEventIds: summary.eventIds,
|
||||
});
|
||||
seenIds.add(entry.id);
|
||||
}
|
||||
|
||||
for (const event of allEvents) {
|
||||
if (event.outcome !== "absorbed") continue;
|
||||
const memory = event.memory;
|
||||
if (!memory?.memoryId || !memory.type || !memory.source || seenIds.has(memory.memoryId)) continue;
|
||||
const summary = await summarizeMemoryEvidence(input.root, { memoryId: memory.memoryId });
|
||||
memoryRows.push({
|
||||
id: memory.memoryId,
|
||||
type: memory.type,
|
||||
source: memory.source,
|
||||
status: "omitted_absorbed_duplicate",
|
||||
reasonCodes: uniqueStrings([...renderStatusReason("omitted_absorbed_duplicate"), ...summary.reasonCodes]),
|
||||
evidenceEventIds: summary.eventIds.length > 0 ? summary.eventIds : [event.eventId],
|
||||
});
|
||||
seenIds.add(memory.memoryId);
|
||||
}
|
||||
|
||||
const recentEvents = recentEvidence.map(event => ({
|
||||
eventId: event.eventId,
|
||||
type: event.type,
|
||||
outcome: event.outcome,
|
||||
createdAt: event.createdAt,
|
||||
memoryId: eventMemoryId(event),
|
||||
reasonCodes: uniqueStrings([
|
||||
...event.reasonCodes,
|
||||
...(event.type === "storage_corrupt_json_quarantined" ? ["quarantined_corrupt_store"] : []),
|
||||
]),
|
||||
}));
|
||||
|
||||
return {
|
||||
store,
|
||||
journal,
|
||||
retention,
|
||||
memories: memoryRows,
|
||||
recentEvents,
|
||||
allEvents,
|
||||
summary: {
|
||||
storedActive: store.entries.filter(entry => entry.status !== "superseded").length,
|
||||
rendered: retention.rendered.length,
|
||||
pending: journal.entries.length,
|
||||
rejectedLast7Days: allEvents.filter(event => event.outcome === "rejected" && isWithinDays(event.createdAt, 7)).length,
|
||||
corruptStoresQuarantinedLast30Days: allEvents.filter(event => event.type === "storage_corrupt_json_quarantined" && isWithinDays(event.createdAt, 30)).length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function buildMemoryDiagJSON(root: string): Promise<MemoryDiagJSON> {
|
||||
const key = await workspaceKey(root);
|
||||
const snapshot = await buildWorkspaceDiagSnapshot({
|
||||
root,
|
||||
key,
|
||||
memoryPath: await workspaceMemoryPath(root),
|
||||
pendingPath: await workspacePendingJournalPath(root),
|
||||
});
|
||||
|
||||
return {
|
||||
version: 1,
|
||||
workspace: { rootHash: workspaceRootHash(snapshot.store.workspace.root || root), key: snapshot.store.workspace.key || key },
|
||||
generatedAt: new Date().toISOString(),
|
||||
summary: snapshot.summary,
|
||||
memories: snapshot.memories,
|
||||
recentEvents: snapshot.recentEvents,
|
||||
};
|
||||
}
|
||||
|
||||
async function runHealth(options: CliOptions): Promise<void> {
|
||||
if (options.json) {
|
||||
const root = options.workspace ?? process.cwd();
|
||||
console.log(JSON.stringify(await buildMemoryDiagJSON(root), null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.all) {
|
||||
const scan = await scanWorkspaceResidues({ includeOrphans: true, minAgeMs: 0 });
|
||||
console.log("Workspace memory health");
|
||||
@@ -707,8 +955,125 @@ async function runAudit(options: CliOptions): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
async function snapshotForOptions(options: CliOptions): Promise<WorkspaceDiagSnapshot> {
|
||||
const root = options.workspace ?? process.cwd();
|
||||
const key = await workspaceKey(root);
|
||||
return buildWorkspaceDiagSnapshot({
|
||||
root,
|
||||
key,
|
||||
memoryPath: await workspaceMemoryPath(root),
|
||||
pendingPath: await workspacePendingJournalPath(root),
|
||||
});
|
||||
}
|
||||
|
||||
function formatEvidenceRefs(eventIds: string[], allEvents: EvidenceEventV1[]): string {
|
||||
if (eventIds.length === 0) return "(none)";
|
||||
const byId = new Map(allEvents.map(event => [event.eventId, event]));
|
||||
return eventIds
|
||||
.map(id => {
|
||||
const event = byId.get(id);
|
||||
return event ? `${event.eventId} ${event.type}` : id;
|
||||
})
|
||||
.join(", ");
|
||||
}
|
||||
|
||||
async function runExplain(options: CliOptions): Promise<void> {
|
||||
const snapshot = await snapshotForOptions(options);
|
||||
console.log("Workspace memory explainability");
|
||||
console.log("");
|
||||
|
||||
if (snapshot.memories.length === 0) {
|
||||
console.log("No memories found.");
|
||||
}
|
||||
|
||||
for (const memory of snapshot.memories) {
|
||||
console.log(`Memory ${memory.id}: ${memory.status}`);
|
||||
const strength = typeof memory.strength === "number" ? formatStrength(memory.strength) : "n/a";
|
||||
console.log(`- strength=${strength}, type=${memory.type}, source=${memory.source}`);
|
||||
console.log(`- reasons: ${memory.reasonCodes.length > 0 ? memory.reasonCodes.join(", ") : "(none)"}`);
|
||||
console.log(`- evidence: ${formatEvidenceRefs(memory.evidenceEventIds, snapshot.allEvents)}`);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
const quarantines = snapshot.recentEvents.filter(event => event.type === "storage_corrupt_json_quarantined");
|
||||
if (quarantines.length > 0) {
|
||||
console.log("Quarantined stores:");
|
||||
for (const event of quarantines) {
|
||||
console.log(`- quarantined_corrupt_store: ${event.eventId} ${event.type}; reasons=${event.reasonCodes.join(",")}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function statusFromTraceEvent(event: EvidenceEventV1 | undefined): string {
|
||||
if (!event) return "unknown";
|
||||
if (event.type === "render_selected") return "rendered";
|
||||
if (event.type === "render_omitted") return statusFromOmissionReason(event.reasonCodes[0]);
|
||||
if (event.type === "promotion_absorbed_exact" || event.type === "promotion_absorbed_identity") return "omitted_absorbed_duplicate";
|
||||
if (event.type === "promotion_retry_scheduled") return "pending_retry";
|
||||
if (event.type === "promotion_rejected_capacity" || event.type === "promotion_retry_exhausted") return "pending_rejected_capacity";
|
||||
if (event.type === "storage_corrupt_json_quarantined") return "quarantined_corrupt_store";
|
||||
if (event.outcome === "superseded") return "omitted_superseded";
|
||||
return event.outcome;
|
||||
}
|
||||
|
||||
function formatTraceEvent(event: EvidenceEventV1): string {
|
||||
const reasons = event.reasonCodes.length > 0 ? event.reasonCodes.join(",") : "none";
|
||||
const relations = (event.relations ?? [])
|
||||
.map(relation => relation.memory?.memoryId ? `${relation.role}=${relation.memory.memoryId}` : undefined)
|
||||
.filter((value): value is string => Boolean(value));
|
||||
const relationText = relations.length > 0 ? `; ${relations.join(", ")}` : "";
|
||||
return `- ${event.eventId} ${event.type}: ${event.outcome}; reasons=${reasons}${relationText}`;
|
||||
}
|
||||
|
||||
function relationMemoryIds(events: EvidenceEventV1[], role: string): string[] {
|
||||
return uniqueStrings(events.flatMap(event => (event.relations ?? [])
|
||||
.filter(relation => relation.role === role)
|
||||
.map(relation => relation.memory?.memoryId ?? "")));
|
||||
}
|
||||
|
||||
async function runTrace(options: CliOptions): Promise<void> {
|
||||
const root = options.workspace ?? process.cwd();
|
||||
const memoryId = options.memory;
|
||||
if (!memoryId) die("--memory requires an id");
|
||||
|
||||
const [snapshot, trace] = await Promise.all([
|
||||
snapshotForOptions(options),
|
||||
traceMemoryLifecycle(root, { memoryId }),
|
||||
]);
|
||||
const memoryRow = snapshot.memories.find(memory => memory.id === memoryId);
|
||||
const status = memoryRow?.status ?? statusFromTraceEvent(trace.events.at(-1));
|
||||
|
||||
console.log(`Memory ${memoryId}: ${status}`);
|
||||
console.log("");
|
||||
console.log("Lifecycle:");
|
||||
if (trace.events.length === 0) {
|
||||
console.log("(none)");
|
||||
return;
|
||||
}
|
||||
|
||||
for (const event of trace.events) {
|
||||
console.log(formatTraceEvent(event));
|
||||
}
|
||||
|
||||
const supersededBy = relationMemoryIds(trace.events, "superseded_by");
|
||||
if (supersededBy.length > 0) {
|
||||
console.log("");
|
||||
console.log("Superseded by:");
|
||||
for (const id of supersededBy) console.log(`- ${id}`);
|
||||
}
|
||||
|
||||
const reinforcedBy = relationMemoryIds(trace.events, "reinforced_by");
|
||||
if (reinforcedBy.length > 0) {
|
||||
console.log("");
|
||||
console.log("Reinforced by:");
|
||||
for (const id of reinforcedBy) console.log(`- ${id}`);
|
||||
}
|
||||
}
|
||||
|
||||
const { command, options } = parseArgs(process.argv.slice(2));
|
||||
|
||||
if (command === "health") await runHealth(options);
|
||||
else if (command === "rejections") await runRejections(options);
|
||||
else await runAudit(options);
|
||||
else if (command === "audit") await runAudit(options);
|
||||
else if (command === "explain") await runExplain(options);
|
||||
else await runTrace(options);
|
||||
|
||||
+175
-6
@@ -1,14 +1,15 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { appendEvidenceEvents, type EvidenceEventInput } from "../src/evidence-log.ts";
|
||||
import type { LongTermMemoryEntry, WorkspaceMemoryStore } from "../src/types.ts";
|
||||
import { LONG_TERM_LIMITS } from "../src/types.ts";
|
||||
import { workspaceKey, workspaceMemoryPath } from "../src/paths.ts";
|
||||
import { LONG_TERM_LIMITS, PROMOTION_RETRY_LIMITS, type PendingMemoryJournalStore } from "../src/types.ts";
|
||||
import { workspaceKey, workspaceMemoryPath, workspacePendingJournalPath } from "../src/paths.ts";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
@@ -46,17 +47,43 @@ async function writeWorkspaceStore(root: string, entries: LongTermMemoryEntry[],
|
||||
}
|
||||
|
||||
async function runMemoryDiagHealth(root: string): Promise<string> {
|
||||
return runMemoryDiag(["health", "--workspace", root]);
|
||||
}
|
||||
|
||||
async function runMemoryDiag(args: string[]): Promise<string> {
|
||||
const { stdout } = await execFileAsync(process.execPath, [
|
||||
"--experimental-strip-types",
|
||||
"scripts/memory-diag.ts",
|
||||
"health",
|
||||
"--workspace",
|
||||
root,
|
||||
...args,
|
||||
], { cwd: repoRoot });
|
||||
|
||||
return stdout;
|
||||
}
|
||||
|
||||
async function writePendingJournal(root: string, entries: LongTermMemoryEntry[]): Promise<void> {
|
||||
const key = await workspaceKey(root);
|
||||
const path = await workspacePendingJournalPath(root);
|
||||
const store: PendingMemoryJournalStore = {
|
||||
version: 1,
|
||||
workspace: { root, key },
|
||||
entries,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, JSON.stringify(store, null, 2), "utf8");
|
||||
}
|
||||
|
||||
function evidence(overrides: Partial<EvidenceEventInput>): EvidenceEventInput {
|
||||
return {
|
||||
type: "promotion_promoted",
|
||||
phase: "promotion",
|
||||
outcome: "promoted",
|
||||
memory: { memoryId: "mem-rendered", type: "feedback", source: "explicit", status: "active" },
|
||||
reasonCodes: ["new_workspace_entry"],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("memory health reports stored vs rendered retention counts", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-"));
|
||||
try {
|
||||
@@ -141,3 +168,145 @@ test("memory health reports missing dormancy and non-alert monitoring defaults",
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("memory health --json prints parseable privacy-safe diagnostics matching human counts", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-json-"));
|
||||
try {
|
||||
const rendered = { ...entry("mem-rendered", "Prefer small focused changes", "feedback"), source: "explicit" as const };
|
||||
const secret = { ...entry("mem-secret", "Use password: sushi only in test fixtures", "decision"), source: "manual" as const };
|
||||
const superseded = { ...entry("mem-old", "Old decision that was superseded", "decision"), status: "superseded" as const };
|
||||
const pending = { ...entry("mem-pending", "Retry this pending memory later", "project"), promotionAttempts: 1 };
|
||||
await writeWorkspaceStore(root, [rendered, secret, superseded]);
|
||||
await writePendingJournal(root, [pending]);
|
||||
await appendEvidenceEvents(root, [
|
||||
evidence({ type: "extraction_candidate_rejected", phase: "extraction", outcome: "rejected", memory: { memoryId: "mem-rejected", type: "feedback", source: "explicit" }, reasonCodes: ["raw_secret"], textPreview: "password: sushi should not leak" }),
|
||||
evidence({ type: "storage_corrupt_json_quarantined", phase: "storage", outcome: "quarantined", memory: undefined, reasonCodes: ["invalid_json"] }),
|
||||
evidence({ type: "promotion_promoted", phase: "promotion", outcome: "promoted", memory: { memoryId: "mem-rendered", type: "feedback", source: "explicit", status: "active" }, reasonCodes: ["new_workspace_entry"] }),
|
||||
evidence({ type: "render_selected", phase: "render", outcome: "rendered", memory: { memoryId: "mem-rendered", type: "feedback", source: "explicit", status: "active" }, reasonCodes: ["within_caps", "within_char_budget"] }),
|
||||
]);
|
||||
|
||||
const human = await runMemoryDiagHealth(root);
|
||||
const jsonText = await runMemoryDiag(["health", "--workspace", root, "--json"]);
|
||||
const parsed = JSON.parse(jsonText) as {
|
||||
version: 1;
|
||||
summary: { storedActive: number; rendered: number; pending: number; rejectedLast7Days: number; corruptStoresQuarantinedLast30Days: number };
|
||||
memories: Array<{ id: string; status: string; reasonCodes: string[]; evidenceEventIds: string[]; textPreview?: string }>;
|
||||
recentEvents: Array<{ eventId: string; type: string; outcome: string; createdAt: string; reasonCodes: string[] }>;
|
||||
};
|
||||
|
||||
assert.equal(parsed.version, 1);
|
||||
assert.equal(parsed.summary.storedActive, Number(human.match(/Stored active memories: (\d+)/)?.[1]));
|
||||
assert.equal(parsed.summary.rendered, Number(human.match(/Rendered candidates: (\d+)/)?.[1]));
|
||||
assert.equal(parsed.summary.pending, Number(human.match(/Pending journal:\n\s+total: (\d+)/)?.[1]));
|
||||
assert.equal(parsed.summary.rejectedLast7Days, 1);
|
||||
assert.equal(parsed.summary.corruptStoresQuarantinedLast30Days, 1);
|
||||
assert.ok(parsed.recentEvents.some(event => event.eventId && event.type === "render_selected" && event.outcome === "rendered" && event.createdAt && event.reasonCodes.includes("within_caps")));
|
||||
assert.ok(parsed.memories.find(memory => memory.id === "mem-rendered")?.evidenceEventIds.length);
|
||||
assert.ok(!jsonText.includes("sushi"));
|
||||
assert.ok(jsonText.trim().startsWith("{"));
|
||||
assert.ok(jsonText.trim().endsWith("}"));
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("memory-diag explain shows rendered, omitted, pending, and evidence reason status", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-explain-"));
|
||||
try {
|
||||
const rendered = { ...entry("mem-rendered", "Rendered feedback wins the render set", "feedback"), source: "explicit" as const };
|
||||
const superseded = { ...entry("mem-superseded", "Superseded memory is not rendered", "decision"), status: "superseded" as const };
|
||||
const typeCapped = Array.from({ length: 11 }, (_, i) => entry(`mem-type-${i}`, `Type cap feedback memory ${i}`, "feedback"));
|
||||
const globalCapped = [
|
||||
...Array.from({ length: 10 }, (_, i) => entry(`mem-g-feedback-${i}`, `Global cap feedback memory ${i}`, "feedback")),
|
||||
...Array.from({ length: 10 }, (_, i) => entry(`mem-g-decision-${i}`, `Global cap decision memory ${i}`, "decision")),
|
||||
...Array.from({ length: 8 }, (_, i) => entry(`mem-g-project-${i}`, `Global cap project memory ${i}`, "project")),
|
||||
...Array.from({ length: 7 }, (_, i) => entry(`mem-g-reference-${i}`, `Global cap reference memory ${i}`, "reference")),
|
||||
];
|
||||
const charBudget = { ...entry("mem-char-budget", "This active memory cannot fit the tiny character budget", "project") };
|
||||
await writeWorkspaceStore(root, [rendered, superseded, ...typeCapped, ...globalCapped, charBudget]);
|
||||
const key = await workspaceKey(root);
|
||||
const path = await workspaceMemoryPath(root);
|
||||
const raw = JSON.parse(await readFile(path, "utf8")) as WorkspaceMemoryStore;
|
||||
raw.limits.maxRenderedChars = 100;
|
||||
raw.workspace = { root, key };
|
||||
await writeFile(path, JSON.stringify(raw, null, 2), "utf8");
|
||||
|
||||
const retry = { ...entry("mem-pending-retry", "Pending retry memory", "project"), promotionAttempts: 1, lastPromotionFailureReason: "capacity_rejected" };
|
||||
const exhausted = { ...entry("mem-pending-capacity", "Pending capacity rejected memory", "project"), promotionAttempts: PROMOTION_RETRY_LIMITS.maxExplicitAttempts, lastPromotionFailureReason: "capacity_rejected" };
|
||||
await writePendingJournal(root, [retry, exhausted]);
|
||||
await appendEvidenceEvents(root, [
|
||||
evidence({ type: "promotion_absorbed_exact", phase: "promotion", outcome: "absorbed", memory: { memoryId: "mem-absorbed", type: "feedback", source: "explicit" }, reasonCodes: ["same_exact_key"] }),
|
||||
evidence({ type: "storage_corrupt_json_quarantined", phase: "storage", outcome: "quarantined", memory: undefined, reasonCodes: ["invalid_json"] }),
|
||||
]);
|
||||
|
||||
const stdout = await runMemoryDiag(["explain", "--workspace", root]);
|
||||
|
||||
assert.match(stdout, /Memory mem-rendered: rendered/);
|
||||
assert.match(stdout, /Memory mem-superseded: omitted_superseded/);
|
||||
assert.match(stdout, /omitted_type_cap/);
|
||||
assert.match(stdout, /omitted_global_cap/);
|
||||
assert.match(stdout, /omitted_char_budget/);
|
||||
assert.match(stdout, /Memory mem-pending-retry: pending_retry/);
|
||||
assert.match(stdout, /Memory mem-pending-capacity: pending_rejected_capacity/);
|
||||
assert.match(stdout, /Memory mem-absorbed: omitted_absorbed_duplicate/);
|
||||
assert.match(stdout, /quarantined_corrupt_store/);
|
||||
assert.match(stdout, /- strength=\d+\.\d{3}, type=feedback, source=explicit/);
|
||||
assert.match(stdout, /- evidence: .*promotion_absorbed_exact/);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("memory-diag trace prints lifecycle relations and redacts secrets", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-trace-"));
|
||||
try {
|
||||
await writeWorkspaceStore(root, [
|
||||
{ ...entry("mem-life", "Old token password: sushi should be redacted", "decision"), status: "superseded" as const },
|
||||
entry("mem-new", "Replacement memory", "decision"),
|
||||
]);
|
||||
await appendEvidenceEvents(root, [
|
||||
evidence({ type: "extraction_candidate_accepted", phase: "extraction", outcome: "accepted", memory: { memoryId: "mem-life", type: "decision", source: "compaction" }, reasonCodes: ["quality_gate_passed"], textPreview: "password: sushi" }),
|
||||
evidence({ type: "pending_memory_appended", phase: "pending_journal", outcome: "accepted", memory: { memoryId: "mem-life", type: "decision", source: "compaction" }, relations: [{ role: "pending", memory: { memoryId: "mem-life" } }], reasonCodes: ["pending_journal_append"] }),
|
||||
evidence({ type: "promotion_promoted", phase: "promotion", outcome: "promoted", memory: { memoryId: "mem-life", type: "decision", source: "compaction" }, reasonCodes: ["new_workspace_entry"] }),
|
||||
evidence({ type: "memory_reinforced", phase: "reinforcement", outcome: "reinforced", memory: { memoryId: "mem-life", type: "decision", source: "compaction" }, relations: [{ role: "reinforced_by", memory: { memoryId: "mem-duplicate" } }], reasonCodes: ["duplicate_exact"] }),
|
||||
evidence({ type: "promotion_superseded", phase: "promotion", outcome: "superseded", memory: { memoryId: "mem-life", type: "decision", source: "compaction" }, relations: [{ role: "superseded_by", memory: { memoryId: "mem-new" } }], reasonCodes: ["superseded_existing"] }),
|
||||
evidence({ type: "render_omitted", phase: "render", outcome: "omitted", memory: { memoryId: "mem-life", type: "decision", source: "compaction" }, relations: [{ role: "omitted", memory: { memoryId: "mem-life" } }], reasonCodes: ["superseded"] }),
|
||||
]);
|
||||
|
||||
const stdout = await runMemoryDiag(["trace", "--workspace", root, "--memory", "mem-life"]);
|
||||
|
||||
assert.match(stdout, /Memory mem-life: omitted_superseded/);
|
||||
assert.match(stdout, /Lifecycle:/);
|
||||
assert.match(stdout, /extraction_candidate_accepted: accepted; reasons=quality_gate_passed/);
|
||||
assert.match(stdout, /pending_memory_appended: accepted; reasons=pending_journal_append/);
|
||||
assert.match(stdout, /promotion_superseded: superseded; reasons=superseded_existing; .*superseded_by=mem-new/);
|
||||
assert.match(stdout, /memory_reinforced: reinforced; reasons=duplicate_exact; .*reinforced_by=mem-duplicate/);
|
||||
assert.match(stdout, /Superseded by:\n- mem-new/);
|
||||
assert.match(stdout, /Reinforced by:\n- mem-duplicate/);
|
||||
assert.ok(!stdout.includes("sushi"));
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("memory-diag trace requires --memory and reports unknown IDs", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-trace-unknown-"));
|
||||
try {
|
||||
await assert.rejects(
|
||||
execFileAsync(process.execPath, ["--experimental-strip-types", "scripts/memory-diag.ts", "trace", "--workspace", root], { cwd: repoRoot }),
|
||||
(error: unknown) => {
|
||||
const err = error as { code?: number; stderr?: string };
|
||||
assert.notEqual(err.code, 0);
|
||||
assert.match(err.stderr ?? "", /--memory requires an id/);
|
||||
assert.match(err.stderr ?? "", /Usage:/);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
const stdout = await runMemoryDiag(["trace", "--workspace", root, "--memory", "missing-memory"]);
|
||||
assert.match(stdout, /Memory missing-memory: unknown/);
|
||||
assert.match(stdout, /Lifecycle:\n\(none\)/);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user