diff --git a/docs/architecture.md b/docs/architecture.md index 7fb88ac..76ad6d6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -55,6 +55,30 @@ Long-term memory that persists across sessions within the same workspace. Perfec } ``` +### Evidence Log Schema + +Workspace diagnostics also read the append-only evidence log for the current workspace. New evidence records are additive and keep historical records valid: + +```typescript +{ + version: 1, + eventId: string, + createdAt: string, + workspaceKey: string, + workspaceRootHash: string, + producerName?: string, + producerVersion?: string, + instrumentationVersion?: number, + type: EvidenceEventType, + phase: EvidencePhase, + outcome: EvidenceOutcome, + reasonCodes: string[], + details?: Record +} +``` + +Instrumentation version 2 adds optional causal details for future diagnostics without backfilling old JSONL records. Reinforcement-block events may include `details.blockReason` (for example `same_session` or `same_utc_day`). Capacity-removal events may include `details.strengthAtRemoval`, `details.rankAtRemoval`, `details.typeRankAtRemoval`, and `details.ageDaysAtRemoval`. `memory-diag quality` treats missing producer/instrumentation fields as historical or ambiguous rather than proof of current behavior. + ### Entry Types | Type | Purpose | Example | diff --git a/docs/diagnostics.md b/docs/diagnostics.md index c8a9a04..a2b13dd 100644 --- a/docs/diagnostics.md +++ b/docs/diagnostics.md @@ -22,6 +22,25 @@ The npm package is `opencode-working-memory`; the installed bin is `memory-diag` - `--verbose` — show detailed diagnostics. - `--json` — print machine-readable output where supported. +## Diagnostic Answerability Contract + +Every diagnostic section must document: + +1. **Question:** What does the reviewer want to know? +2. **Decision:** What action could the answer inform? +3. **Competing explanations:** At least two interpretations of the same metric. +4. **Required signals:** What fields/events distinguish those explanations? +5. **Current signals:** What currently exists? +6. **Answerability level:** `supported` | `partial` | `inventory_only` | `not_instrumented` +7. **Output permission:** What the tool may say without overclaiming. + +For `memory-diag quality`: +- `reinforcementRules`: `inventory_only` (cannot distinguish spam from legitimate blocks) +- `evictionAndCaps`: `inventory_only` (cannot distinguish healthy turnover from premature eviction) +- Old evidence remains ambiguous. Answerability improves only for events produced after instrumentation version 2. Mixed old/new logs will show a mix of `inventory_only` and `partial` sections. +- Producer-instrumented reinforcement blocks can upgrade `reinforcementRules` to `partial` by showing exact block reasons and UTC-day grouping; they still require human content judgment. +- Producer-instrumented capacity removals with rank/strength snapshots can upgrade `evictionAndCaps` to `partial`; fullness alone remains occupancy inventory, not proof of a capacity problem. + ## Examples ```bash @@ -36,10 +55,11 @@ npx --package opencode-working-memory memory-diag revert --memory { const report = buildQualityReviewBoard(model, { verbose: options.verbose, raw: options.raw, - noEmoji: options.noEmoji, json: options.json, }); @@ -16,5 +15,5 @@ export async function runQuality(options: CliOptions): Promise { return { stdout: JSON.stringify(buildQualityJSON(report, options.raw), null, 2) }; } - return { stdout: formatQualityReviewBoard(report, { verbose: options.verbose, noEmoji: options.noEmoji }) }; + return { stdout: formatQualityReviewBoard(report, { verbose: options.verbose }) }; } diff --git a/scripts/memory-diag/formatters/quality.ts b/scripts/memory-diag/formatters/quality.ts index 31f16c5..96c38cf 100644 --- a/scripts/memory-diag/formatters/quality.ts +++ b/scripts/memory-diag/formatters/quality.ts @@ -3,9 +3,15 @@ import type { CandidateProvenance, HeuristicFlag, ProvenanceClassification, + RejectionVersionFacts, + ReinforcementVersionFacts, ReviewBoardActiveMemory, ReviewBoardCandidate, ReviewBoardReport, + EvictionVersionFacts, + VersionAvailability, + VersionCoverage, + VersionedMechanismFacts, } from "../quality-review-model.ts"; const PROVENANCE_ORDER: ProvenanceClassification[] = [ @@ -26,7 +32,7 @@ export function buildQualityJSON(report: ReviewBoardReport, raw = false): unknow export function formatQualityReviewBoard( report: ReviewBoardReport, - options: { verbose?: boolean; noEmoji?: boolean }, + options: { verbose?: boolean }, ): string { const bullet = "-"; const lines: string[] = []; @@ -57,9 +63,13 @@ export function formatQualityReviewBoard( function pushEvidenceProvenance(lines: string[], report: ReviewBoardReport, bullet: string): void { const context = report.provenanceContext; + const instrumentation = report.facts.systemMechanisms.instrumentation; lines.push("Evidence provenance"); lines.push(` ${bullet} method: migration/timestamp/format inference`); lines.push(` ${bullet} confidence: ${context.confidenceDisclaimer}`); + lines.push(` ${bullet} Producer coverage: ${instrumentation.evidenceEventsWithProducer} of ${instrumentation.evidenceEventsTotal} evidence events instrumented`); + lines.push(` ${bullet} Rejection producer coverage: ${instrumentation.rejectionRecordsWithProducer} of ${instrumentation.rejectionRecordsTotal} rejection records instrumented`); + lines.push(` ${bullet} instrumentation versions: ${formatCounts(instrumentation.instrumentationVersions)}`); lines.push(` ${bullet} migration timeline: ${formatMigrationTimeline(context.migrationTimeline)}`); if (context.lastActivityAt) lines.push(` ${bullet} last activity: ${context.lastActivityAt}`); } @@ -70,26 +80,41 @@ function pushSystemMechanismFacts(lines: string[], report: ReviewBoardReport, bu lines.push(" Provenance counts for mechanism evidence"); lines.push(` ${bullet} ${formatProvenanceCounts(report.provenanceContext.countsByClassification)}`); lines.push(" Rejection filters"); + pushAnswerability(lines, report.answerability?.rejectionFilters, " "); lines.push(` ${bullet} rejected records: ${facts.rejectionFilters.totalRecords} (unique: ${facts.rejectionFilters.uniqueTexts})`); lines.push(` ${bullet} raw reason-code distribution: ${formatCounts(facts.rejectionFilters.byRawReasonCode)}`); lines.push(` ${bullet} type distribution: ${formatCounts(facts.rejectionFilters.byType)}`); lines.push(` ${bullet} ambiguous/architecture-like rejected candidates: ${facts.rejectionFilters.ambiguousOrArchitectureLike}`); lines.push(` ${bullet} status-or-hard-reason evidence: ${facts.rejectionFilters.hardReasonOrNoiseHeuristic}`); lines.push(` ${bullet} re-absorbed rejected texts: ${facts.rejectionFilters.reabsorbedRejectedTexts}`); + if (facts.versionedFacts) pushVersionAnalysis(lines, facts.versionedFacts.rejectionFilters, facts.versionedFacts.versionCoverage, formatRejectionVersionFacts, bullet); lines.push(" Reinforcement rules"); + pushAnswerability(lines, report.answerability?.reinforcementRules, " "); lines.push(` ${bullet} reinforce attempts: ${facts.reinforcementRules.reinforceEvents}, reinforced: ${facts.reinforcementRules.reinforcedEvents}, rejected/blocked: ${facts.reinforcementRules.rejectedOrBlockedEvents}`); lines.push(` ${bullet} reinforcement-window blocked: ${facts.reinforcementRules.windowBlockedEvents} (rate: ${formatPercent(facts.reinforcementRules.windowBlockRate)})`); + lines.push(` ${bullet} Exact block reasons: ${formatCounts(facts.reinforcementRules.blocksByExactReason)}`); + lines.push(` ${bullet} window blocks by UTC day: ${formatCounts(facts.reinforcementRules.windowBlocksByUtcDay)}`); + lines.push(` ${bullet} block details missing: ${facts.reinforcementRules.blockDetailsMissing}`); lines.push(` ${bullet} repeated blocks by memory: ${formatRepeatedBlocks(facts.reinforcementRules.repeatedBlocksByMemory)}`); lines.push(` ${bullet} malformed command events: ${facts.reinforcementRules.malformedCommandEvents}`); + if (facts.versionedFacts) pushVersionAnalysis(lines, facts.versionedFacts.reinforcementRules, facts.versionedFacts.versionCoverage, formatReinforcementVersionFacts, bullet); lines.push(" Eviction and caps"); + pushAnswerability(lines, report.answerability?.evictionAndCaps, " "); lines.push(` ${bullet} active memories: ${facts.evictionAndCaps.activeMemories} / ${facts.evictionAndCaps.maxEntries}`); lines.push(` ${bullet} rendered memories: ${facts.evictionAndCaps.renderedMemories}`); - lines.push(` ${bullet} full caps: ${formatFullCaps(facts.evictionAndCaps.fullCaps, facts.evictionAndCaps.typeCounts, facts.evictionAndCaps.typeCaps, facts.evictionAndCaps.activeMemories, facts.evictionAndCaps.maxEntries)}`); + lines.push(` ${bullet} cap occupancy: ${formatFullCaps(facts.evictionAndCaps.fullCaps, facts.evictionAndCaps.typeCounts, facts.evictionAndCaps.typeCaps, facts.evictionAndCaps.activeMemories, facts.evictionAndCaps.maxEntries)}`); lines.push(` ${bullet} capacity removals: total=${facts.evictionAndCaps.removedByCapacity}, global=${facts.evictionAndCaps.removedByGlobalCap}, type=${facts.evictionAndCaps.removedByTypeCap}`); + lines.push(` ${bullet} Removals with snapshot: ${facts.evictionAndCaps.recentCapacityRemovalsWithSnapshot}`); + lines.push(` ${bullet} Removals without snapshot: ${facts.evictionAndCaps.capacitySnapshotsMissing} (historical)`); + if (facts.evictionAndCaps.highestRankRemoved) lines.push(` ${bullet} highest-rank removed snapshot: ${formatHighestRankRemoved(facts.evictionAndCaps.highestRankRemoved)}`); lines.push(` ${bullet} recent evictions by type: ${formatCounts(facts.evictionAndCaps.recentEvictionsByType)}`); lines.push(` ${bullet} recent evicted content shown: ${facts.evictionAndCaps.recentEvictedContentShown}`); - lines.push(` ${bullet} evidence-only disappearances: ${facts.evictionAndCaps.missingEvidenceOnly} (unknown: ${facts.evictionAndCaps.unknownDisappearances})`); + if (facts.versionedFacts) pushVersionAnalysis(lines, facts.versionedFacts.evictionAndCaps, facts.versionedFacts.versionCoverage, formatEvictionVersionFacts, bullet); + lines.push(" Unknown disappearances"); + pushAnswerability(lines, report.answerability?.unknownDisappearances, " "); + lines.push(` ${bullet} unversioned disappearance inventory: evidence-only=${facts.evictionAndCaps.missingEvidenceOnly}, unknown=${facts.evictionAndCaps.unknownDisappearances}`); lines.push(" Identity and dedup"); + pushAnswerability(lines, report.answerability?.identityAndDedup, " "); lines.push(` ${bullet} replacements: total=${facts.identityAndDedup.replacementEvents}, same-type=${facts.identityAndDedup.sameTypeReplacementEvents}, cross-type=${facts.identityAndDedup.crossTypeReplacementEvents}`); lines.push(` ${bullet} superseded entries: ${facts.identityAndDedup.supersededEntries}`); lines.push(` ${bullet} exact duplicate/identity groups identified: ${facts.identityAndDedup.duplicateTextOrIdentityGroups}`); @@ -98,12 +123,120 @@ function pushSystemMechanismFacts(lines: string[], report: ReviewBoardReport, bu function pushMemoryContentFacts(lines: string[], report: ReviewBoardReport, bullet: string): void { const facts = report.facts.memoryContent; lines.push("Facts - memory content"); + pushAnswerability(lines, report.answerability?.memoryContent, " "); lines.push(` ${bullet} rendered memories: ${facts.renderedMemories}`); lines.push(` ${bullet} evidence coverage: ${facts.evidenceCoverage.covered} / ${facts.evidenceCoverage.total}`); lines.push(` ${bullet} type counts: ${formatTypeCountsWithCaps(facts.typeCounts, facts.typeCaps)}`); lines.push(` ${bullet} weakest/strongest active memory previews: weakest=${formatMemoryPreviews(facts.weakestActiveMemories)}; strongest=${formatMemoryPreviews(facts.strongestActiveMemories)}`); } +function pushAnswerability( + lines: string[], + assessment: NonNullable[keyof NonNullable] | undefined, + indent: string, +): void { + if (!assessment) return; + const suffix = assessment.level === "partial" ? " — causal fields exist, but human content judgment is still required" : ""; + lines.push(`${indent}(Answerability: ${assessment.level}${suffix})`); + lines.push(`${indent}Output permission: ${assessment.outputPermission}`); +} + +function pushVersionAnalysis( + lines: string[], + mechanism: VersionedMechanismFacts, + coverage: VersionCoverage, + formatFacts: (facts: TFacts) => string, + bullet: string, +): void { + lines.push(" Version analysis by producer version"); + lines.push(` Version-stamp coverage (all evidence/rejection records, not mechanism problem counts): Coverage: ${formatCoveragePercent(coverage.coveragePercent)} of ${formatInteger(coverage.totalEvents)} records carry a version stamp (${formatInteger(coverage.currentVersionEvents)} current, ${formatInteger(coverage.previousVersionEvents)} previous, ${formatInteger(coverage.unknownVersionEvents)} unknown/unversioned). Comparison will become meaningful as new events accumulate.`); + if (coverage.isTransitional) { + lines.push(" NOTE: Version coverage is below 50%. Current-version comparisons may not be representative."); + } + lines.push(` ${mechanismOpportunityDescription(mechanism)}`); + for (const group of ["current", "previous", "unknown_unversioned"] as const) { + const bucket = mechanism.buckets[group]; + lines.push(` ${bullet} ${bucket.label}: opportunities=${bucket.opportunityCount}, observed=${bucket.observedPatternCount}, sample=${bucket.sampleAssessment}, answerability=${bucket.answerabilityLevel}`); + if (Object.keys(bucket.producerVersions).length > 0) lines.push(` producer versions: ${formatCounts(bucket.producerVersions)}`); + lines.push(` composition: ${formatVersionAvailability(bucket.versionAvailability)}`); + lines.push(` facts: ${formatFacts(bucket.facts)}`); + } + lines.push(` ${bullet} inference: ${mechanism.inference.message}`); + lines.push(` diagnostic strength: ${diagnosticStrengthLabel(mechanism)}`); + const diagnosticLine = currentMechanismDiagnosticLine(mechanism); + if (diagnosticLine) lines.push(` ${diagnosticLine}`); + if (mechanism.diagnosticQuestions) { + for (const question of mechanism.diagnosticQuestions) { + lines.push(` diagnostic question: ${question.question} Evidence: ${question.evidence.join(", ")}`); + } + } + lines.push(` caveat: ${mechanism.inference.caveat}`); +} + +function diagnosticStrengthLabel(mechanism: VersionedMechanismFacts): string { + const current = mechanism.buckets.current; + const strength = mechanism.inference.status === "no_current_version_opportunities" || current.opportunityCount === 0 + ? "unavailable" + : current.opportunityCount < mechanism.sampleThreshold + ? "weak" + : "moderate"; + return hasCurrentCausalDetail(mechanism) ? `${strength}; causal detail available` : strength; +} + +function hasCurrentCausalDetail(mechanism: VersionedMechanismFacts): boolean { + const currentFacts = mechanism.buckets.current.facts; + return isReinforcementVersionFacts(currentFacts) && Object.keys(currentFacts.blocksByExactReason).length > 0; +} + +function currentMechanismDiagnosticLine(mechanism: VersionedMechanismFacts): string | undefined { + const current = mechanism.buckets.current; + if (!isReinforcementVersionFacts(current.facts)) return undefined; + const facts = current.facts; + const parts: string[] = []; + if (Object.keys(facts.blocksByExactReason).length > 0) parts.push(`current block reasons=${formatCounts(facts.blocksByExactReason)}`); + if (facts.blockDetailsMissing > 0) parts.push(`current block details missing=${facts.blockDetailsMissing}`); + if (parts.length === 0) return undefined; + parts.push(`sample=${current.opportunityCount} attempts`); + return `diagnostic: ${parts.join("; ")}`; +} + +function isReinforcementVersionFacts(facts: unknown): facts is ReinforcementVersionFacts { + return typeof facts === "object" + && facts !== null + && "blocksByExactReason" in facts + && "blockDetailsMissing" in facts + && "windowBlockedEvents" in facts; +} + +function mechanismOpportunityDescription(mechanism: VersionedMechanismFacts): string { + if (mechanism.opportunityName === "rejection candidates") return "Mechanism opportunities below are reviewable rejection candidates only; render/accounting events are excluded."; + if (mechanism.opportunityName === "attempts") return "Mechanism opportunities below are reinforcement attempts only; render/accounting events are excluded."; + if (mechanism.opportunityName === "capacity removals") return "Mechanism opportunities below are capacity removals only; render/accounting events are excluded."; + return `Mechanism opportunities below are ${mechanism.opportunityName} only; render/accounting events are excluded.`; +} + +function formatVersionAvailability(availability: VersionAvailability): string { + const parts = [ + availability.noProducerFields > 0 ? `no producer fields=${availability.noProducerFields}` : undefined, + availability.unknownProducerVersion > 0 ? `unknown version=${availability.unknownProducerVersion}` : undefined, + availability.emptyProducerVersion > 0 ? `empty version=${availability.emptyProducerVersion}` : undefined, + availability.knownProducerVersion > 0 ? `known version=${availability.knownProducerVersion}` : undefined, + ].filter((part): part is string => Boolean(part)); + return parts.length === 0 ? "(empty bucket)" : parts.join(", "); +} + +function formatRejectionVersionFacts(facts: RejectionVersionFacts): string { + return `records=${facts.totalRecords}, candidates=${facts.candidateRecords}, raw reason codes=${formatCounts(facts.byRawReasonCode)}, types=${formatCounts(facts.byType)}`; +} + +function formatReinforcementVersionFacts(facts: ReinforcementVersionFacts): string { + return `reinforce attempts=${facts.reinforceEvents}, window blocked=${facts.windowBlockedEvents}, exact reasons=${formatCounts(facts.blocksByExactReason)}, block details missing=${facts.blockDetailsMissing}`; +} + +function formatEvictionVersionFacts(facts: EvictionVersionFacts): string { + return `capacity removals=${facts.removedByCapacity}, with snapshot=${facts.recentCapacityRemovalsWithSnapshot}, missing snapshot=${facts.capacitySnapshotsMissing}`; +} + function pushSystemMechanismCandidates(lines: string[], report: ReviewBoardReport, bullet: string): void { const display = report.provenanceContext.candidateDisplay; if (report.provenanceContext.candidateLimit && display && display.shown < display.total) { @@ -257,6 +390,17 @@ function formatRepeatedBlocks(blocks: ReviewBoardReport["facts"]["systemMechanis return blocks.map(block => `${block.memoryId} count=${block.count} refs=${block.refs.join("|") || "none"} raw reason codes=${block.rawReasonCodes.join("|") || "none"}`).join(", "); } +function formatHighestRankRemoved(snapshot: NonNullable): string { + const parts = [ + `eventId=${snapshot.eventId}`, + snapshot.memoryId ? `memoryId=${snapshot.memoryId}` : undefined, + snapshot.type ? `type=${snapshot.type}` : undefined, + `rankAtRemoval=${snapshot.rankAtRemoval}`, + typeof snapshot.strengthAtRemoval === "number" ? `strengthAtRemoval=${snapshot.strengthAtRemoval}` : undefined, + ].filter((part): part is string => Boolean(part)); + return parts.join(" "); +} + function formatMemoryPreviews(items: ReviewBoardReport["facts"]["memoryContent"]["weakestActiveMemories"]): string { if (items.length === 0) return "(none)"; return items.map(item => `${item.id} type=${item.type} strength=${typeof item.strength === "number" ? item.strength.toFixed(3) : "unknown"} text=${JSON.stringify(item.textPreview)}`).join(" | "); @@ -266,6 +410,15 @@ function formatPercent(value: number): string { return `${(Number.isFinite(value) ? value * 100 : 0).toFixed(1)}%`; } +function formatCoveragePercent(value: number): string { + if (!Number.isFinite(value)) return "0%"; + return Number.isInteger(value) ? `${value}%` : `${value.toFixed(1)}%`; +} + +function formatInteger(value: number): string { + return new Intl.NumberFormat("en-US", { maximumFractionDigits: 0 }).format(value); +} + function formatProvenance(provenance: CandidateProvenance): string { return `${provenance.classification} confidence=${provenance.confidence}; basis=${provenance.basis.join("; ") || "unavailable"}; caveat=${provenance.interpretationCaveat}`; } diff --git a/scripts/memory-diag/quality-review-model.ts b/scripts/memory-diag/quality-review-model.ts index 0ad4ea0..e259a49 100644 --- a/scripts/memory-diag/quality-review-model.ts +++ b/scripts/memory-diag/quality-review-model.ts @@ -1,13 +1,144 @@ import { createHash } from "node:crypto"; import type { EvidenceEventV1 } from "../../src/evidence-log.ts"; +import { producerFields } from "../../src/instrumentation.ts"; import { RETENTION_TYPE_MAX } from "../../src/retention.ts"; import type { LongTermMemoryEntry, LongTermType } from "../../src/types.ts"; import { TYPES } from "./constants.ts"; import { disappearanceRows } from "./inspection-model.ts"; -import { rejectionQualitySummary, uniqueByCanonicalText } from "./rejections-model.ts"; +import { hasWorkspaceScope, rejectionQualitySummary, uniqueByCanonicalText } from "./rejections-model.ts"; import { canonicalMemoryText, cleanText, countBy, objectFromCounts, truncate, uniqueStrings, workspaceRootHash } from "./text.ts"; import type { MemoryInspectionReadModel, NormalizedRejection } from "./types.ts"; +export type AnswerabilityLevel = "supported" | "partial" | "inventory_only" | "not_instrumented"; + +export type ProducerVersionGroup = "current" | "previous" | "unknown_unversioned"; + +export type VersionSampleAssessment = + | "observed" + | "not_observed_but_sample_small" + | "not_observed_with_sufficient_sample" + | "no_current_version_opportunities"; + +export type VersionAvailability = { + noProducerFields: number; + unknownProducerVersion: number; + emptyProducerVersion: number; + knownProducerVersion: number; +}; + +export type VersionCoverage = { + totalEvents: number; + currentVersionEvents: number; + previousVersionEvents: number; + unknownVersionEvents: number; + coveragePercent: number; + isTransitional: boolean; +}; + +export type VersionedMechanismInference = { + status: + | "current_recurrence_detected" + | "pattern_persists_across_versions" + | "no_current_evidence_observed" + | "no_current_evidence_sample_small" + | "no_current_version_opportunities" + | "no_previous_pattern_observed"; + message: string; + caveat: "Version grouping is based only on producerVersion strings in evidence"; +}; + +export type VersionBucketFacts = { + group: ProducerVersionGroup; + label: string; + opportunityCount: number; + observedPatternCount: number; + producerVersions: Record; + versionAvailability: VersionAvailability; + answerabilityLevel: AnswerabilityLevel; + sampleAssessment: VersionSampleAssessment; + facts: TFacts; +}; + +export type VersionedMechanismDiagnosticQuestion = { + mechanism: "reinforcement_rule"; + group: ProducerVersionGroup; + question: string; + evidence: string[]; +}; + +export type VersionedMechanismFacts = { + currentPackageVersion: string; + opportunityName: string; + sampleThreshold: number; + buckets: Record>; + inference: VersionedMechanismInference; + diagnosticQuestions?: VersionedMechanismDiagnosticQuestion[]; +}; + +export type RejectionVersionFacts = { + totalRecords: number; + candidateRecords: number; + byRawReasonCode: Record; + byType: Record; + ambiguousOrArchitectureLike: number; +}; + +export type ReinforcementVersionFacts = { + reinforceEvents: number; + reinforcedEvents: number; + rejectedOrBlockedEvents: number; + windowBlockedEvents: number; + windowBlockRate: number; + repeatedBlocksByMemory: Array<{ memoryId: string; count: number; refs: string[]; rawReasonCodes: string[] }>; + blocksByExactReason: Record; + windowBlocksByUtcDay: Record; + blockDetailsMissing: number; + malformedCommandEvents: number; +}; + +export type EvictionVersionFacts = { + removedByCapacity: number; + removedByGlobalCap: number; + removedByTypeCap: number; + recentEvictionsByType: Record; + recentCapacityRemovalsWithSnapshot: number; + capacitySnapshotsMissing: number; + highestRankRemoved?: { memoryId?: string; rankAtRemoval: number; strengthAtRemoval?: number; type?: string; eventId: string }; +}; + +export type VersionedSystemMechanismFacts = { + currentPackageVersion: string; + versionCoverage: VersionCoverage; + buckets: ProducerVersionGroup[]; + sampleThresholds: { + rejectionFilters: 5; + reinforcementRules: 5; + evictionAndCaps: 5; + }; + rejectionFilters: VersionedMechanismFacts; + reinforcementRules: VersionedMechanismFacts; + evictionAndCaps: VersionedMechanismFacts; +}; + +export type AnswerabilityAssessment = { + level: AnswerabilityLevel; + question: string; + decision: string; + competingExplanations: string[]; + requiredSignals: string[]; + currentSignals: string[]; + outputPermission: string; +}; + +export type AnswerabilityReport = { + rejectionFilters: AnswerabilityAssessment; + reinforcementRules: AnswerabilityAssessment; + evictionAndCaps: AnswerabilityAssessment; + unknownDisappearances: AnswerabilityAssessment; + identityAndDedup: AnswerabilityAssessment; + memoryContent: AnswerabilityAssessment; +}; + export type ReviewBoardReport = { version: 1; generatedAt: string; @@ -17,7 +148,7 @@ export type ReviewBoardReport = { nonAuthoritative: true; mutation: "none"; rawReasonCodesAreEvidence: true; - producerVersionRecorded: false; + producerVersionRecorded: boolean; provenanceInferenceOnly: true; primaryReviewPurpose: "system_mechanism_observations"; secondaryReviewPurpose: "memory_content_quality"; @@ -26,7 +157,7 @@ export type ReviewBoardReport = { method: "migration_timestamp_and_format_inference"; confidenceDisclaimer: string; falseCurrentRiskBias: "prefer_unversioned_ambiguous_when_uncertain"; - producerVersionAvailable: false; + producerVersionAvailable: boolean; migrationTimeline: Array<{ migrationId: string; presentInStore: boolean; firstEvidenceAt?: string }>; lastActivityAt?: string; countsByClassification: Record; @@ -39,6 +170,13 @@ export type ReviewBoardReport = { }; facts: { systemMechanisms: { + instrumentation: { + evidenceEventsTotal: number; + evidenceEventsWithProducer: number; + rejectionRecordsTotal: number; + rejectionRecordsWithProducer: number; + instrumentationVersions: Record; + }; rejectionFilters: { totalRecords: number; uniqueTexts: number; @@ -55,6 +193,9 @@ export type ReviewBoardReport = { windowBlockedEvents: number; windowBlockRate: number; repeatedBlocksByMemory: Array<{ memoryId: string; count: number; refs: string[]; rawReasonCodes: string[] }>; + blocksByExactReason: Record; + windowBlocksByUtcDay: Record; + blockDetailsMissing: number; malformedCommandEvents: number; }; evictionAndCaps: { @@ -71,6 +212,9 @@ export type ReviewBoardReport = { removedByTypeCap: number; recentEvictionsByType: Record; recentEvictedContentShown: number; + recentCapacityRemovalsWithSnapshot: number; + capacitySnapshotsMissing: number; + highestRankRemoved?: { memoryId?: string; rankAtRemoval: number; strengthAtRemoval?: number; type?: string; eventId: string }; }; identityAndDedup: { replacementEvents: number; @@ -79,6 +223,7 @@ export type ReviewBoardReport = { supersededEntries: number; duplicateTextOrIdentityGroups: number; }; + versionedFacts?: VersionedSystemMechanismFacts; }; memoryContent: { activeMemories: number; @@ -102,6 +247,7 @@ export type ReviewBoardReport = { systemMechanism: string[]; memoryContent: string[]; }; + answerability?: AnswerabilityReport; nextCommands: string[]; }; @@ -133,6 +279,12 @@ export type ReviewBoardCandidate = { facts: Record; evidence: { eventIds?: string[]; rawReasonCodes?: string[]; textPreview?: string; textAvailable: boolean }; provenance?: CandidateProvenance; + versionContext?: { + group: ProducerVersionGroup; + currentPackageVersion: string; + producerVersion?: string; + basis: string; + }; heuristicFlags: HeuristicFlag[]; reviewQuestions: string[]; nextCommands: string[]; @@ -160,9 +312,12 @@ export type HeuristicFlag = { caveat: string; }; -export const ACTIVE_MEMORY_FULL_TEXT_THRESHOLD = 40; -export const REPRESENTATIVE_CANDIDATE_LIMIT = 10; -export const RECENT_EVICTION_DAYS = 7; +const ACTIVE_MEMORY_FULL_TEXT_THRESHOLD = 40; +const REPRESENTATIVE_CANDIDATE_LIMIT = 10; +const RECENT_EVICTION_DAYS = 7; +const VERSION_ANALYSIS_SAMPLE_THRESHOLD = 5; +const VERSION_GROUPS: ProducerVersionGroup[] = ["current", "previous", "unknown_unversioned"]; +const VERSION_GROUPING_CAVEAT = "Version grouping is based only on producerVersion strings in evidence" as const; const KNOWN_MIGRATION_IDS = [ "2026-04-26-p0-cleanup", @@ -217,10 +372,11 @@ type DatedCandidateInput = { export function buildQualityReviewBoard( model: MemoryInspectionReadModel, - options: { verbose?: boolean; raw?: boolean; noEmoji?: boolean; json?: boolean }, + options: { verbose?: boolean; raw?: boolean; json?: boolean; currentProducerVersion?: string }, generatedAt = new Date().toISOString(), ): ReviewBoardReport { const raw = options.raw === true; + const currentPackageVersion = options.currentProducerVersion ?? producerFields().producerVersion; const activeMemories = model.store.entries.filter(entry => entry.status !== "superseded"); const typeCounts = typeCountsFor(activeMemories); const typeCaps = Object.fromEntries(TYPES.map(type => [type, RETENTION_TYPE_MAX[type]])); @@ -232,18 +388,21 @@ export function buildQualityReviewBoard( const reabsorbedKeys = new Set(activeKeyMatches.map(match => match.key)); const activeMemoryByKey = new Map(activeKeyMatches.map(match => [match.key, match.activeMemory])); const disappearances = disappearanceRows(model); + const instrumentationFacts = buildInstrumentationFacts(model.evidenceEvents, model.rejectionRecords); const reinforcementFacts = buildReinforcementFacts(model.evidenceEvents); const evictionFacts = buildEvictionFacts(model, activeMemories, typeCounts, typeCaps, disappearances, generatedAt); const identityFacts = buildIdentityFacts(model, activeMemories); + const versionedFacts = buildVersionedSystemMechanismFacts(model.evidenceEvents, model.rejectionRecords, currentPackageVersion, generatedAt); const memoryContentFacts = buildMemoryContentFacts(model, activeMemories, typeCounts, typeCaps, raw); + const producerVersionAvailable = [...model.evidenceEvents, ...model.rejectionRecords].some(hasKnownProducerVersion); const systemMechanismCandidateInputs = { rejection_filter: [ - ...buildRejectionCandidates(model.rejectionRecords, provenanceInputs, raw), - ...buildReabsorptionCandidates(activeKeyMatches, provenanceInputs, raw), + ...buildRejectionCandidates(model.rejectionRecords, provenanceInputs, raw, currentPackageVersion), + ...buildReabsorptionCandidates(activeKeyMatches, provenanceInputs, raw, currentPackageVersion), ], - reinforcement_rule: buildReinforcementCandidates(model.evidenceEvents, provenanceInputs, raw), - eviction_cap: buildEvictionCandidates(disappearances, model.evidenceEvents, provenanceInputs, raw, generatedAt), - identity_dedup: buildIdentityCandidates(model, activeMemories, provenanceInputs, raw), + reinforcement_rule: buildReinforcementCandidates(model.evidenceEvents, provenanceInputs, raw, currentPackageVersion), + eviction_cap: buildEvictionCandidates(disappearances, model.evidenceEvents, provenanceInputs, raw, generatedAt, currentPackageVersion), + identity_dedup: buildIdentityCandidates(model, activeMemories, provenanceInputs, raw, currentPackageVersion), }; const showAllSystemMechanismCandidates = options.verbose === true || options.json === true; const systemCandidateDisplay = buildSystemCandidateDisplay(systemMechanismCandidateInputs, showAllSystemMechanismCandidates); @@ -256,7 +415,9 @@ export function buildQualityReviewBoard( const activeMemoryDisplay = buildActiveMemoryDisplay(model, activeMemories, reabsorbedKeys, activeMemoryByKey, provenanceInputs, raw, options.verbose === true); const countsByClassification = countProvenanceClassifications(allSystemMechanismCandidates); - return { + const answerabilityReport = buildAnswerabilityReport(); + applyInstrumentedAnswerability(answerabilityReport, model.evidenceEvents, reinforcementFacts, evictionFacts, currentPackageVersion); + const report: ReviewBoardReport = { version: 1, generatedAt, workspace: { @@ -268,7 +429,7 @@ export function buildQualityReviewBoard( nonAuthoritative: true, mutation: "none", rawReasonCodesAreEvidence: true, - producerVersionRecorded: false, + producerVersionRecorded: producerVersionAvailable, provenanceInferenceOnly: true, primaryReviewPurpose: "system_mechanism_observations", secondaryReviewPurpose: "memory_content_quality", @@ -277,7 +438,7 @@ export function buildQualityReviewBoard( method: "migration_timestamp_and_format_inference", confidenceDisclaimer: "Producer version is not recorded in historical evidence; provenance is inferred and should not be used as proof of current behavior.", falseCurrentRiskBias: "prefer_unversioned_ambiguous_when_uncertain", - producerVersionAvailable: false, + producerVersionAvailable, migrationTimeline, lastActivityAt: model.store.lastActivityAt, countsByClassification, @@ -285,6 +446,7 @@ export function buildQualityReviewBoard( }, facts: { systemMechanisms: { + instrumentation: instrumentationFacts, rejectionFilters: { totalRecords: rejectionSummary.totalRecords, uniqueTexts: rejectionSummary.uniqueTexts, @@ -299,6 +461,7 @@ export function buildQualityReviewBoard( reinforcementRules: reinforcementFacts, evictionAndCaps: evictionFacts, identityAndDedup: identityFacts, + versionedFacts, }, memoryContent: memoryContentFacts, }, @@ -310,6 +473,480 @@ export function buildQualityReviewBoard( }, nextCommands: nextCommands(), }; + report.answerability = answerabilityReport; + return report; +} + +function buildAnswerabilityReport(): AnswerabilityReport { + return { + rejectionFilters: { + level: "partial", + question: "Are durable memories being rejected?", + decision: "Tune rejection filter or leave unchanged", + competingExplanations: [ + "Rejection filter correctly identifies non-durable candidates", + "Rejection filter over-filters durable architectural decisions", + ], + requiredSignals: ["full rejected text", "reason codes", "source/origin", "decision logic version", "later reabsorption"], + currentSignals: ["redacted text preview", "reason code distribution", "type distribution"], + outputPermission: "Show candidates and ask for review; do not claim false positives.", + }, + reinforcementRules: { + level: "inventory_only", + question: "Is the reinforcement window appropriate?", + decision: "Change reinforcement policy or leave unchanged", + competingExplanations: [ + "Window blocks same-day repeated command inventory", + "Window blocks legitimate recurring reinforcement across days", + ], + requiredSignals: ["per-attempt timestamp", "prior reinforcement timestamp", "exact block reason", "session/day grouping"], + currentSignals: ["block count", "repeated block groups", "window block rate"], + outputPermission: "Show counts and event IDs only; do not characterize the policy as strict or lenient.", + }, + evictionAndCaps: { + level: "inventory_only", + question: "Is capacity causing important memory loss?", + decision: "Raise caps or leave unchanged", + competingExplanations: [ + "Full caps are naturally occupied with healthy turnover", + "Full caps are causing premature eviction of important memories", + ], + requiredSignals: ["strength/rank at removal", "cutoff context", "age", "source", "reinforcement count"], + currentSignals: ["removal count", "cap occupancy", "type counts"], + outputPermission: "Show occupancy and removal counts only; do not infer a capacity problem from full caps.", + }, + unknownDisappearances: { + level: "inventory_only", + question: "Are missing memories current instrumentation defects?", + decision: "Fix removal logging or accept historical gaps", + competingExplanations: [ + "Missing memories were removed before terminal-removal instrumentation existed", + "Missing memories indicate a current data-loss defect", + ], + requiredSignals: ["producer/instrumentation version", "terminal-removal instrumentation epoch", "latest event timestamp"], + currentSignals: ["evidence absence", "latest known event timestamp"], + outputPermission: "Call them unversioned/ambiguous disappearance inventory unless producer data proves current instrumentation.", + }, + identityAndDedup: { + level: "partial", + question: "Are separate concepts incorrectly merged?", + decision: "Review identity rules or accept current behavior", + competingExplanations: [ + "Replacements correctly update semantic duplicates", + "Replacements incorrectly collapse distinct concepts with similar text", + ], + requiredSignals: ["before/after content", "identity keys", "replacement reason"], + currentSignals: ["replacement events", "superseded entries"], + outputPermission: "Show replacement/duplicate inventory; do not infer semantic correctness.", + }, + memoryContent: { + level: "partial", + question: "Are active memories useful and current?", + decision: "Review active memories or leave unchanged", + competingExplanations: [ + "Low text preview indicates a naturally ephemeral memory", + "Low text preview indicates potentially stale content", + ], + requiredSignals: ["full text", "type", "source", "age", "evidence coverage"], + currentSignals: ["text preview", "type distribution", "strength distribution"], + outputPermission: "Show review surface with text previews; invariant heuristic flags only.", + }, + }; +} + +function applyInstrumentedAnswerability( + report: AnswerabilityReport, + events: EvidenceEventV1[], + reinforcementFacts: ReviewBoardReport["facts"]["systemMechanisms"]["reinforcementRules"], + evictionFacts: ReviewBoardReport["facts"]["systemMechanisms"]["evictionAndCaps"], + currentPackageVersion: string, +): void { + const hasInstrumentedBlocks = events.some(event => + isReinforcementEvent(event) + && event.reasonCodes.includes("reinforcement_window_blocked") + && typeof event.details?.blockReason === "string" + && hasProducerFields(event) + && producerVersionGroupFor(event, currentPackageVersion) === "current" + ); + if (hasInstrumentedBlocks && Object.keys(reinforcementFacts.blocksByExactReason).length > 0) { + report.reinforcementRules.level = "partial"; + report.reinforcementRules.currentSignals = uniqueStrings([ + ...report.reinforcementRules.currentSignals, + "exact block reasons", + "UTC day grouping", + ]); + report.reinforcementRules.outputPermission = "Show exact block reasons and day grouping; causal fields exist but human content judgment is still required."; + } + + const hasCapacitySnapshots = evictionFacts.recentCapacityRemovalsWithSnapshot > 0 + && events.some(event => event.type === "memory_removed_capacity" && hasProducerFields(event) && hasCapacitySnapshot(event) && producerVersionGroupFor(event, currentPackageVersion) === "current"); + if (hasCapacitySnapshots) { + report.evictionAndCaps.level = "partial"; + report.evictionAndCaps.currentSignals = uniqueStrings([ + ...report.evictionAndCaps.currentSignals, + "rank at removal", + "strength at removal", + ]); + report.evictionAndCaps.outputPermission = "Show removal snapshots with rank/strength/age; causal fields exist but human judgment of importance is still required."; + } +} + +function buildInstrumentationFacts( + events: EvidenceEventV1[], + rejections: NormalizedRejection[], +): ReviewBoardReport["facts"]["systemMechanisms"]["instrumentation"] { + const instrumentedEvents = events.filter(hasProducerFields); + const instrumentedRejections = rejections.filter(hasProducerFields); + const instrumentationVersions: Record = {}; + for (const record of [...instrumentedEvents, ...instrumentedRejections]) { + const version = record.instrumentationVersion; + if (typeof version !== "number") continue; + const key = String(version); + instrumentationVersions[key] = (instrumentationVersions[key] ?? 0) + 1; + } + return { + evidenceEventsTotal: events.length, + evidenceEventsWithProducer: instrumentedEvents.length, + rejectionRecordsTotal: rejections.length, + rejectionRecordsWithProducer: instrumentedRejections.length, + instrumentationVersions, + }; +} + +function buildVersionedSystemMechanismFacts( + events: EvidenceEventV1[], + rejections: NormalizedRejection[], + currentPackageVersion: string, + generatedAt: string, +): VersionedSystemMechanismFacts { + const versionCoverage = buildVersionCoverage(events, rejections, currentPackageVersion); + return { + currentPackageVersion, + versionCoverage, + buckets: VERSION_GROUPS, + sampleThresholds: { + rejectionFilters: VERSION_ANALYSIS_SAMPLE_THRESHOLD, + reinforcementRules: VERSION_ANALYSIS_SAMPLE_THRESHOLD, + evictionAndCaps: VERSION_ANALYSIS_SAMPLE_THRESHOLD, + }, + rejectionFilters: buildVersionedRejectionFacts(rejections, currentPackageVersion), + reinforcementRules: buildVersionedReinforcementFacts(events, currentPackageVersion), + evictionAndCaps: buildVersionedEvictionFacts(events, currentPackageVersion, generatedAt), + }; +} + +function buildVersionedRejectionFacts( + records: NormalizedRejection[], + currentPackageVersion: string, +): VersionedMechanismFacts { + const buckets = buildVersionBuckets(records, currentPackageVersion, bucketRecords => { + const candidateRecords = bucketRecords.filter(isReviewableRejectionCandidate); + const facts: RejectionVersionFacts = { + totalRecords: bucketRecords.length, + candidateRecords: candidateRecords.length, + byRawReasonCode: objectFromCounts(countBy(bucketRecords.flatMap(record => record.reasons))), + byType: objectFromCounts(countBy(bucketRecords.map(record => record.type))), + ambiguousOrArchitectureLike: candidateRecords.length, + }; + return { facts, opportunityCount: candidateRecords.length, observedPatternCount: candidateRecords.length }; + }); + const base: Omit, "inference"> = { + currentPackageVersion, + opportunityName: "rejection candidates", + sampleThreshold: VERSION_ANALYSIS_SAMPLE_THRESHOLD, + buckets, + }; + return { + ...base, + inference: computeVersionedInference(base, { + observedPattern: "rejected candidates", + patternName: "reviewable_rejection_candidate", + }), + }; +} + +function buildVersionedReinforcementFacts( + events: EvidenceEventV1[], + currentPackageVersion: string, +): VersionedMechanismFacts { + const mechanismEvents = events.filter(event => isReinforcementEvent(event) || isMalformedCommandEvent(event)); + const buckets = buildVersionBuckets(mechanismEvents, currentPackageVersion, bucketEvents => { + const attempts = bucketEvents.filter(isReinforcementEvent); + const facts = buildReinforcementFacts(bucketEvents) as ReinforcementVersionFacts; + return { + facts, + opportunityCount: attempts.length, + observedPatternCount: attempts.filter(event => event.reasonCodes.includes("reinforcement_window_blocked")).length, + }; + }); + const base: Omit, "inference"> = { + currentPackageVersion, + opportunityName: "attempts", + sampleThreshold: VERSION_ANALYSIS_SAMPLE_THRESHOLD, + buckets, + }; + return { + ...base, + inference: computeVersionedInference(base, { + observedPattern: "blocked", + patternName: "reinforcement_window_blocked", + }), + ...diagnosticQuestionsProperty(buildReinforcementDiagnosticQuestions(mechanismEvents, currentPackageVersion)), + }; +} + +function diagnosticQuestionsProperty(questions: VersionedMechanismDiagnosticQuestion[]): { diagnosticQuestions?: VersionedMechanismDiagnosticQuestion[] } { + return questions.length > 0 ? { diagnosticQuestions: questions } : {}; +} + +function buildReinforcementDiagnosticQuestions(events: EvidenceEventV1[], currentPackageVersion: string): VersionedMechanismDiagnosticQuestion[] { + const matching = events + .filter(event => isReinforcementEvent(event) + && event.reasonCodes.includes("reinforcement_window_blocked") + && producerVersionGroupFor(event, currentPackageVersion) === "current" + && event.details?.blockReason === "same_session") + .map(event => { + const attemptedAtIso = stringDetail(event, "attemptedAtIso"); + const lastReinforcedAtIso = stringDetail(event, "lastReinforcedAtIso"); + return { event, attemptedAtIso, lastReinforcedAtIso }; + }) + .filter((item): item is { event: EvidenceEventV1; attemptedAtIso: string; lastReinforcedAtIso: string } => + typeof item.attemptedAtIso === "string" + && typeof item.lastReinforcedAtIso === "string" + && isValidIsoDate(item.attemptedAtIso) + && isValidIsoDate(item.lastReinforcedAtIso) + && item.attemptedAtIso.slice(0, 10) !== item.lastReinforcedAtIso.slice(0, 10) + ) + .sort((a, b) => compareIso(b.attemptedAtIso, a.attemptedAtIso) || a.event.eventId.localeCompare(b.event.eventId)); + + const representative = matching[0]; + if (!representative) return []; + return [{ + mechanism: "reinforcement_rule", + group: "current", + question: "Should same_session reinforcement blocking apply across UTC days?", + evidence: [ + `count=${matching.length}`, + `eventId=${representative.event.eventId}`, + `attemptedAtIso=${representative.attemptedAtIso}`, + `lastReinforcedAtIso=${representative.lastReinforcedAtIso}`, + ], + }]; +} + +function buildVersionedEvictionFacts( + events: EvidenceEventV1[], + currentPackageVersion: string, + generatedAt: string, +): VersionedMechanismFacts { + const capacityEvents = events.filter(event => event.type === "memory_removed_capacity"); + const buckets = buildVersionBuckets(capacityEvents, currentPackageVersion, bucketEvents => { + const facts = buildEvictionVersionFacts(bucketEvents, generatedAt); + return { + facts, + opportunityCount: bucketEvents.length, + observedPatternCount: facts.capacitySnapshotsMissing, + }; + }); + const base: Omit, "inference"> = { + currentPackageVersion, + opportunityName: "capacity removals", + sampleThreshold: VERSION_ANALYSIS_SAMPLE_THRESHOLD, + buckets, + }; + return { + ...base, + inference: computeVersionedInference(base, { + observedPattern: "missing snapshots", + patternName: "capacity_snapshot_missing", + }), + }; +} + +function buildVersionBuckets( + records: TRecord[], + currentPackageVersion: string, + summarize: (records: TRecord[]) => { facts: TFacts; opportunityCount: number; observedPatternCount: number }, +): Record> { + const grouped = Object.fromEntries(VERSION_GROUPS.map(group => [group, []])) as Record; + for (const record of records) grouped[producerVersionGroupFor(record, currentPackageVersion)].push(record); + return Object.fromEntries(VERSION_GROUPS.map(group => { + const bucketRecords = grouped[group]; + const summary = summarize(bucketRecords); + return [group, { + group, + label: versionGroupLabel(group, currentPackageVersion), + opportunityCount: summary.opportunityCount, + observedPatternCount: summary.observedPatternCount, + producerVersions: producerVersionCounts(bucketRecords), + versionAvailability: buildVersionAvailability(bucketRecords), + answerabilityLevel: group === "current" && summary.opportunityCount > 0 ? "partial" : "inventory_only", + sampleAssessment: sampleAssessmentFor(group, summary.opportunityCount, summary.observedPatternCount, currentPackageVersion), + facts: summary.facts, + } satisfies VersionBucketFacts]; + })) as Record>; +} + +function buildEvictionVersionFacts(capacityEvents: EvidenceEventV1[], generatedAt: string): EvictionVersionFacts { + const recentCapacityEvents = capacityEvents.filter(event => isWithinDaysOf(event.createdAt, generatedAt, RECENT_EVICTION_DAYS)); + const capacityEventsWithSnapshot = capacityEvents.filter(hasCapacitySnapshot); + const capacityEventsWithRank = capacityEvents.filter(event => numberDetail(event, "rankAtRemoval") !== undefined); + const highestRankRemovedEvent = [...capacityEventsWithRank] + .sort((a, b) => (numberDetail(a, "rankAtRemoval") ?? Number.POSITIVE_INFINITY) - (numberDetail(b, "rankAtRemoval") ?? Number.POSITIVE_INFINITY))[0]; + return { + removedByCapacity: capacityEvents.length, + removedByGlobalCap: capacityEvents.filter(event => event.reasonCodes.includes("global_cap")).length, + removedByTypeCap: capacityEvents.filter(event => event.reasonCodes.includes("type_cap")).length, + recentEvictionsByType: objectFromCounts(countBy(recentCapacityEvents.map(event => event.memory?.type ?? "unknown"))), + recentCapacityRemovalsWithSnapshot: capacityEventsWithSnapshot.length, + capacitySnapshotsMissing: capacityEvents.length - capacityEventsWithSnapshot.length, + ...(highestRankRemovedEvent ? { highestRankRemoved: highestRankRemoved(highestRankRemovedEvent) } : {}), + }; +} + +function isReviewableRejectionCandidate(record: NormalizedRejection): boolean { + if (!record.reasons.includes("bad_decision")) return false; + const label = neutralRejectionLabel(record); + return label === "architecture_like_rejected_candidate" || label === "ambiguous_rejected_candidate"; +} + +function producerVersionCounts(records: ProducerBearingRecord[]): Record { + const counts: Record = {}; + for (const record of records) { + if (!hasKnownProducerVersion(record)) continue; + const version = String(record.producerVersion).trim(); + counts[version] = (counts[version] ?? 0) + 1; + } + return counts; +} + +function versionGroupLabel(group: ProducerVersionGroup, currentPackageVersion: string): string { + if (group === "current") return `current version ${currentPackageVersion}`; + if (group === "previous") return "previous versions"; + return "unknown/unversioned"; +} + +function sampleAssessmentFor( + group: ProducerVersionGroup, + opportunityCount: number, + observedPatternCount: number, + currentPackageVersion: string, +): VersionSampleAssessment { + if (observedPatternCount > 0) return "observed"; + if (group === "current" && (!isAssessableCurrentPackageVersion(currentPackageVersion) || opportunityCount === 0)) return "no_current_version_opportunities"; + if (opportunityCount < VERSION_ANALYSIS_SAMPLE_THRESHOLD) return "not_observed_but_sample_small"; + return "not_observed_with_sufficient_sample"; +} + +function isAssessableCurrentPackageVersion(currentPackageVersion: string): boolean { + const trimmed = currentPackageVersion.trim(); + return trimmed.length > 0 && trimmed !== "unknown"; +} + +function computeVersionedInference( + mechanism: Omit, "inference">, + text: { observedPattern: string; patternName: string }, +): VersionedMechanismInference { + const current = mechanism.buckets.current; + const previous = mechanism.buckets.previous; + const currentFact = `Current version: ${current.observedPatternCount} ${text.observedPattern} in ${current.opportunityCount} ${mechanism.opportunityName}.`; + const previousFact = `Previous versions: ${previous.observedPatternCount} ${text.observedPattern} in ${previous.opportunityCount} ${mechanism.opportunityName}.`; + const unknownUnversioned = mechanism.buckets.unknown_unversioned; + if (!isAssessableCurrentPackageVersion(mechanism.currentPackageVersion) || current.opportunityCount === 0) { + return inference("no_current_version_opportunities", "Current package version is unknown or has no events; cannot assess recurrence."); + } + if (current.observedPatternCount > 0 && previous.observedPatternCount === 0 && unknownUnversioned.observedPatternCount === 0) { + return inference("no_previous_pattern_observed", `${currentFact} No previous pattern observed — this is a new pattern, not a recurrence.`); + } + if (current.observedPatternCount > 0) { + if (previous.observedPatternCount > 0) { + return inference("pattern_persists_across_versions", `${currentFact} ${previousFact} Current recurrence detected — ${text.patternName} observed in current version. Pattern persists across versions.`); + } + // Current has signal, previous has none, but unknown/unversioned has signal + return inference("current_recurrence_detected", `${currentFact} No known previous-version pattern observed, but unknown/unversioned evidence shows ${unknownUnversioned.observedPatternCount} ${text.observedPattern}. Pattern may persist — version grouping cannot confirm or deny.`); + } + if (current.opportunityCount < mechanism.sampleThreshold) { + return inference("no_current_evidence_sample_small", `${currentFact} ${previousFact} No current evidence observed, but current-version opportunity count is ${current.opportunityCount} (<${mechanism.sampleThreshold}); do not infer absence.`); + } + return inference("no_current_evidence_observed", `${currentFact} ${previousFact} No recurrence observed with sufficient current-version sample.`); +} + +function inference(status: VersionedMechanismInference["status"], message: string): VersionedMechanismInference { + return { status, message, caveat: VERSION_GROUPING_CAVEAT }; +} + +function hasProducerFields(record: Pick | Pick): boolean { + return typeof record.producerName === "string" + && record.producerName.length > 0 + && typeof record.producerVersion === "string" + && record.producerVersion.length > 0 + && typeof record.instrumentationVersion === "number"; +} + +type ProducerBearingRecord = Pick; + +export function hasKnownProducerVersion(record: ProducerBearingRecord): boolean { + if (typeof record.producerVersion !== "string") return false; + const producerVersion = record.producerVersion.trim(); + return producerVersion.length > 0 && producerVersion !== "unknown"; +} + +export function producerVersionGroupFor(record: ProducerBearingRecord, currentPackageVersion: string): ProducerVersionGroup { + if (!hasKnownProducerVersion(record)) return "unknown_unversioned"; + const producerVersion = String(record.producerVersion).trim(); + const currentVersion = currentPackageVersion.trim(); + if (currentVersion.length > 0 && currentVersion !== "unknown" && producerVersion === currentVersion) return "current"; + return "previous"; +} + +function buildVersionAvailability(records: ProducerBearingRecord[]): VersionAvailability { + const availability: VersionAvailability = { + noProducerFields: 0, + unknownProducerVersion: 0, + emptyProducerVersion: 0, + knownProducerVersion: 0, + }; + for (const record of records) { + const hasAnyProducerField = typeof record.producerName === "string" + || typeof record.producerVersion === "string" + || typeof record.instrumentationVersion === "number"; + if (!hasAnyProducerField) { + availability.noProducerFields += 1; + continue; + } + if (typeof record.producerVersion !== "string" || record.producerVersion.trim().length === 0) { + availability.emptyProducerVersion += 1; + continue; + } + if (record.producerVersion.trim() === "unknown") { + availability.unknownProducerVersion += 1; + continue; + } + availability.knownProducerVersion += 1; + } + return availability; +} + +function buildVersionCoverage(events: EvidenceEventV1[], rejections: NormalizedRejection[], currentPackageVersion: string): VersionCoverage { + const coverage: VersionCoverage = { + totalEvents: events.length + rejections.length, + currentVersionEvents: 0, + previousVersionEvents: 0, + unknownVersionEvents: 0, + coveragePercent: 0, + isTransitional: true, + }; + for (const record of [...events, ...rejections]) { + const group = producerVersionGroupFor(record, currentPackageVersion); + if (group === "current") coverage.currentVersionEvents += 1; + if (group === "previous") coverage.previousVersionEvents += 1; + if (group === "unknown_unversioned") coverage.unknownVersionEvents += 1; + } + coverage.coveragePercent = coverage.totalEvents === 0 + ? 0 + : Math.round(((coverage.currentVersionEvents + coverage.previousVersionEvents) / coverage.totalEvents) * 1000) / 10; + coverage.isTransitional = coverage.coveragePercent < 50; + return coverage; } function typeCountsFor(entries: LongTermMemoryEntry[]): Record { @@ -381,10 +1018,23 @@ function provenance(classification: ProvenanceClassification, confidence: Candid }; } -function hasWorkspaceScope(record: NormalizedRejection): boolean { - return Boolean(record.workspaceKey || record.workspaceRoot || record.workspaceRootHash); +function versionContextFor(record: ProducerBearingRecord | undefined, currentPackageVersion: string): ReviewBoardCandidate["versionContext"] | undefined { + if (!record) return undefined; + const group = producerVersionGroupFor(record, currentPackageVersion); + const producerVersion = typeof record.producerVersion === "string" ? record.producerVersion.trim() : undefined; + return { + group, + currentPackageVersion, + ...(producerVersion ? { producerVersion } : {}), + basis: group === "current" + ? "producerVersion matches current package version" + : group === "previous" + ? "producerVersion differs from current package version" + : "producerVersion is missing, unknown, or empty", + }; } + function compareIso(a: string, b: string): number { const aTime = new Date(a).getTime(); const bTime = new Date(b).getTime(); @@ -414,6 +1064,9 @@ function buildReinforcementFacts(events: EvidenceEventV1[]): ReviewBoardReport[" const attempts = events.filter(isReinforcementEvent); const windowBlocked = attempts.filter(event => event.reasonCodes.includes("reinforcement_window_blocked")); const grouped = new Map; rawReasonCodes: Set; eventIds: string[] }>(); + const blocksByExactReason: Record = {}; + const windowBlocksByUtcDay: Record = {}; + let blockDetailsMissing = 0; for (const event of windowBlocked) { const memoryId = event.memory?.memoryId ?? "unknown"; const current = grouped.get(memoryId) ?? { memoryId, count: 0, refs: new Set(), rawReasonCodes: new Set(), eventIds: [] }; @@ -423,6 +1076,15 @@ function buildReinforcementFacts(events: EvidenceEventV1[]): ReviewBoardReport[" if (ref) current.refs.add(ref); for (const reason of event.reasonCodes) current.rawReasonCodes.add(reason); grouped.set(memoryId, current); + + const blockReason = typeof event.details?.blockReason === "string" ? event.details.blockReason : undefined; + if (blockReason) { + blocksByExactReason[blockReason] = (blocksByExactReason[blockReason] ?? 0) + 1; + const utcDay = event.createdAt?.slice(0, 10) || "unknown"; + windowBlocksByUtcDay[utcDay] = (windowBlocksByUtcDay[utcDay] ?? 0) + 1; + } else { + blockDetailsMissing += 1; + } } return { @@ -435,6 +1097,9 @@ function buildReinforcementFacts(events: EvidenceEventV1[]): ReviewBoardReport[" .filter(group => group.count > 1) .sort((a, b) => b.count - a.count || a.memoryId.localeCompare(b.memoryId)) .map(group => ({ memoryId: group.memoryId, count: group.count, refs: [...group.refs].sort(), rawReasonCodes: [...group.rawReasonCodes].sort() })), + blocksByExactReason, + windowBlocksByUtcDay, + blockDetailsMissing, malformedCommandEvents: events.filter(isMalformedCommandEvent).length, }; } @@ -462,6 +1127,10 @@ function buildEvictionFacts( ): ReviewBoardReport["facts"]["systemMechanisms"]["evictionAndCaps"] { const capacityEvents = model.evidenceEvents.filter(event => event.type === "memory_removed_capacity"); const recentCapacityEvents = capacityEvents.filter(event => isWithinDaysOf(event.createdAt, generatedAt, RECENT_EVICTION_DAYS)); + const capacityEventsWithSnapshot = capacityEvents.filter(hasCapacitySnapshot); + const capacityEventsWithRank = capacityEvents.filter(event => numberDetail(event, "rankAtRemoval") !== undefined); + const highestRankRemovedEvent = [...capacityEventsWithRank] + .sort((a, b) => (numberDetail(a, "rankAtRemoval") ?? Number.POSITIVE_INFINITY) - (numberDetail(b, "rankAtRemoval") ?? Number.POSITIVE_INFINITY))[0]; const fullCaps = [ ...(activeMemories.length >= model.store.limits.maxEntries ? ["global"] : []), ...TYPES.filter(type => (typeCounts[type] ?? 0) >= (typeCaps[type] ?? Number.POSITIVE_INFINITY)), @@ -481,9 +1150,44 @@ function buildEvictionFacts( removedByTypeCap: capacityEvents.filter(event => event.reasonCodes.includes("type_cap")).length, recentEvictionsByType: objectFromCounts(countBy(recentCapacityEvents.map(event => event.memory?.type ?? "unknown"))), recentEvictedContentShown: recentCapacityEvents.length, + recentCapacityRemovalsWithSnapshot: capacityEventsWithSnapshot.length, + capacitySnapshotsMissing: capacityEvents.length - capacityEventsWithSnapshot.length, + ...(highestRankRemovedEvent ? { highestRankRemoved: highestRankRemoved(highestRankRemovedEvent) } : {}), }; } +function hasCapacitySnapshot(event: EvidenceEventV1): boolean { + return event.type === "memory_removed_capacity" + && numberDetail(event, "strengthAtRemoval") !== undefined + && numberDetail(event, "rankAtRemoval") !== undefined; +} + +function highestRankRemoved(event: EvidenceEventV1): NonNullable { + const rankAtRemoval = numberDetail(event, "rankAtRemoval") ?? Number.POSITIVE_INFINITY; + const strengthAtRemoval = numberDetail(event, "strengthAtRemoval"); + return { + ...(event.memory?.memoryId ? { memoryId: event.memory.memoryId } : {}), + rankAtRemoval, + ...(strengthAtRemoval !== undefined ? { strengthAtRemoval } : {}), + ...(event.memory?.type ? { type: event.memory.type } : {}), + eventId: event.eventId, + }; +} + +function numberDetail(event: EvidenceEventV1, key: string): number | undefined { + const value = event.details?.[key]; + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function stringDetail(event: EvidenceEventV1, key: string): string | undefined { + const value = event.details?.[key]; + return typeof value === "string" && value.trim().length > 0 ? value : undefined; +} + +function isValidIsoDate(value: string): boolean { + return Number.isFinite(new Date(value).getTime()) && /^\d{4}-\d{2}-\d{2}/.test(value); +} + function isWithinDaysOf(iso: string, referenceIso: string, days: number): boolean { const time = new Date(iso).getTime(); const reference = new Date(referenceIso).getTime(); @@ -638,7 +1342,7 @@ function buildSystemCandidateDisplay( return { candidates, limited, summary: { shown, total, byMechanism } }; } -function buildRejectionCandidates(records: NormalizedRejection[], context: ProvenanceContextInputs, raw: boolean): DatedCandidateInput[] { +function buildRejectionCandidates(records: NormalizedRejection[], context: ProvenanceContextInputs, raw: boolean, currentPackageVersion: string): DatedCandidateInput[] { const candidateRecords = records .filter(record => record.reasons.includes("bad_decision")) .map(record => ({ record, label: neutralRejectionLabel(record) })) @@ -660,7 +1364,8 @@ function buildRejectionCandidates(records: NormalizedRejection[], context: Prove facts: { type: record.type, neutralLabel: label, timestamp: record.timestamp || undefined, origin: record.origin }, evidence: { rawReasonCodes: record.reasons, textPreview: truncate(cleanText(record.text, raw), 120), textAvailable: true }, provenance: classifyProvenance({ rejection: record }, context), - heuristicFlags: [flag(label, label.replaceAll("_", " "), "existing rejection summary grouped this record for human review")], + versionContext: versionContextFor(record, currentPackageVersion), + heuristicFlags: [], reviewQuestions: ["Are rejection filters over-filtering durable decisions or under-filtering non-durable candidates for this workspace?"], nextCommands: ["memory-diag rejected --verbose"], }), @@ -680,7 +1385,7 @@ function neutralRejectionLabel(record: NormalizedRejection): "architecture_like_ return "ambiguous_rejected_candidate"; } -function buildReabsorptionCandidates(matches: ReabsorbedMatch[], context: ProvenanceContextInputs, raw: boolean): DatedCandidateInput[] { +function buildReabsorptionCandidates(matches: ReabsorbedMatch[], context: ProvenanceContextInputs, raw: boolean, currentPackageVersion: string): DatedCandidateInput[] { const candidates = matches.map(match => ({ candidate: candidate({ concernKind: "system_mechanism", @@ -690,7 +1395,8 @@ function buildReabsorptionCandidates(matches: ReabsorbedMatch[], context: Proven facts: { activeMemoryId: match.activeMemory.id, type: match.activeMemory.type, rejectedAt: match.record.timestamp || undefined }, evidence: { rawReasonCodes: match.record.reasons, textPreview: truncate(cleanText(match.record.text, raw), 120), textAvailable: true }, provenance: classifyProvenance({ rejection: match.record, reabsorbed: true }, context), - heuristicFlags: [flag("reabsorbed_rejected_text", "Rejected text appears in active memory", "typed canonical text is present in both rejection records and active memory")], + versionContext: versionContextFor(match.record, currentPackageVersion), + heuristicFlags: [], reviewQuestions: ["Did later context make this rejected candidate worth reviewing for filter calibration?"], nextCommands: ["memory-diag rejected --verbose", `memory-diag explain ${match.activeMemory.id}`], }), @@ -701,7 +1407,7 @@ function buildReabsorptionCandidates(matches: ReabsorbedMatch[], context: Proven return candidates; } -function buildReinforcementCandidates(events: EvidenceEventV1[], context: ProvenanceContextInputs, raw: boolean): DatedCandidateInput[] { +function buildReinforcementCandidates(events: EvidenceEventV1[], context: ProvenanceContextInputs, raw: boolean, currentPackageVersion: string): DatedCandidateInput[] { const blocked = events.filter(event => isReinforcementEvent(event) && event.reasonCodes.includes("reinforcement_window_blocked")); const grouped = [...groupBy(blocked, event => event.memory?.memoryId ?? "unknown").entries()].map(([memoryId, group]) => ({ memoryId, group })); const repeated = grouped.filter(item => item.group.length > 1).map(item => { @@ -715,8 +1421,9 @@ function buildReinforcementCandidates(events: EvidenceEventV1[], context: Proven facts: { memoryId: item.memoryId, blockCount: item.group.length, refs: uniqueStrings(item.group.map(event => String(event.details?.ref ?? "")).filter(Boolean)).sort() }, evidence: { eventIds: item.group.map(event => event.eventId), rawReasonCodes: uniqueStrings(item.group.flatMap(event => event.reasonCodes)).sort(), textAvailable: false }, provenance: classifyProvenance({ event: latest }, context), - heuristicFlags: [flag("repeated_reinforcement_window_block", "Repeated reinforcement window block", `${item.group.length} reinforcement attempts were blocked for memory ${item.memoryId}`)], - reviewQuestions: ["Is the day-based reinforcement window too restrictive when the same memory receives repeated reinforce intent?"], + versionContext: versionContextFor(latest, currentPackageVersion), + heuristicFlags: [flag("repeated_reinforcement_window_block", "Repeated reinforcement window block inventory", `${item.group.length} reinforcement attempts were blocked for memory ${item.memoryId}`)], + reviewQuestions: ["What reinforcement block patterns are present for repeated reinforce intent?"], nextCommands: ["memory-diag commands --verbose", `memory-diag explain ${item.memoryId}`], }), timestamp: latest?.createdAt, @@ -733,6 +1440,7 @@ function buildReinforcementCandidates(events: EvidenceEventV1[], context: Proven facts: { eventType: event.type, createdAt: event.createdAt }, evidence: { eventIds: [event.eventId], rawReasonCodes: event.reasonCodes, textPreview: event.textPreview ? truncate(cleanText(event.textPreview, raw), 120) : undefined, textAvailable: Boolean(event.textPreview) }, provenance: classifyProvenance({ event }, context), + versionContext: versionContextFor(event, currentPackageVersion), heuristicFlags: [flag("malformed_numbered_command", "Malformed numbered-memory command evidence", "command parser rejected a memory command form")], reviewQuestions: ["Do numbered-memory command rules match how agents actually express reinforcement intent?"], nextCommands: ["memory-diag commands --verbose"], @@ -750,6 +1458,7 @@ function buildEvictionCandidates( context: ProvenanceContextInputs, raw: boolean, generatedAt: string, + currentPackageVersion: string, ): DatedCandidateInput[] { const recentCapacity = events.filter(event => event.type === "memory_removed_capacity" && isWithinDaysOf(event.createdAt, generatedAt, RECENT_EVICTION_DAYS)); const capacityCandidates = recentCapacity.map(event => ({ @@ -761,8 +1470,9 @@ function buildEvictionCandidates( facts: { ...(safeDetails(event.details, raw) ?? {}), createdAt: event.createdAt, memoryId: event.memory?.memoryId, type: event.memory?.type }, evidence: { eventIds: [event.eventId], rawReasonCodes: event.reasonCodes, textPreview: event.textPreview ? truncate(cleanText(event.textPreview, raw), 120) : undefined, textAvailable: Boolean(event.textPreview) }, provenance: classifyProvenance({ event }, context), - heuristicFlags: [flag("recent_capacity_removal", "Recent capacity-removal evidence", "memory_removed_capacity appeared within the recent eviction window")], - reviewQuestions: ["Are eviction and cap rules preserving the intended memories under pressure?"], + versionContext: versionContextFor(event, currentPackageVersion), + heuristicFlags: [flag("recent_capacity_removal", "Recent capacity-removal inventory", "memory_removed_capacity appeared within the recent eviction window")], + reviewQuestions: ["What capacity-removal inventory is present for this memory?"], nextCommands: ["memory-diag missing --verbose --explain"], }), timestamp: event.createdAt, @@ -783,8 +1493,9 @@ function buildEvictionCandidates( facts: { memoryId: row.id, terminalType: row.terminalType, eventCount: row.events.length }, evidence: { eventIds: row.events.map(event => event.eventId), rawReasonCodes: uniqueStrings(row.events.flatMap(event => event.reasonCodes)).sort(), textPreview: latest?.textPreview ? truncate(cleanText(latest.textPreview, raw), 120) : undefined, textAvailable: Boolean(latest?.textPreview) }, provenance: classifyProvenance({ event: latest }, context), - heuristicFlags: [flag("unknown_disappearance", "Evidence-only disappearance without terminal removal evidence", `memory ${row.id} has evidence but is not active`)], - reviewQuestions: ["Does missing-memory evidence indicate a cap, retention, or recording rule needs review?"], + versionContext: versionContextFor(latest, currentPackageVersion), + heuristicFlags: [flag("unknown_disappearance", "Unversioned disappearance inventory", `memory ${row.id} has evidence but is not active`)], + reviewQuestions: ["What unversioned disappearance inventory exists for this memory?"], nextCommands: ["memory-diag missing --verbose --explain"], }), timestamp: latest?.createdAt, @@ -800,7 +1511,7 @@ function safeDetails(details: EvidenceEventV1["details"], raw: boolean): Record< return Object.fromEntries(Object.entries(details).map(([key, value]) => [key, typeof value === "string" ? cleanText(value, raw) : value])); } -function buildIdentityCandidates(model: MemoryInspectionReadModel, activeMemories: LongTermMemoryEntry[], context: ProvenanceContextInputs, raw: boolean): DatedCandidateInput[] { +function buildIdentityCandidates(model: MemoryInspectionReadModel, activeMemories: LongTermMemoryEntry[], context: ProvenanceContextInputs, raw: boolean, currentPackageVersion: string): DatedCandidateInput[] { const replacementCandidates = model.evidenceEvents .filter(event => event.type === "memory_replaced_numbered_ref" || event.type === "promotion_superseded") .map(event => ({ @@ -812,7 +1523,8 @@ function buildIdentityCandidates(model: MemoryInspectionReadModel, activeMemorie facts: { eventType: event.type, memoryId: event.memory?.memoryId, relationRoles: event.relations?.map(relation => relation.role) ?? [] }, evidence: { eventIds: [event.eventId], rawReasonCodes: event.reasonCodes, textPreview: event.textPreview ? truncate(cleanText(event.textPreview, raw), 120) : undefined, textAvailable: Boolean(event.textPreview) }, provenance: classifyProvenance({ event }, context), - heuristicFlags: [flag("replacement_or_supersession", "Replacement or supersession evidence", `${event.type} records identity/dedup behavior`)], + versionContext: versionContextFor(event, currentPackageVersion), + heuristicFlags: [], reviewQuestions: ["Are identity and dedup rules preserving separate memories when expected to remain distinct?"], nextCommands: ["memory-diag commands --verbose", event.memory?.memoryId ? `memory-diag explain ${event.memory.memoryId}` : "memory-diag missing --verbose --explain"], }), @@ -829,7 +1541,7 @@ function buildIdentityCandidates(model: MemoryInspectionReadModel, activeMemorie facts: { memoryIds: group.memoryIds, basis: group.basis }, evidence: { eventIds: group.memoryIds.flatMap(id => (model.evidenceByMemoryId.get(id) ?? []).map(event => event.eventId)), rawReasonCodes: [], textAvailable: false }, provenance: classifyProvenance({}, context), - heuristicFlags: [flag("exact_duplicate_group", "Exact duplicate text or identity group", `${group.memoryIds.length} memories share ${group.basis}`)], + heuristicFlags: [], reviewQuestions: ["Are exact duplicate text or identity groups expected for this workspace?"], nextCommands: group.memoryIds.map(id => `memory-diag explain ${id}`).slice(0, 3), }), @@ -893,8 +1605,8 @@ function flag(id: string, label: string, evidence: string): HeuristicFlag { function systemMechanismQuestions(): string[] { return [ "Are rejection rules over-filtering durable decisions or under-filtering non-durable candidates for this workspace?", - "Is the reinforcement window too restrictive when the same memory receives repeated reinforce intent?", - "Are eviction and cap rules preserving target memories under full caps?", + "What block patterns are present for repeated reinforcement intent?", + "What cap occupancy and capacity-removal inventory is present?", "Are identity and dedup rules collapsing items expected to remain separate, or not collapsing equivalent items?", ]; } diff --git a/scripts/memory-diag/rejections-model.ts b/scripts/memory-diag/rejections-model.ts index 8f02cc6..4bafc04 100644 --- a/scripts/memory-diag/rejections-model.ts +++ b/scripts/memory-diag/rejections-model.ts @@ -27,6 +27,11 @@ export function normalizeRejection(record: RejectionLogRecord): NormalizedReject source: record.source, origin, fromTrigger: typeof record.fromTrigger === "boolean" ? record.fromTrigger : origin === "explicit_trigger", + producerName: typeof record.producerName === "string" ? record.producerName : undefined, + producerVersion: typeof record.producerVersion === "string" ? record.producerVersion : undefined, + instrumentationVersion: typeof record.instrumentationVersion === "number" ? record.instrumentationVersion : undefined, + decisionLogicName: typeof record.decisionLogicName === "string" ? record.decisionLogicName : undefined, + decisionLogicVersion: typeof record.decisionLogicVersion === "number" ? record.decisionLogicVersion : undefined, text: record.text, reasons: record.reasons, }; diff --git a/scripts/memory-diag/types.ts b/scripts/memory-diag/types.ts index 8678cfd..95dd6f2 100644 --- a/scripts/memory-diag/types.ts +++ b/scripts/memory-diag/types.ts @@ -93,6 +93,11 @@ export type RejectionLogRecord = { fromTrigger?: boolean; text?: string; reasons?: string[]; + producerName?: string; + producerVersion?: string; + instrumentationVersion?: number; + decisionLogicName?: string; + decisionLogicVersion?: number; }; export type NormalizedRejection = Required> & { @@ -102,6 +107,11 @@ export type NormalizedRejection = Required { diff --git a/src/extractors.ts b/src/extractors.ts index 247c733..67f6b7a 100644 --- a/src/extractors.ts +++ b/src/extractors.ts @@ -7,6 +7,7 @@ import { assessMemoryQuality } from "./memory-quality.ts"; import { extractionRejectionLogPath } from "./paths.ts"; import { redactCredentials } from "./redaction.ts"; import type { EvidenceEventInput } from "./evidence-log.ts"; +import { producerFields } from "./instrumentation.ts"; function id(prefix: string): string { return `${prefix}_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; @@ -329,6 +330,11 @@ type ExtractionRejectionLogEntry = { source: "compaction"; workspaceKey?: string; workspaceRootHash?: string; + producerName?: string; + producerVersion?: string; + instrumentationVersion?: number; + decisionLogicName?: string; + decisionLogicVersion?: number; }; type WorkspaceMemoryCandidateParseOptions = { @@ -381,6 +387,9 @@ function evaluateWorkspaceMemoryCandidate( source: "compaction", workspaceKey: options.workspaceKey, workspaceRootHash: options.workspaceRootHash, + ...producerFields(), + decisionLogicName: "assessMemoryQuality", + decisionLogicVersion: 1, }); return { accepted: false, reasons: quality.reasons }; } diff --git a/src/instrumentation.ts b/src/instrumentation.ts new file mode 100644 index 0000000..22268e0 --- /dev/null +++ b/src/instrumentation.ts @@ -0,0 +1,41 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +let cachedVersion: string | undefined; + +const MEMORY_PRODUCER_NAME = "opencode-working-memory"; +const MEMORY_INSTRUMENTATION_VERSION = 2; + +function producerVersion(): string { + if (cachedVersion) return cachedVersion; + try { + const candidates = [ + join(__dirname, "..", "package.json"), + join(__dirname, "..", "..", "package.json"), + // resolve from compiled dist/src/ -> repo root + ]; + for (const path of candidates) { + try { + const pkg = JSON.parse(readFileSync(path, "utf8")); + cachedVersion = pkg.version as string; + break; + } catch { + // try next + } + } + if (!cachedVersion) cachedVersion = "unknown"; + } catch { + cachedVersion = "unknown"; + } + return cachedVersion; +} + +export function producerFields(): { producerName: string; producerVersion: string; instrumentationVersion: number } { + return { + producerName: MEMORY_PRODUCER_NAME, + producerVersion: producerVersion(), + instrumentationVersion: MEMORY_INSTRUMENTATION_VERSION, + }; +} diff --git a/src/plugin.ts b/src/plugin.ts index 275fc28..f4d7dae 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -44,7 +44,7 @@ import { workspaceMemoryExactKey, workspaceMemoryIdentityKey, } from "./workspace-memory.ts"; -import { reinforceMemory } from "./retention.ts"; +import { tryReinforceMemory } from "./retention.ts"; import { appendPendingMemories, clearPendingMemories, @@ -429,17 +429,31 @@ export const MemoryV2Plugin: Plugin = async (input) => { const { refSnapshot, target, targetIndex } = resolution; if (command.kind === "REINFORCE") { - const reinforced = reinforceMemory(target, sessionID, now); - if (reinforced === target) { - evidence.push(memoryReinforcedEvidence(target, command.ref, "rejected", ["numbered_ref_reinforce", "reinforcement_window_blocked"], { + const decision = tryReinforceMemory(target, sessionID, now); + if (decision.outcome === "blocked") { + evidence.push(memoryReinforcedEvidence(target, command.ref, "rejected", ["numbered_ref_reinforce", "reinforcement_window_blocked", `reinforcement_block_${decision.blockReason}`], { memoryId: refSnapshot.memoryId, + blockReason: decision.blockReason, + attemptedAtMs: now, + attemptedAtIso: new Date(now).toISOString(), + ...(decision.lastReinforcedAt ? { + lastReinforcedAtMs: decision.lastReinforcedAt, + lastReinforcedAtIso: new Date(decision.lastReinforcedAt).toISOString(), + } : {}), + reinforcementCount: decision.reinforcementCount, + maxReinforcementCount: decision.maxReinforcementCount, + minIntervalMs: decision.minIntervalMs, })); continue; } + const reinforced = decision.memory; workspaceMemory.entries[targetIndex] = reinforced; evidence.push(memoryReinforcedEvidence(reinforced, command.ref, "reinforced", ["numbered_ref_reinforce", "reinforcement_window_allowed"], { memoryId: refSnapshot.memoryId, + reinforcementOutcome: "reinforced", + previousReinforcementCount: decision.previousReinforcementCount, + newReinforcementCount: decision.newReinforcementCount, })); continue; } @@ -662,11 +676,12 @@ export const MemoryV2Plugin: Plugin = async (input) => { const key = memoryKey(memory); const existing = existingByKey.get(key); if (existing) { - const reinforced = reinforceMemory( + const decision = tryReinforceMemory( existing.memory, sessionID ?? memory.pendingOwnerSessionID ?? "workspace-promotion", promotedAt, ); + const reinforced = decision.memory; if (reinforced !== existing.memory) { workspaceMemory.entries[existing.index] = reinforced; existingByKey.set(key, { memory: reinforced, index: existing.index }); diff --git a/src/retention.ts b/src/retention.ts index b975a0f..40bca10 100644 --- a/src/retention.ts +++ b/src/retention.ts @@ -1,5 +1,11 @@ import type { LongTermMemoryEntry, WorkspaceMemoryStore } from "./types.ts"; +export type ReinforcementBlockReason = "same_session" | "same_utc_day" | "min_interval" | "max_count"; + +export type ReinforcementDecision = + | { outcome: "reinforced"; memory: LongTermMemoryEntry; previousReinforcementCount: number; newReinforcementCount: number } + | { outcome: "blocked"; memory: LongTermMemoryEntry; blockReason: ReinforcementBlockReason; lastReinforcedAt?: number; reinforcementCount: number; maxReinforcementCount: number; minIntervalMs: number }; + // Retention decay model constants (v1.5) export const BASE_HALF_LIFE_DAYS = 45; export const REINFORCEMENT_HALFLIFE_FACTOR = 0.85; @@ -116,33 +122,59 @@ function isSameUTCCalendarDay(ts1: number, ts2: number): boolean { && d1.getUTCDate() === d2.getUTCDate(); } -export function reinforceMemory( +export function tryReinforceMemory( memory: LongTermMemoryEntry, sessionId: string, now: number, -): LongTermMemoryEntry { - if (memory.lastReinforcedSessionID === sessionId) { - return memory; +): ReinforcementDecision { + const count = memory.reinforcementCount ?? 0; + const lastAt = memory.lastReinforcedAt ?? 0; + const lastSession = memory.lastReinforcedSessionID; + + if (lastSession === sessionId) { + return blockedDecision(memory, "same_session", count, lastAt); } - // Calendar-day diversity gate (OQ-2): same UTC day = no reinforcement. - if (memory.lastReinforcedAt && isSameUTCCalendarDay(memory.lastReinforcedAt, now)) { - return memory; + if (count >= REINFORCEMENT_MAX_COUNT) { + return blockedDecision(memory, "max_count", count, lastAt); } - if (memory.lastReinforcedAt && now - memory.lastReinforcedAt < REINFORCEMENT_MIN_INTERVAL_MS) { - return memory; + if (lastAt > 0 && now < lastAt + REINFORCEMENT_MIN_INTERVAL_MS) { + return blockedDecision(memory, "min_interval", count, lastAt); } - if ((memory.reinforcementCount ?? 0) >= REINFORCEMENT_MAX_COUNT) { - return memory; + if (lastAt > 0 && isSameUTCCalendarDay(lastAt, now)) { + return blockedDecision(memory, "same_utc_day", count, lastAt); } - return { + const reinforced: LongTermMemoryEntry = { ...memory, - reinforcementCount: (memory.reinforcementCount ?? 0) + 1, + reinforcementCount: count + 1, lastReinforcedAt: now, lastReinforcedSessionID: sessionId, retentionClock: now, }; + return { + outcome: "reinforced", + memory: reinforced, + previousReinforcementCount: count, + newReinforcementCount: count + 1, + }; +} + +function blockedDecision( + memory: LongTermMemoryEntry, + blockReason: ReinforcementBlockReason, + reinforcementCount: number, + lastReinforcedAt: number, +): ReinforcementDecision { + return { + outcome: "blocked", + memory, + blockReason, + ...(lastReinforcedAt > 0 ? { lastReinforcedAt } : {}), + reinforcementCount, + maxReinforcementCount: REINFORCEMENT_MAX_COUNT, + minIntervalMs: REINFORCEMENT_MIN_INTERVAL_MS, + }; } diff --git a/src/workspace-memory.ts b/src/workspace-memory.ts index b7e97c6..3be5953 100644 --- a/src/workspace-memory.ts +++ b/src/workspace-memory.ts @@ -9,7 +9,8 @@ import { redactCredentials } from "./redaction.ts"; import { RETENTION_TYPE_MAX, calculateRetentionStrength, - reinforceMemory, + tryReinforceMemory, + type ReinforcementDecision, } from "./retention.ts"; import type { EvidenceEventInput, MemoryEvidenceRef } from "./evidence-log.ts"; import { appendEvidenceEvents } from "./evidence-log.ts"; @@ -559,7 +560,13 @@ function consolidationEvent( function capacityRemovalEvidence( memory: LongTermMemoryEntry, - reason: "type_cap" | "global_cap" | "capacity", + reason: "type_cap" | "global_cap", + details: { + strengthAtRemoval: number; + rankAtRemoval: number; + typeRankAtRemoval: number; + ageDaysAtRemoval: number; + }, ): EvidenceEventInput { return { type: "memory_removed_capacity", @@ -578,6 +585,7 @@ function capacityRemovalEvidence( ...(typeof memory.retentionClock === "number" && Number.isFinite(memory.retentionClock) ? { retentionClock: memory.retentionClock } : {}), ...(memory.createdAt ? { createdAt: memory.createdAt } : {}), ...(memory.source ? { source: memory.source } : {}), + ...details, }, }; } @@ -664,8 +672,8 @@ export function enforceLongTermLimitsWithAccounting( const typeCapLosers = sorted.filter(entry => !cappedIds.has(entry.id)); const globalCapLosers = capped.filter(entry => !keptIds.has(entry.id)); const capacityEvidence: EvidenceEventInput[] = [ - ...typeCapLosers.map(entry => capacityRemovalEvidence(entry, "type_cap")), - ...globalCapLosers.map(entry => capacityRemovalEvidence(entry, "global_cap")), + ...typeCapLosers.map(entry => capacityRemovalEvidence(entry, "type_cap", capacityRemovalSnapshot(entry, sorted, now, lastActivityAt))), + ...globalCapLosers.map(entry => capacityRemovalEvidence(entry, "global_cap", capacityRemovalSnapshot(entry, sorted, now, lastActivityAt))), ]; const capacityDropped = sorted .filter(entry => !keptIds.has(entry.id)) @@ -680,6 +688,28 @@ export function enforceLongTermLimitsWithAccounting( }; } +function capacityRemovalSnapshot( + memory: LongTermMemoryEntry, + sorted: LongTermMemoryEntry[], + now: number, + lastActivityAt?: string, +): { + strengthAtRemoval: number; + rankAtRemoval: number; + typeRankAtRemoval: number; + ageDaysAtRemoval: number; +} { + const createdAtMs = new Date(memory.createdAt).getTime(); + const rank = sorted.findIndex(entry => entry.id === memory.id); + const typeRank = sorted.filter(entry => entry.type === memory.type).findIndex(entry => entry.id === memory.id); + return { + strengthAtRemoval: calculateRetentionStrength(memory, now, lastActivityAt), + rankAtRemoval: rank >= 0 ? rank + 1 : -1, + typeRankAtRemoval: typeRank >= 0 ? typeRank + 1 : -1, + ageDaysAtRemoval: Number.isFinite(createdAtMs) ? Math.floor(Math.max(0, now - createdAtMs) / 86_400_000) : 0, + }; +} + function applyTypeMaxCaps(entries: LongTermMemoryEntry[]): LongTermMemoryEntry[] { return applyTypeMaxCapsWithOmissions(entries).kept; } @@ -732,12 +762,13 @@ export function dedupeLongTermEntriesWithAccounting(entries: LongTermMemoryEntry const reason = workspaceMemoryExactKey(entry) === workspaceMemoryExactKey(existing) ? "absorbed_exact" as const : "absorbed_identity" as const; - const reinforced = reinforceMemory( + const decision = tryReinforceMemory( retained, reinforcementSessionId(retained, dropped), now, ); - const reinforcedEvent = reinforcementEvidence(retained, dropped, reinforced, reason); + const reinforced = decision.memory; + const reinforcedEvent = reinforcementEvidence(retained, dropped, decision, reason, now); if (reinforcedEvent) evidence.push(reinforcedEvent); absorbed.push(consolidationEvent(dropped, reason, reinforced)); @@ -760,12 +791,13 @@ export function dedupeLongTermEntriesWithAccounting(entries: LongTermMemoryEntry const reason = workspaceMemoryExactKey(entry) === workspaceMemoryExactKey(existing) ? "absorbed_exact" as const : "superseded_existing" as const; // v1.5.4 placeholder: unreachable until numbered refs - const reinforced = reinforceMemory( + const decision = tryReinforceMemory( retained, reinforcementSessionId(retained, dropped), now, ); - const reinforcedEvent = reinforcementEvidence(retained, dropped, reinforced, reason); + const reinforced = decision.memory; + const reinforcedEvent = reinforcementEvidence(retained, dropped, decision, reason, now); if (reinforcedEvent) evidence.push(reinforcedEvent); if (reason === "superseded_existing") { @@ -807,11 +839,41 @@ function memoryEvidenceRef(memory: LongTermMemoryEntry): MemoryEvidenceRef { function reinforcementEvidence( retained: LongTermMemoryEntry, dropped: LongTermMemoryEntry, - reinforced: LongTermMemoryEntry, + decision: ReinforcementDecision, reason: "absorbed_exact" | "absorbed_identity" | "superseded_existing", + attemptedAt: number, ): EvidenceEventInput | undefined { - if ((reinforced.reinforcementCount ?? 0) <= (retained.reinforcementCount ?? 0)) return undefined; const duplicateReason = reason === "absorbed_identity" ? "duplicate_identity" : "duplicate_exact"; + if (decision.outcome === "blocked") { + return { + type: "memory_reinforced", + phase: "reinforcement", + outcome: "rejected", + memory: memoryEvidenceRef(retained), + relations: [ + { role: "target", memory: memoryEvidenceRef(retained) }, + { role: "reinforced_by", memory: memoryEvidenceRef(dropped) }, + ], + reasonCodes: [duplicateReason, "reinforcement_window_blocked", `reinforcement_block_${decision.blockReason}`], + details: { + memoryId: retained.id, + droppedMemoryId: dropped.id, + blockReason: decision.blockReason, + attemptedAtMs: attemptedAt, + attemptedAtIso: new Date(attemptedAt).toISOString(), + ...(decision.lastReinforcedAt ? { + lastReinforcedAtMs: decision.lastReinforcedAt, + lastReinforcedAtIso: new Date(decision.lastReinforcedAt).toISOString(), + } : {}), + reinforcementCount: decision.reinforcementCount, + maxReinforcementCount: decision.maxReinforcementCount, + minIntervalMs: decision.minIntervalMs, + }, + textPreview: retained.text, + }; + } + + const reinforced = decision.memory; return { type: "memory_reinforced", phase: "reinforcement", @@ -822,6 +884,13 @@ function reinforcementEvidence( { role: "reinforced_by", memory: memoryEvidenceRef(dropped) }, ], reasonCodes: [duplicateReason, "reinforcement_window_allowed"], + details: { + memoryId: reinforced.id, + droppedMemoryId: dropped.id, + reinforcementOutcome: "reinforced", + previousReinforcementCount: decision.previousReinforcementCount, + newReinforcementCount: decision.newReinforcementCount, + }, textPreview: reinforced.text, }; } diff --git a/tests/memory-diag-quality.test.ts b/tests/memory-diag-quality.test.ts index a99f331..6c63b7d 100644 --- a/tests/memory-diag-quality.test.ts +++ b/tests/memory-diag-quality.test.ts @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, readFileSync } from "node:fs"; import { mkdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; @@ -13,12 +13,14 @@ import { workspaceKey, workspaceMemoryPath } from "../src/paths.ts"; import { buildQualityJSON, formatQualityReviewBoard } from "../scripts/memory-diag/formatters/quality.ts"; import { groupEvidenceByMemoryId } from "../scripts/memory-diag/evidence-model.ts"; import { buildQualityReviewBoard, type ProvenanceClassification, type ReviewBoardReport } from "../scripts/memory-diag/quality-review-model.ts"; +import { normalizeRejection } from "../scripts/memory-diag/rejections-model.ts"; import { retentionCandidatesForDiag } from "../scripts/memory-diag/retention-model.ts"; import type { MemoryInspectionReadModel, NormalizedRejection, WorkspaceDiagSnapshot } from "../scripts/memory-diag/types.ts"; const execFileAsync = promisify(execFile); const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); const generatedAt = "2026-05-11T12:00:00.000Z"; +const packageJson = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")) as { version: string }; async function runMemoryDiag(args: string[]): Promise { const { stdout } = await execFileAsync(process.execPath, [ @@ -351,6 +353,117 @@ test("quality json includes all system mechanism candidates without verbose", as } }); +test("new evidence events include producer metadata and historical events remain accepted", async () => { + const root = mkdtempSync(join(tmpdir(), "opencode-memory-diag-quality-producer-")); + try { + const [stored] = await appendEvidenceEvents(root, [event("evt-input-producer", { + type: "render_selected", + phase: "render", + outcome: "rendered", + memory: { memoryId: "mem-produced", type: "decision", source: "compaction" }, + })]); + + assert.equal(stored.producerName, "opencode-working-memory"); + assert.equal(stored.producerVersion, packageJson.version); + assert.equal(stored.instrumentationVersion, 2); + + const historical = event("evt-historical-no-producer", { + type: "render_selected", + phase: "render", + outcome: "rendered", + memory: { memoryId: "mem-historical", type: "decision", source: "compaction" }, + }); + const report = buildQualityReviewBoard(inspectionModel([ + entry("mem-historical", "Historical uninstrumented memory", "decision"), + ], [historical]), {}, generatedAt); + assert.equal(report.facts.memoryContent.evidenceCoverage.covered, 1); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("rejection normalization preserves producer and decision metadata", () => { + const normalized = normalizeRejection({ + timestamp: "2026-05-12T00:00:00.000Z", + type: "decision", + source: "compaction", + text: "Rejected diagnostic metadata candidate", + reasons: ["bad_decision"], + producerName: "opencode-working-memory", + producerVersion: packageJson.version, + instrumentationVersion: 2, + decisionLogicName: "assessMemoryQuality", + decisionLogicVersion: 1, + }); + + assert.ok(normalized); + assert.equal(normalized.producerName, "opencode-working-memory"); + assert.equal(normalized.producerVersion, packageJson.version); + assert.equal(normalized.instrumentationVersion, 2); + assert.equal(normalized.decisionLogicName, "assessMemoryQuality"); + assert.equal(normalized.decisionLogicVersion, 1); +}); + +test("quality upgrades answerability when producer-instrumented block reasons exist", () => { + const model = inspectionModelWithProducerEvents(); + + const report = buildQualityReviewBoard(model, {}, generatedAt); + + assert.ok(report.facts.systemMechanisms.instrumentation); + assert.ok(report.facts.systemMechanisms.instrumentation.evidenceEventsWithProducer > 0); + assert.equal(report.facts.systemMechanisms.instrumentation.evidenceEventsTotal, 3); + assert.equal(report.facts.systemMechanisms.reinforcementRules.blocksByExactReason.same_utc_day, 1); + assert.equal(report.facts.systemMechanisms.reinforcementRules.blocksByExactReason.same_session, 1); + assert.deepEqual(report.facts.systemMechanisms.reinforcementRules.windowBlocksByUtcDay, { "2026-05-11": 2 }); + assert.equal(report.facts.systemMechanisms.reinforcementRules.blockDetailsMissing, 1); + + assert.equal(report.answerability?.reinforcementRules?.level, "partial"); + + const output = formatQualityReviewBoard(report, {}); + assert.match(output, /Producer coverage: 3 of 3 evidence events instrumented/); + assert.match(output, /Exact block reasons: same_session=1, same_utc_day=1/); + assert.match(output, /Answerability: partial.*causal fields exist, but human content judgment is still required/); +}); + +test("quality keeps reinforcement inventory-only for uninstrumented block reasons", () => { + const model = inspectionModel([], [ + event("evt-old-block", { + type: "memory_reinforced", + phase: "reinforcement", + outcome: "rejected", + memory: { memoryId: "mem-old-block", type: "decision", source: "compaction" }, + reasonCodes: ["reinforcement_window_blocked"], + details: { blockReason: "same_utc_day" }, + }), + ]); + + const report = buildQualityReviewBoard(model, {}, generatedAt); + + assert.equal(report.facts.systemMechanisms.instrumentation.evidenceEventsWithProducer, 0); + assert.equal(report.answerability?.reinforcementRules?.level, "inventory_only"); +}); + +test("quality shows capacity snapshot facts when present", () => { + const model = inspectionModelWithCapacitySnapshots(); + + const report = buildQualityReviewBoard(model, {}, generatedAt); + + assert.equal(report.facts.systemMechanisms.evictionAndCaps.recentCapacityRemovalsWithSnapshot, 1); + assert.equal(report.facts.systemMechanisms.evictionAndCaps.capacitySnapshotsMissing, 1); + assert.deepEqual(report.facts.systemMechanisms.evictionAndCaps.highestRankRemoved, { + memoryId: "mem-cap-snapshot", + rankAtRemoval: 2, + strengthAtRemoval: 0.82, + type: "decision", + eventId: "evt-cap-snapshot", + }); + assert.equal(report.answerability?.evictionAndCaps?.level, "partial"); + + const output = formatQualityReviewBoard(report, {}); + assert.match(output, /Removals with snapshot: 1/); + assert.match(output, /Removals without snapshot: 1 \(historical\)/); +}); + test("quality review model builds system mechanism facts and neutral candidates", () => { const active = [ entry("mem-a", "Retention architecture uses evidence windows for durable review", "decision"), @@ -416,6 +529,34 @@ test("quality review model builds system mechanism facts and neutral candidates" assert.doesNotMatch(JSON.stringify(report), /secret-value|\/tmp\/private/); }); +test("quality report includes answerability for all mechanism sections", () => { + const report = buildQualityReviewBoard(inspectionModel([ + entry("mem-answerability", "Answerability contract memory", "decision"), + ], []), {}, generatedAt); + + // Every diagnostic section must have an answerability contract. + assert.ok(report.answerability, "report must have answerability field"); + assert.equal(report.answerability.reinforcementRules?.level, "inventory_only"); + assert.equal(report.answerability.evictionAndCaps?.level, "inventory_only"); + assert.equal(report.answerability.unknownDisappearances?.level, "inventory_only"); +}); + +test("quality human output avoids misleading phrases", () => { + const report = buildQualityReviewBoard(inspectionModel([ + entry("mem-wording", "Neutral wording memory", "decision"), + ], []), {}, generatedAt); + const output = formatQualityReviewBoard(report, { verbose: true }); + const lower = output.toLowerCase(); + assert.match(output, /Answerability: inventory_only/); + assert.match(output, /Output permission:/); + assert.doesNotMatch(lower, /reference shortage/); + assert.doesNotMatch(lower, /reference insufficient/); + assert.doesNotMatch(lower, /cap pressure/); + assert.doesNotMatch(lower, /window too strict/); + assert.doesNotMatch(lower, /current bug/); + assert.doesNotMatch(lower, /highest-value/); +}); + test("quality review model includes provenance timeline and classification counts", () => { const active = [entry("mem-active", "Reabsorbed post rejection candidate", "decision")]; const events = [ @@ -465,6 +606,385 @@ test("quality review model exposes required JSON shape with neutral language", ( } }); +test("versioned quality facts are additive in JSON shape", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-version-shape", packageJson.version, false), + ]), {}, generatedAt); + + assertReviewBoardShape(report); + const versioned = report.facts.systemMechanisms.versionedFacts; + assert.ok(versioned); + assert.equal(versioned.currentPackageVersion, packageJson.version); + assert.deepEqual(versioned.buckets, ["current", "previous", "unknown_unversioned"]); + assert.ok(report.facts.systemMechanisms.reinforcementRules); +}); + +test("versioned quality distinguishes all-evidence coverage from mechanism opportunities", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + ...Array.from({ length: 20 }, (_, index) => versionedEvent(`evt-render-current-${index}`, packageJson.version, { + type: "render_selected", + phase: "render", + outcome: "rendered", + memory: { memoryId: `mem-${index}`, type: "decision", source: "compaction" }, + })), + reinforcementAttempt("evt-current-block", packageJson.version, true), + ]), {}, generatedAt); + const output = formatQualityReviewBoard(report, {}); + + assert.match(output, /version-stamp coverage/i); + assert.match(output, /not mechanism problem counts/i); + assert.match(output, /Mechanism opportunities below are reinforcement attempts only; render\/accounting events are excluded\./); + assert.match(output, /current version .*opportunities=1, observed=1/); +}); + +test("versioned quality detects previous-only reinforcement pattern with sufficient current sample", () => { + const events = [ + reinforcementAttempt("evt-prev-block-1", "1.6.0", true), + reinforcementAttempt("evt-prev-block-2", "1.6.0", true), + ...Array.from({ length: 5 }, (_, index) => reinforcementAttempt(`evt-current-ok-${index}`, packageJson.version, false)), + reinforcementAttempt("evt-unknown-block", undefined, true), + ]; + const report = buildQualityReviewBoard(inspectionModel([], events), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + + assert.equal(facts?.buckets.current.opportunityCount, 5); + assert.equal(facts?.buckets.current.observedPatternCount, 0); + assert.equal(facts?.buckets.current.sampleAssessment, "not_observed_with_sufficient_sample"); + assert.match(facts?.inference.message ?? "", /No recurrence observed/); + assert.doesNotMatch(facts?.inference.message ?? "", /fixed|resolved|absent in current version/i); +}); + +test("versioned quality reports current reinforcement recurrence", () => { + const events = [ + reinforcementAttempt("evt-prev-block", "1.6.0", true), + reinforcementAttempt("evt-current-block", packageJson.version, true), + ...Array.from({ length: 5 }, (_, index) => reinforcementAttempt(`evt-current-any-${index}`, packageJson.version, false)), + ]; + const report = buildQualityReviewBoard(inspectionModel([], events), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + const output = formatQualityReviewBoard(report, {}); + + assert.equal(facts?.inference.status, "pattern_persists_across_versions"); + assert.match(output, /Current recurrence detected/); +}); + +test("versioned reinforcement inference surfaces current exact block reasons", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-current-same-session", packageJson.version, true, { + details: { blockReason: "same_session" }, + reasonCodes: ["numbered_ref_reinforce", "reinforcement_window_blocked", "reinforcement_block_same_session"], + }), + ...Array.from({ length: 4 }, (_, index) => reinforcementAttempt(`evt-current-ok-${index}`, packageJson.version, false)), + ]), {}, generatedAt); + const output = formatQualityReviewBoard(report, {}); + const reinforcementBlock = output.slice(output.indexOf(" Reinforcement rules"), output.indexOf(" Eviction and caps")); + + assert.match(reinforcementBlock, /(diagnostic: current block reasons=same_session=1|inference: .*same_session=1)/s); +}); + +test("versioned reinforcement inference does not invent missing block reasons", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-current-missing-reason", packageJson.version, true, { + details: {}, + reasonCodes: ["numbered_ref_reinforce", "reinforcement_window_blocked"], + }), + ...Array.from({ length: 4 }, (_, index) => reinforcementAttempt(`evt-current-ok-missing-${index}`, packageJson.version, false)), + ]), {}, generatedAt); + const output = formatQualityReviewBoard(report, {}); + + assert.match(output, /block details missing=1/); + assert.doesNotMatch(output, /same_session=1/); + assert.doesNotMatch(output, /same_utc_day=1/); +}); + +test("quality surfaces same-session cross-day reinforcement as design diagnostic question", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-cross-day-same-session", packageJson.version, true, { + createdAt: "2026-05-13T07:48:21.364Z", + reasonCodes: ["numbered_ref_reinforce", "reinforcement_window_blocked", "reinforcement_block_same_session"], + details: { + blockReason: "same_session", + attemptedAtIso: "2026-05-13T07:48:21.361Z", + lastReinforcedAtIso: "2026-05-12T09:19:08.708Z", + }, + }), + reinforcementAttempt("evt-cross-day-same-session-older", packageJson.version, true, { + createdAt: "2026-05-12T07:48:21.364Z", + reasonCodes: ["numbered_ref_reinforce", "reinforcement_window_blocked", "reinforcement_block_same_session"], + details: { + blockReason: "same_session", + attemptedAtIso: "2026-05-12T07:48:21.361Z", + lastReinforcedAtIso: "2026-05-11T09:19:08.708Z", + }, + }), + ...Array.from({ length: 4 }, (_, index) => reinforcementAttempt(`evt-cross-day-ok-${index}`, packageJson.version, false)), + ]), {}, generatedAt); + const output = formatQualityReviewBoard(report, {}); + const diagnosticQuestions = report.facts.systemMechanisms.versionedFacts?.reinforcementRules.diagnosticQuestions; + + assert.equal(diagnosticQuestions?.length, 1); + assert.equal(diagnosticQuestions?.[0]?.question, "Should same_session reinforcement blocking apply across UTC days?"); + assert.deepEqual(diagnosticQuestions?.[0]?.evidence, [ + "count=2", + "eventId=evt-cross-day-same-session", + "attemptedAtIso=2026-05-13T07:48:21.361Z", + "lastReinforcedAtIso=2026-05-12T09:19:08.708Z", + ]); + assert.match(output, /Should same_session reinforcement blocking apply across UTC days/i); + assert.match(output, /2026-05-13/); + assert.match(output, /2026-05-12/); + assert.doesNotMatch(output, /definitely a bug/i); +}); + +test("versioned quality warns when current reinforcement sample is small", () => { + const events = [ + reinforcementAttempt("evt-prev-small-block", "1.6.0", true), + reinforcementAttempt("evt-current-small-1", packageJson.version, false), + reinforcementAttempt("evt-current-small-2", packageJson.version, false), + ]; + const report = buildQualityReviewBoard(inspectionModel([], events), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + + assert.equal(facts?.buckets.current.sampleAssessment, "not_observed_but_sample_small"); + assert.match(formatQualityReviewBoard(report, {}), /do not infer absence/); +}); + +test("versioned reinforcement diagnostic strength is weak when below threshold", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + ...Array.from({ length: 3 }, (_, index) => reinforcementAttempt(`evt-strength-weak-${index}`, packageJson.version, false)), + ]), {}, generatedAt); + + assert.match(formatQualityReviewBoard(report, {}), /diagnostic strength: weak/i); +}); + +test("versioned reinforcement diagnostic strength is unavailable when no current opportunities", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-strength-prev", "1.6.0", false), + ]), {}, generatedAt); + + assert.match(formatQualityReviewBoard(report, {}), /diagnostic strength: unavailable/i); +}); + +test("versioned reinforcement diagnostic strength is moderate with causal detail", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-strength-blocked", packageJson.version, true, { + details: { blockReason: "same_session" }, + reasonCodes: ["numbered_ref_reinforce", "reinforcement_window_blocked", "reinforcement_block_same_session"], + }), + ...Array.from({ length: 4 }, (_, index) => reinforcementAttempt(`evt-strength-ok-${index}`, packageJson.version, false)), + ]), {}, generatedAt); + + assert.match(formatQualityReviewBoard(report, {}), /diagnostic strength: moderate; causal detail available/i); +}); + +test("versioned quality groups eviction snapshot recurrence by producer version", () => { + const events = [ + capacityRemoval("evt-prev-missing", "1.6.0", false), + ...Array.from({ length: 5 }, (_, index) => capacityRemoval(`evt-current-snapshot-${index}`, packageJson.version, true)), + ]; + const report = buildQualityReviewBoard(inspectionModel([], events), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.evictionAndCaps; + + assert.equal(facts?.buckets.current.facts.capacitySnapshotsMissing, 0); + assert.equal(facts?.buckets.previous.facts.capacitySnapshotsMissing, 1); + assert.equal(facts?.buckets.current.sampleAssessment, "not_observed_with_sufficient_sample"); + assert.equal(report.answerability?.evictionAndCaps.level, "partial"); +}); + +test("versioned quality treats unknown producerVersion as unknown/unversioned", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-unknown-version", "unknown", true), + ]), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + + assert.equal(facts?.buckets.current.opportunityCount, 0); + assert.equal(facts?.buckets.unknown_unversioned.opportunityCount, 1); + assert.equal(report.answerability?.reinforcementRules.level, "inventory_only"); +}); + +test("versioned quality buckets rejection filter candidates and candidate version context", () => { + const report = buildQualityReviewBoard(inspectionModel([], [], [ + rejection("Architecture current rejection candidate", { type: "decision", reasons: ["bad_decision"], timestamp: "2026-05-10T00:00:00.000Z", producerVersion: packageJson.version }), + rejection("Architecture previous rejection candidate", { type: "decision", reasons: ["bad_decision"], timestamp: "2026-05-09T00:00:00.000Z", producerVersion: "1.6.0" }), + rejection("Architecture unknown rejection candidate", { type: "decision", reasons: ["bad_decision"], timestamp: "2026-05-08T00:00:00.000Z" }), + ]), { verbose: true }, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.rejectionFilters; + + assert.equal(facts?.buckets.current.opportunityCount, 1); + assert.equal(facts?.buckets.previous.opportunityCount, 1); + assert.equal(facts?.buckets.unknown_unversioned.opportunityCount, 1); + assert.equal(report.reviewCandidates.find(candidate => candidate.facts.timestamp === "2026-05-10T00:00:00.000Z")?.versionContext?.group, "current"); + assert.equal(report.reviewCandidates.find(candidate => candidate.facts.timestamp === "2026-05-09T00:00:00.000Z")?.versionContext?.group, "previous"); +}); + +test("versioned quality formatter avoids forbidden version-inference wording", () => { + const output = formatQualityReviewBoard(buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-wording-prev", "1.6.0", true), + ...Array.from({ length: 5 }, (_, index) => reinforcementAttempt(`evt-wording-current-${index}`, packageJson.version, false)), + ]), {}, generatedAt), {}).toLowerCase(); + + for (const forbidden of ["fixed", "resolved", "bug fixed", "regression fixed", "absent in current version"]) { + assert.equal(output.includes(forbidden), false, `output should not contain ${forbidden}`); + } +}); + +test("quality CLI JSON includes versionedFacts end to end", async () => { + const root = mkdtempSync(join(tmpdir(), "opencode-memory-diag-quality-versioned-json-")); + try { + await seedWorkspace(root, []); + await appendEvidenceEvents(root, [{ type: "memory_reinforced", phase: "reinforcement", outcome: "reinforced", reasonCodes: [] }]); + const report = JSON.parse(await runMemoryDiag(["quality", "--workspace", root, "--json"])) as ReviewBoardReport; + assert.ok(report.facts.systemMechanisms.versionedFacts); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("versioned quality cannot assess when no current-version events exist", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-no-current-prev", "1.6.0", true), + reinforcementAttempt("evt-no-current-unknown", undefined, true), + ]), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + + assert.equal(facts?.buckets.current.sampleAssessment, "no_current_version_opportunities"); + assert.equal(facts?.inference.status, "no_current_version_opportunities"); + assert.match(formatQualityReviewBoard(report, {}), /cannot assess recurrence/); +}); + +test("versioned quality reports pattern persisting across all version buckets", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-all-current-block", packageJson.version, true), + ...Array.from({ length: 5 }, (_, index) => reinforcementAttempt(`evt-all-current-${index}`, packageJson.version, false)), + reinforcementAttempt("evt-all-prev-block", "1.6.0", true), + reinforcementAttempt("evt-all-unknown-block", undefined, true), + ]), {}, generatedAt); + + assert.equal(report.facts.systemMechanisms.versionedFacts?.reinforcementRules.inference.status, "pattern_persists_across_versions"); + assert.match(formatQualityReviewBoard(report, {}).toLowerCase(), /current recurrence detected.*persists across versions/s); +}); + +test("versioned quality labels current-only pattern as new, not recurrence", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-new-current-block", packageJson.version, true), + ...Array.from({ length: 5 }, (_, index) => reinforcementAttempt(`evt-new-current-${index}`, packageJson.version, false)), + ]), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + + assert.equal(facts?.inference.status, "no_previous_pattern_observed"); + assert.match(facts?.inference.message ?? "", /new pattern, not a recurrence/); +}); + +test("versioned quality emits current_recurrence_detected when current has signal and unknown/unversioned has signal but previous has none", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-current-unknown-current-block", packageJson.version, true), + ...Array.from({ length: 4 }, (_, index) => reinforcementAttempt(`evt-current-unknown-current-ok-${index}`, packageJson.version, false)), + reinforcementAttempt("evt-current-unknown-unknown-block", "unknown", true), + ]), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + + assert.equal(facts?.buckets.current.opportunityCount, 5); + assert.equal(facts?.buckets.current.observedPatternCount, 1); + assert.equal(facts?.buckets.previous.observedPatternCount, 0); + assert.equal(facts?.buckets.unknown_unversioned.observedPatternCount, 1); + assert.equal(facts?.inference.status, "current_recurrence_detected"); + assert.match(facts?.inference.message ?? "", /unknown\/unversioned evidence/); + assert.match(facts?.inference.message ?? "", /cannot confirm or deny/); +}); + +test("versioned quality still emits no_previous_pattern_observed when both previous and unknown/unversioned have no signal", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-current-only-current-block", packageJson.version, true), + ...Array.from({ length: 4 }, (_, index) => reinforcementAttempt(`evt-current-only-current-ok-${index}`, packageJson.version, false)), + reinforcementAttempt("evt-current-only-unknown-ok", "unknown", false), + ]), {}, generatedAt); + const facts = report.facts.systemMechanisms.versionedFacts?.reinforcementRules; + + assert.equal(facts?.buckets.current.opportunityCount, 5); + assert.equal(facts?.buckets.current.observedPatternCount, 1); + assert.equal(facts?.buckets.previous.observedPatternCount, 0); + assert.equal(facts?.buckets.unknown_unversioned.observedPatternCount, 0); + assert.equal(facts?.inference.status, "no_previous_pattern_observed"); + assert.doesNotMatch(facts?.inference.message ?? "", /Current recurrence detected/); +}); + +test("versioned quality treats empty and whitespace producerVersion as unknown/unversioned", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-empty-version", "", true), + reinforcementAttempt("evt-space-version", " ", true), + ]), {}, generatedAt); + + assert.equal(report.facts.systemMechanisms.versionedFacts?.reinforcementRules.buckets.current.opportunityCount, 0); + assert.equal(report.facts.systemMechanisms.versionedFacts?.reinforcementRules.buckets.previous.opportunityCount, 0); + assert.equal(report.facts.systemMechanisms.versionedFacts?.reinforcementRules.buckets.unknown_unversioned.opportunityCount, 2); +}); + +test("versioned quality leaves current bucket empty when current package version is unknown", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-current-unknown-override", packageJson.version, true), + ]), { currentProducerVersion: "unknown" }, generatedAt); + + assert.equal(report.facts.systemMechanisms.versionedFacts?.reinforcementRules.buckets.current.opportunityCount, 0); + assert.equal(report.facts.systemMechanisms.versionedFacts?.reinforcementRules.inference.status, "no_current_version_opportunities"); + assert.match(report.facts.systemMechanisms.versionedFacts?.reinforcementRules.inference.message ?? "", /unknown/); +}); + +test("quality answerability stays conservative when only previous version has reinforcement signal", () => { + const report = buildQualityReviewBoard(inspectionModel([], [ + reinforcementAttempt("evt-prev-only-block", "1.6.0", true), + ]), {}, generatedAt); + + assert.equal(report.answerability?.reinforcementRules.level, "inventory_only"); +}); + +test("versioned quality exposes unknown/unversioned composition breakdown", () => { + const unknownEvents = [ + ...Array.from({ length: 4 }, (_, index) => reinforcementAttempt(`evt-no-fields-${index}`, undefined, true)), + ...Array.from({ length: 3 }, (_, index) => reinforcementAttempt(`evt-unknown-fields-${index}`, "unknown", true)), + reinforcementAttempt("evt-empty-fields-1", "", true), + reinforcementAttempt("evt-empty-fields-2", "", true), + reinforcementAttempt("evt-space-fields", " ", true), + reinforcementAttempt("evt-known-current", packageJson.version, false), + reinforcementAttempt("evt-known-prev", "1.6.0", false), + ]; + const report = buildQualityReviewBoard(inspectionModel([], unknownEvents), {}, generatedAt); + const buckets = report.facts.systemMechanisms.versionedFacts?.reinforcementRules.buckets; + + assert.deepEqual(buckets?.unknown_unversioned.versionAvailability, { noProducerFields: 4, unknownProducerVersion: 3, emptyProducerVersion: 3, knownProducerVersion: 0 }); + assert.equal(buckets?.current.versionAvailability.knownProducerVersion, 1); + assert.equal(buckets?.previous.versionAvailability.knownProducerVersion, 1); + assert.equal(buckets?.current.versionAvailability.emptyProducerVersion, 0); + assert.equal(buckets?.previous.versionAvailability.noProducerFields, 0); +}); + +test("versioned quality marks low version coverage as transitional", () => { + const events = Array.from({ length: 2_292 }, (_, index) => versionedEvent(`evt-transitional-${index}`, "unknown", { + type: "render_selected", + phase: "render", + outcome: "rendered", + memory: { memoryId: `mem-${index}`, type: "decision", source: "compaction" }, + })); + const report = buildQualityReviewBoard(inspectionModel([], events), {}, generatedAt); + + assert.equal(report.facts.systemMechanisms.versionedFacts?.versionCoverage.isTransitional, true); + assert.equal(report.facts.systemMechanisms.versionedFacts?.versionCoverage.coveragePercent, 0); + const output = formatQualityReviewBoard(report, {}); + assert.match(output, /Coverage: 0% of 2,292/); + assert.match(output, /Version coverage is below 50%/); +}); + +test("versioned quality marks fully stamped evidence as non-transitional", () => { + const events = [ + ...Array.from({ length: 15 }, (_, index) => reinforcementAttempt(`evt-nontrans-current-${index}`, packageJson.version, false)), + ...Array.from({ length: 5 }, (_, index) => reinforcementAttempt(`evt-nontrans-prev-${index}`, "1.6.0", false)), + ]; + const report = buildQualityReviewBoard(inspectionModel([], events), {}, generatedAt); + + assert.equal(report.facts.systemMechanisms.versionedFacts?.versionCoverage.isTransitional, false); + assert.equal(report.facts.systemMechanisms.versionedFacts?.versionCoverage.coveragePercent, 100); + assert.doesNotMatch(formatQualityReviewBoard(report, {}), /Version coverage is below 50%/); +}); + function entry(id: string, text: string, type: LongTermMemoryEntry["type"], overrides: Partial = {}): LongTermMemoryEntry { return { id, @@ -495,6 +1015,55 @@ function event( }; } +function versionedEvent( + eventId: string, + producerVersion: string | undefined, + overrides: Partial & { type: EvidenceEventType; phase: EvidencePhase; outcome: EvidenceOutcome }, +): EvidenceEventV1 { + const base = event(eventId, overrides); + if (producerVersion === undefined) return base; + return { + ...base, + producerName: "opencode-working-memory", + producerVersion, + instrumentationVersion: 2, + }; +} + +function reinforcementAttempt( + id: string, + producerVersion: string | undefined, + blocked = false, + overrides: Partial = {}, +): EvidenceEventV1 { + const base = versionedEvent(id, producerVersion, { + type: "memory_reinforced", + phase: "reinforcement", + outcome: blocked ? "rejected" : "reinforced", + memory: { memoryId: `mem-${id}`, type: "decision", source: "compaction" }, + reasonCodes: blocked ? ["reinforcement_window_blocked"] : [], + details: blocked ? { blockReason: "same_utc_day", ref: id } : undefined, + }); + return { + ...base, + ...overrides, + details: overrides.details ?? base.details, + reasonCodes: overrides.reasonCodes ?? base.reasonCodes, + }; +} + +function capacityRemoval(id: string, producerVersion: string | undefined, withSnapshot: boolean): EvidenceEventV1 { + return versionedEvent(id, producerVersion, { + type: "memory_removed_capacity", + phase: "storage", + outcome: "removed", + createdAt: "2026-05-11T10:00:00.000Z", + memory: { memoryId: `mem-${id}`, type: "decision", source: "compaction" }, + reasonCodes: ["global_cap"], + details: withSnapshot ? { strengthAtRemoval: 0.8, rankAtRemoval: 2 } : undefined, + }); +} + function capacityEvents(count: number): EvidenceEventV1[] { return Array.from({ length: count }, (_, index) => event(`evt-cap-${index.toString().padStart(2, "0")}`, { type: "memory_removed_capacity", @@ -516,7 +1085,72 @@ function capacityEventInputs(count: number): EvidenceEventInput[] { })); } -function rejection(text: string, options: { type: NormalizedRejection["type"]; reasons: string[]; timestamp: string; legacy?: boolean }): NormalizedRejection { +function inspectionModelWithProducerEvents(): MemoryInspectionReadModel { + return inspectionModel([], [ + event("evt-new-block-same-day", { + type: "memory_reinforced", + phase: "reinforcement", + outcome: "rejected", + createdAt: "2026-05-11T09:00:00.000Z", + memory: { memoryId: "mem-producer-block", type: "decision", source: "compaction" }, + reasonCodes: ["reinforcement_window_blocked", "reinforcement_block_same_utc_day"], + details: { blockReason: "same_utc_day", ref: "1" }, + producerName: "opencode-working-memory", + producerVersion: packageJson.version, + instrumentationVersion: 2, + }), + event("evt-new-block-same-session", { + type: "memory_reinforced", + phase: "reinforcement", + outcome: "rejected", + createdAt: "2026-05-11T10:00:00.000Z", + memory: { memoryId: "mem-producer-block", type: "decision", source: "compaction" }, + reasonCodes: ["reinforcement_window_blocked", "reinforcement_block_same_session"], + details: { blockReason: "same_session", ref: "2" }, + producerName: "opencode-working-memory", + producerVersion: packageJson.version, + instrumentationVersion: 2, + }), + event("evt-new-block-missing-detail", { + type: "memory_reinforced", + phase: "reinforcement", + outcome: "rejected", + createdAt: "2026-05-11T11:00:00.000Z", + memory: { memoryId: "mem-producer-block", type: "decision", source: "compaction" }, + reasonCodes: ["reinforcement_window_blocked"], + producerName: "opencode-working-memory", + producerVersion: packageJson.version, + instrumentationVersion: 2, + }), + ]); +} + +function inspectionModelWithCapacitySnapshots(): MemoryInspectionReadModel { + return inspectionModel([], [ + event("evt-cap-snapshot", { + type: "memory_removed_capacity", + phase: "storage", + outcome: "removed", + createdAt: "2026-05-11T10:00:00.000Z", + memory: { memoryId: "mem-cap-snapshot", type: "decision", source: "compaction" }, + reasonCodes: ["global_cap"], + details: { strengthAtRemoval: 0.82, rankAtRemoval: 2, typeRankAtRemoval: 1, ageDaysAtRemoval: 3 }, + producerName: "opencode-working-memory", + producerVersion: packageJson.version, + instrumentationVersion: 2, + }), + event("evt-cap-historical", { + type: "memory_removed_capacity", + phase: "storage", + outcome: "removed", + createdAt: "2026-05-11T09:00:00.000Z", + memory: { memoryId: "mem-cap-historical", type: "feedback", source: "compaction" }, + reasonCodes: ["type_cap"], + }), + ]); +} + +function rejection(text: string, options: { type: NormalizedRejection["type"]; reasons: string[]; timestamp: string; legacy?: boolean; producerVersion?: string; producerName?: string; instrumentationVersion?: number }): NormalizedRejection { return { timestamp: options.timestamp, workspaceKey: options.legacy ? undefined : "workspace-key", @@ -528,6 +1162,13 @@ function rejection(text: string, options: { type: NormalizedRejection["type"]; r fromTrigger: false, text, reasons: options.reasons, + ...(options.producerName !== undefined || options.producerVersion !== undefined || options.instrumentationVersion !== undefined + ? { + producerName: options.producerName ?? "opencode-working-memory", + producerVersion: options.producerVersion, + instrumentationVersion: options.instrumentationVersion ?? 2, + } + : {}), }; } @@ -609,7 +1250,7 @@ function assertReviewBoardShape(report: ReviewBoardReport): void { assert.equal(report.languageGuidance.nonAuthoritative, true); assert.equal(report.languageGuidance.mutation, "none"); assert.equal(report.languageGuidance.rawReasonCodesAreEvidence, true); - assert.equal(report.languageGuidance.producerVersionRecorded, false); + assert.equal(typeof report.languageGuidance.producerVersionRecorded, "boolean"); assert.equal(report.languageGuidance.provenanceInferenceOnly, true); assert.equal(report.languageGuidance.primaryReviewPurpose, "system_mechanism_observations"); assert.equal(report.languageGuidance.secondaryReviewPurpose, "memory_content_quality"); @@ -619,7 +1260,11 @@ function assertReviewBoardShape(report: ReviewBoardReport): void { assert.equal(typeof report.facts.systemMechanisms.reinforcementRules.windowBlockRate, "number"); assert.ok(Array.isArray(report.facts.systemMechanisms.evictionAndCaps.fullCaps)); assert.equal(typeof report.facts.systemMechanisms.identityAndDedup.duplicateTextOrIdentityGroups, "number"); + assert.ok(report.facts.systemMechanisms.versionedFacts); + assert.equal(typeof report.facts.systemMechanisms.versionedFacts.currentPackageVersion, "string"); assert.equal(typeof report.facts.memoryContent.evidenceCoverage.covered, "number"); + assert.equal(report.answerability?.reinforcementRules.level, "inventory_only"); + assert.equal(report.answerability?.evictionAndCaps.level, "inventory_only"); assert.ok(Array.isArray(report.activeMemoryDisplay.items)); assert.ok(Array.isArray(report.reviewCandidates)); assert.ok(Array.isArray(report.reviewQuestions.systemMechanism)); diff --git a/tests/retention.test.ts b/tests/retention.test.ts new file mode 100644 index 0000000..915aded --- /dev/null +++ b/tests/retention.test.ts @@ -0,0 +1,103 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + REINFORCEMENT_MAX_COUNT, + REINFORCEMENT_MIN_INTERVAL_MS, + tryReinforceMemory, +} from "../src/retention.ts"; +import type { LongTermMemoryEntry } from "../src/types.ts"; + +const baseMemory = (overrides: Partial = {}): LongTermMemoryEntry => ({ + id: "mem-retention", + type: "decision", + text: "Durable decision for reinforcement", + source: "compaction", + confidence: 0.75, + status: "active", + createdAt: "2026-05-10T00:00:00.000Z", + updatedAt: "2026-05-10T00:00:00.000Z", + ...overrides, +}); + +test("tryReinforceMemory blocks same session with exact reason", () => { + const now = Date.UTC(2026, 4, 12, 12, 0, 0); + const memory = baseMemory({ + reinforcementCount: 1, + lastReinforcedAt: now - 2 * REINFORCEMENT_MIN_INTERVAL_MS, + lastReinforcedSessionID: "session-a", + }); + + const decision = tryReinforceMemory(memory, "session-a", now); + + assert.equal(decision.outcome, "blocked"); + assert.equal(decision.blockReason, "same_session"); + assert.equal(decision.memory, memory); +}); + +test("tryReinforceMemory blocks different session on same UTC day", () => { + const lastAt = Date.UTC(2026, 4, 12, 0, 15, 0); + const now = Date.UTC(2026, 4, 12, 23, 30, 0); + const memory = baseMemory({ + reinforcementCount: 1, + lastReinforcedAt: lastAt, + lastReinforcedSessionID: "session-a", + }); + + const decision = tryReinforceMemory(memory, "session-b", now); + + assert.equal(decision.outcome, "blocked"); + assert.equal(decision.blockReason, "same_utc_day"); + assert.equal(decision.lastReinforcedAt, lastAt); +}); + +test("tryReinforceMemory blocks min interval across UTC day boundary", () => { + const lastAt = Date.UTC(2026, 4, 12, 23, 45, 0); + const now = Date.UTC(2026, 4, 13, 0, 15, 0); + const memory = baseMemory({ + reinforcementCount: 1, + lastReinforcedAt: lastAt, + lastReinforcedSessionID: "session-a", + }); + + const decision = tryReinforceMemory(memory, "session-b", now); + + assert.equal(decision.outcome, "blocked"); + assert.equal(decision.blockReason, "min_interval"); + assert.equal(decision.minIntervalMs, REINFORCEMENT_MIN_INTERVAL_MS); +}); + +test("tryReinforceMemory blocks max count with exact reason", () => { + const now = Date.UTC(2026, 4, 12, 12, 0, 0); + const memory = baseMemory({ + reinforcementCount: REINFORCEMENT_MAX_COUNT, + lastReinforcedAt: Date.UTC(2026, 4, 10, 12, 0, 0), + lastReinforcedSessionID: "session-a", + }); + + const decision = tryReinforceMemory(memory, "session-b", now); + + assert.equal(decision.outcome, "blocked"); + assert.equal(decision.blockReason, "max_count"); + assert.equal(decision.reinforcementCount, REINFORCEMENT_MAX_COUNT); + assert.equal(decision.maxReinforcementCount, REINFORCEMENT_MAX_COUNT); +}); + +test("tryReinforceMemory reinforces allowed memory and wrapper returns memory only", () => { + const now = Date.UTC(2026, 4, 12, 12, 0, 0); + const memory = baseMemory({ + reinforcementCount: 1, + lastReinforcedAt: Date.UTC(2026, 4, 10, 12, 0, 0), + lastReinforcedSessionID: "session-a", + }); + + const decision = tryReinforceMemory(memory, "session-b", now); + + assert.equal(decision.outcome, "reinforced"); + assert.equal(decision.previousReinforcementCount, 1); + assert.equal(decision.newReinforcementCount, 2); + assert.notEqual(decision.memory, memory); + assert.equal(decision.memory.reinforcementCount, 2); + assert.equal(decision.memory.lastReinforcedAt, now); + assert.equal(decision.memory.lastReinforcedSessionID, "session-b"); + assert.equal(decision.memory.retentionClock, now); +}); diff --git a/tests/workspace-memory.test.ts b/tests/workspace-memory.test.ts index 905a35c..6801b3e 100644 --- a/tests/workspace-memory.test.ts +++ b/tests/workspace-memory.test.ts @@ -30,7 +30,7 @@ import { calculateRetentionStrength, calculateDormantDays, calculateEffectiveAgeDays, - reinforceMemory, + tryReinforceMemory, } from "../src/retention.ts"; import { redactCredentials } from "../src/redaction.ts"; import { assessMemoryQuality, isHardQualityReason, isProgressSnapshotViolation } from "../src/memory-quality.ts"; @@ -583,7 +583,7 @@ test("normalizeWorkspaceMemoryWithAccounting uses dormant workspace days for str test("reinforceMemory enforces session interval and max guards", () => { const now = Date.UTC(2026, 3, 29); const base = entry("reinforce", "Durable memory should reinforce only when gated"); - const reinforced = reinforceMemory(base, "session-a", now); + const reinforced = tryReinforceMemory(base, "session-a", now).memory; assert.notEqual(reinforced, base); assert.equal(reinforced.reinforcementCount, 1); @@ -591,15 +591,15 @@ test("reinforceMemory enforces session interval and max guards", () => { assert.equal(reinforced.lastReinforcedSessionID, "session-a"); assert.equal(reinforced.retentionClock, now); - assert.equal(reinforceMemory(reinforced, "session-a", now + 2 * 60 * 60 * 1000), reinforced); - assert.equal(reinforceMemory(reinforced, "session-b", now + 30 * 60 * 1000), reinforced); + assert.equal(tryReinforceMemory(reinforced, "session-a", now + 2 * 60 * 60 * 1000).memory, reinforced); + assert.equal(tryReinforceMemory(reinforced, "session-b", now + 30 * 60 * 1000).memory, reinforced); const atMax: LongTermMemoryEntry = { ...base, reinforcementCount: 6, lastReinforcedAt: now - 2 * 60 * 60 * 1000, }; - assert.equal(reinforceMemory(atMax, "session-c", now), atMax); + assert.equal(tryReinforceMemory(atMax, "session-c", now).memory, atMax); }); test("reinforceMemory requires distinct UTC calendar days between reinforcements", () => { @@ -613,9 +613,9 @@ test("reinforceMemory requires distinct UTC calendar days between reinforcements lastReinforcedSessionID: "session-a", }; - assert.equal(reinforceMemory(base, "session-b", sameUtcDayMuchLater), base); + assert.equal(tryReinforceMemory(base, "session-b", sameUtcDayMuchLater).memory, base); - const reinforcedNextDay = reinforceMemory(base, "session-b", nextUtcDayAfterInterval); + const reinforcedNextDay = tryReinforceMemory(base, "session-b", nextUtcDayAfterInterval).memory; assert.notEqual(reinforcedNextDay, base); assert.equal(reinforcedNextDay.reinforcementCount, 2); assert.equal(reinforcedNextDay.lastReinforcedAt, nextUtcDayAfterInterval); @@ -626,7 +626,7 @@ test("reinforceMemory requires distinct UTC calendar days between reinforcements ...base, reinforcementCount: 6, }; - assert.equal(reinforceMemory(atMax, "session-c", nextUtcDayAfterInterval), atMax); + assert.equal(tryReinforceMemory(atMax, "session-c", nextUtcDayAfterInterval).memory, atMax); }); test("dedupeLongTermEntriesWithAccounting reinforces absorbed exact duplicates", () => { @@ -737,7 +737,34 @@ test("dedupe reinforcement does not increment for same session", () => { assert.ok(retained, "existing manual memory should be retained"); assert.equal(retained.reinforcementCount, 1); assert.equal(retained.lastReinforcedSessionID, "same-session"); - assert.equal(result.evidence.some(event => event.type === "memory_reinforced"), false); + assert.equal(result.evidence.some(event => event.type === "memory_reinforced" && event.outcome === "reinforced"), false); + assert.ok(result.evidence.some(event => event.type === "memory_reinforced" && event.outcome === "rejected" && event.details?.blockReason === "same_session")); +}); + +test("dedupe blocked reinforcement emits exact block reason details", () => { + const now = Date.now(); + const existing: LongTermMemoryEntry = { + ...entry("existing-blocked", "Prefer deterministic consolidation accounting", "feedback"), + source: "manual", + reinforcementCount: 1, + lastReinforcedAt: now - 30 * 60 * 1000, + lastReinforcedSessionID: "old-session", + }; + const duplicate: LongTermMemoryEntry = { + ...entry("duplicate-blocked", "prefer deterministic consolidation accounting!!!", "feedback"), + pendingOwnerSessionID: "new-session", + }; + + const result = dedupeLongTermEntriesWithAccounting([existing, duplicate]); + const blocked = result.evidence.find(event => event.type === "memory_reinforced" && event.outcome === "rejected"); + + assert.ok(blocked, "blocked duplicate reinforcement should emit diagnostic evidence"); + assert.ok(blocked.reasonCodes.includes("reinforcement_window_blocked")); + assert.ok(blocked.reasonCodes.includes("reinforcement_block_min_interval")); + assert.equal(blocked.details?.blockReason, "min_interval"); + assert.equal(blocked.details?.reinforcementCount, 1); + assert.equal(blocked.details?.maxReinforcementCount, 6); + assert.equal(blocked.details?.minIntervalMs, 60 * 60 * 1000); }); test("dedupe reinforcement does not increment under one hour", () => { @@ -760,7 +787,8 @@ test("dedupe reinforcement does not increment under one hour", () => { assert.ok(retained, "existing manual memory should be retained"); assert.equal(retained.reinforcementCount, 1); assert.equal(retained.lastReinforcedSessionID, "old-session"); - assert.equal(result.evidence.some(event => event.type === "memory_reinforced"), false); + assert.equal(result.evidence.some(event => event.type === "memory_reinforced" && event.outcome === "reinforced"), false); + assert.ok(result.evidence.some(event => event.type === "memory_reinforced" && event.outcome === "rejected" && event.details?.blockReason === "min_interval")); }); test("dedupe reinforcement does not emit evidence at max reinforcement count", () => { @@ -776,7 +804,8 @@ test("dedupe reinforcement does not emit evidence at max reinforcement count", ( const result = dedupeLongTermEntriesWithAccounting([existing, duplicate]); - assert.equal(result.evidence.some(event => event.type === "memory_reinforced"), false); + assert.equal(result.evidence.some(event => event.type === "memory_reinforced" && event.outcome === "reinforced"), false); + assert.ok(result.evidence.some(event => event.type === "memory_reinforced" && event.outcome === "rejected" && event.details?.blockReason === "max_count")); }); test("enforceLongTermLimits orders entries by retention strength", () => { @@ -2644,6 +2673,10 @@ test("enforceLongTermLimitsWithAccounting keeps 11th and 12th decisions and type assert.ok(event.relations?.[0]?.memory.memoryKeyHash, "removed relation should include memory key hash"); assert.ok(event.relations?.[0]?.memory.identityKeyHash, "removed relation should include identity key hash"); assert.deepEqual(event.reasonCodes, ["type_cap"]); + assert.equal(typeof event.details?.strengthAtRemoval, "number"); + assert.equal(typeof event.details?.rankAtRemoval, "number"); + assert.equal(typeof event.details?.typeRankAtRemoval, "number"); + assert.equal(typeof event.details?.ageDaysAtRemoval, "number"); } });