From fc7b6103f351f429f5089510c6e57b796f461ff0 Mon Sep 17 00:00:00 2001 From: Rodrigo Uroz Date: Thu, 12 Mar 2026 16:41:06 +0000 Subject: [PATCH] Compaction: add post-index sync config and coverage --- CHANGELOG.md | 2 + .../pi-embedded-runner/compact.hooks.test.ts | 84 +++++++++++++++++ src/agents/pi-embedded-runner/compact.ts | 3 + src/config/schema.help.ts | 2 + src/config/schema.labels.ts | 1 + src/config/types.agent-defaults.ts | 3 + src/config/zod-schema.agent-defaults.ts | 1 + src/memory/index.test.ts | 91 +++++++++++++++++++ src/memory/manager-sync-ops.ts | 59 ++++++++++-- src/memory/manager.ts | 2 + src/memory/qmd-manager.ts | 1 + src/memory/search-manager.ts | 1 + src/memory/types.ts | 1 + 13 files changed, 244 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8805acc7ec8..c77f597b15b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Docs: https://docs.openclaw.ai - Security/plugins: disable implicit workspace plugin auto-load so cloned repositories cannot execute workspace plugin code without an explicit trust decision. (`GHSA-99qw-6mr3-36qr`)(#44174) Thanks @lintsinghua and @vincentkoc. - Moonshot CN API: respect explicit `baseUrl` (api.moonshot.cn) in implicit provider resolution so platform.moonshot.cn API keys authenticate correctly instead of returning HTTP 401. (#33637) Thanks @chengzhichao-xydt. - Kimi Coding/provider config: respect explicit `models.providers["kimi-coding"].baseUrl` when resolving the implicit provider so custom Kimi Coding endpoints no longer get overwritten by the built-in default. (#36353) Thanks @2233admin. +- Memory/session sync: add mode-aware post-compaction session reindexing with `agents.defaults.compaction.postIndexSync` plus `agents.defaults.memorySearch.sync.sessions.postCompactionForce`, so compacted session memory can refresh immediately without forcing every deployment into synchronous reindexing. (#25561) Thanks @rodrigouroz. - Models/Kimi Coding: send `anthropic-messages` tools in native Anthropic format again so `kimi-coding` stops degrading tool calls into XML/plain-text pseudo invocations instead of real `tool_use` blocks. (#38669, #39907, #40552) Thanks @opriz. - TUI/chat log: reuse the active assistant message component for the same streaming run so `openclaw tui` no longer renders duplicate assistant replies. (#35364) Thanks @lisitan. - Telegram/model picker: make inline model button selections persist the chosen session model correctly, clear overrides when selecting the configured default, and include effective fallback models in `/models` button validation. (#40105) Thanks @avirweb. @@ -55,6 +56,7 @@ Docs: https://docs.openclaw.ai - Gateway/session stores: regenerate the Swift push-test protocol models and align Windows native session-store realpath handling so protocol checks and sync session discovery stop drifting on Windows. (#44266) thanks @jalehman. - Context engine/session routing: forward optional `sessionKey` through context-engine lifecycle calls so plugins can see structured routing metadata during bootstrap, assembly, post-turn ingestion, and compaction. (#44157) thanks @jalehman. - Agents/failover: classify z.ai `network_error` stop reasons as retryable timeouts so provider connectivity failures trigger fallback instead of surfacing raw unhandled-stop-reason errors. (#43884) Thanks @hougangdev. +- Memory/session sync: add mode-aware post-compaction session reindexing with `agents.defaults.compaction.postIndexSync` plus `agents.defaults.memorySearch.sync.sessions.postCompactionForce`, so compacted session memory can refresh immediately without forcing every deployment into synchronous reindexing. (#25561) thanks @rodrigouroz. ## 2026.3.11 diff --git a/src/agents/pi-embedded-runner/compact.hooks.test.ts b/src/agents/pi-embedded-runner/compact.hooks.test.ts index 9292028b5d4..a166dac275c 100644 --- a/src/agents/pi-embedded-runner/compact.hooks.test.ts +++ b/src/agents/pi-embedded-runner/compact.hooks.test.ts @@ -9,6 +9,9 @@ const { triggerInternalHook, sanitizeSessionHistoryMock, contextEngineCompactMock, + getMemorySearchManagerMock, + resolveMemorySearchConfigMock, + resolveSessionAgentIdMock, } = vi.hoisted(() => ({ hookRunner: { hasHooks: vi.fn(), @@ -38,6 +41,20 @@ const { | { summary: string; tokensAfter: number } | undefined, })), + getMemorySearchManagerMock: vi.fn(async () => ({ + manager: { + sync: vi.fn(async () => {}), + }, + })), + resolveMemorySearchConfigMock: vi.fn(() => ({ + sources: ["sessions"], + sync: { + sessions: { + postCompactionForce: true, + }, + }, + })), + resolveSessionAgentIdMock: vi.fn(() => "main"), })); vi.mock("../../plugins/hook-runner-global.js", () => ({ @@ -211,9 +228,18 @@ vi.mock("../agent-paths.js", () => ({ })); vi.mock("../agent-scope.js", () => ({ + resolveSessionAgentId: resolveSessionAgentIdMock, resolveSessionAgentIds: vi.fn(() => ({ defaultAgentId: "main", sessionAgentId: "main" })), })); +vi.mock("../memory-search.js", () => ({ + resolveMemorySearchConfig: resolveMemorySearchConfigMock, +})); + +vi.mock("../../memory/index.js", () => ({ + getMemorySearchManager: getMemorySearchManagerMock, +})); + vi.mock("../date-time.js", () => ({ formatUserTime: vi.fn(() => ""), resolveUserTimeFormat: vi.fn(() => ""), @@ -314,6 +340,23 @@ describe("compactEmbeddedPiSessionDirect hooks", () => { sanitizeSessionHistoryMock.mockImplementation(async (params: { messages: unknown[] }) => { return params.messages; }); + getMemorySearchManagerMock.mockReset(); + getMemorySearchManagerMock.mockResolvedValue({ + manager: { + sync: vi.fn(async () => {}), + }, + }); + resolveMemorySearchConfigMock.mockReset(); + resolveMemorySearchConfigMock.mockReturnValue({ + sources: ["sessions"], + sync: { + sessions: { + postCompactionForce: true, + }, + }, + }); + resolveSessionAgentIdMock.mockReset(); + resolveSessionAgentIdMock.mockReturnValue("main"); unregisterApiProviders(getCustomApiRegistrySourceId("ollama")); }); @@ -452,6 +495,47 @@ describe("compactEmbeddedPiSessionDirect hooks", () => { } }); + it("awaits post-compaction memory sync with the resolved force flag", async () => { + const sync = vi.fn(async () => {}); + getMemorySearchManagerMock.mockResolvedValue({ manager: { sync } }); + resolveMemorySearchConfigMock.mockReturnValue({ + sources: ["sessions"], + sync: { + sessions: { + postCompactionForce: false, + }, + }, + }); + + const result = await compactEmbeddedPiSessionDirect({ + sessionId: "session-1", + sessionKey: "agent:main:session-1", + sessionFile: "/tmp/session.jsonl", + workspaceDir: "/tmp", + customInstructions: "focus on decisions", + config: { + agents: { + defaults: { + compaction: { + postIndexSync: "await", + }, + }, + }, + } as never, + }); + + expect(result.ok).toBe(true); + expect(resolveSessionAgentIdMock).toHaveBeenCalledWith({ + sessionKey: "agent:main:session-1", + config: expect.any(Object), + }); + expect(sync).toHaveBeenCalledWith({ + reason: "post-compaction", + force: false, + sessionFiles: ["/tmp/session.jsonl"], + }); + }); + it("registers the Ollama api provider before compaction", async () => { resolveModelMock.mockReturnValue({ model: { diff --git a/src/agents/pi-embedded-runner/compact.ts b/src/agents/pi-embedded-runner/compact.ts index 4d299c89085..7b813cba7a8 100644 --- a/src/agents/pi-embedded-runner/compact.ts +++ b/src/agents/pi-embedded-runner/compact.ts @@ -281,6 +281,7 @@ function resolvePostCompactionIndexSyncMode(config?: OpenClawConfig): "off" | "a async function syncPostCompactionSessionMemory(params: { config?: OpenClawConfig; sessionKey?: string; + sessionFile: string; mode: "off" | "async" | "await"; }): Promise { if (params.mode === "off" || !params.config) { @@ -305,6 +306,7 @@ async function syncPostCompactionSessionMemory(params: { const syncTask = manager.sync({ reason: "post-compaction", force: resolvedMemory.sync.sessions.postCompactionForce, + sessionFiles: [params.sessionFile], }); if (params.mode === "await") { await syncTask; @@ -863,6 +865,7 @@ export async function compactEmbeddedPiSessionDirect( await syncPostCompactionSessionMemory({ config: params.config, sessionKey: params.sessionKey, + sessionFile: params.sessionFile, mode: resolvePostCompactionIndexSyncMode(params.config), }); // Estimate tokens after compaction by summing token estimates for remaining messages diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 5b5cb1fcc5c..9c45125754c 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -1035,6 +1035,8 @@ export const FIELD_HELP: Record = { "Enables summary quality audits and regeneration retries for safeguard compaction. Default: false, so safeguard mode alone does not turn on retry behavior.", "agents.defaults.compaction.qualityGuard.maxRetries": "Maximum number of regeneration retries after a failed safeguard summary quality audit. Use small values to bound extra latency and token cost.", + "agents.defaults.compaction.postIndexSync": + 'Controls post-compaction session memory reindex mode: "off", "async", or "await" (default: "async"). Use "await" for strongest freshness, "async" for lower compaction latency, and "off" only when session-memory sync is handled elsewhere.', "agents.defaults.compaction.postCompactionSections": 'AGENTS.md H2/H3 section names re-injected after compaction so the agent reruns critical startup guidance. Leave unset to use "Session Startup"/"Red Lines" with legacy fallback to "Every Session"/"Safety"; set to [] to disable reinjection entirely.', "agents.defaults.compaction.model": diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts index 4cf36d40a63..6aa2ae40efd 100644 --- a/src/config/schema.labels.ts +++ b/src/config/schema.labels.ts @@ -470,6 +470,7 @@ export const FIELD_LABELS: Record = { "agents.defaults.compaction.qualityGuard": "Compaction Quality Guard", "agents.defaults.compaction.qualityGuard.enabled": "Compaction Quality Guard Enabled", "agents.defaults.compaction.qualityGuard.maxRetries": "Compaction Quality Guard Max Retries", + "agents.defaults.compaction.postIndexSync": "Compaction Post-Index Sync", "agents.defaults.compaction.postCompactionSections": "Post-Compaction Context Sections", "agents.defaults.compaction.model": "Compaction Model Override", "agents.defaults.compaction.memoryFlush": "Compaction Memory Flush", diff --git a/src/config/types.agent-defaults.ts b/src/config/types.agent-defaults.ts index 5abaab2c169..11d1809c86a 100644 --- a/src/config/types.agent-defaults.ts +++ b/src/config/types.agent-defaults.ts @@ -287,6 +287,7 @@ export type AgentDefaultsConfig = { }; export type AgentCompactionMode = "default" | "safeguard"; +export type AgentCompactionPostIndexSyncMode = "off" | "async" | "await"; export type AgentCompactionIdentifierPolicy = "strict" | "off" | "custom"; export type AgentCompactionQualityGuardConfig = { /** Enable compaction summary quality audits and regeneration retries. Default: false. */ @@ -314,6 +315,8 @@ export type AgentCompactionConfig = { identifierInstructions?: string; /** Optional quality-audit retries for safeguard compaction summaries. */ qualityGuard?: AgentCompactionQualityGuardConfig; + /** Post-compaction session memory index sync mode. */ + postIndexSync?: AgentCompactionPostIndexSyncMode; /** Pre-compaction memory flush (agentic turn). Default: enabled. */ memoryFlush?: AgentCompactionMemoryFlushConfig; /** diff --git a/src/config/zod-schema.agent-defaults.ts b/src/config/zod-schema.agent-defaults.ts index 242d6959729..02148736e2a 100644 --- a/src/config/zod-schema.agent-defaults.ts +++ b/src/config/zod-schema.agent-defaults.ts @@ -103,6 +103,7 @@ export const AgentDefaultsSchema = z }) .strict() .optional(), + postIndexSync: z.enum(["off", "async", "await"]).optional(), postCompactionSections: z.array(z.string()).optional(), model: z.string().optional(), memoryFlush: z diff --git a/src/memory/index.test.ts b/src/memory/index.test.ts index 23371056b18..0cd410daa9d 100644 --- a/src/memory/index.test.ts +++ b/src/memory/index.test.ts @@ -461,6 +461,97 @@ describe("memory index", () => { } }); + it("targets explicit session files during forced sync", async () => { + const stateDir = path.join(fixtureRoot, `state-targeted-${randomUUID()}`); + const sessionDir = path.join(stateDir, "agents", "main", "sessions"); + const firstSessionPath = path.join(sessionDir, "targeted-first.jsonl"); + const secondSessionPath = path.join(sessionDir, "targeted-second.jsonl"); + const storePath = path.join(workspaceDir, `index-targeted-${randomUUID()}.sqlite`); + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = stateDir; + + await fs.mkdir(sessionDir, { recursive: true }); + await fs.writeFile( + firstSessionPath, + `${JSON.stringify({ + type: "message", + message: { role: "user", content: [{ type: "text", text: "first transcript v1" }] }, + })}\n`, + ); + await fs.writeFile( + secondSessionPath, + `${JSON.stringify({ + type: "message", + message: { role: "user", content: [{ type: "text", text: "second transcript v1" }] }, + })}\n`, + ); + + try { + const result = await getMemorySearchManager({ + cfg: createCfg({ + storePath, + sources: ["sessions"], + sessionMemory: true, + }), + agentId: "main", + }); + const manager = requireManager(result); + await manager.sync?.({ reason: "test" }); + + const db = (manager as unknown as { + db: { + prepare: ( + sql: string, + ) => { get: (path: string, source: string) => { hash: string } | undefined }; + }; + }).db; + const getSessionHash = (sessionPath: string) => + db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(sessionPath, "sessions") + ?.hash; + + const firstOriginalHash = getSessionHash("sessions/targeted-first.jsonl"); + const secondOriginalHash = getSessionHash("sessions/targeted-second.jsonl"); + + await fs.writeFile( + firstSessionPath, + `${JSON.stringify({ + type: "message", + message: { + role: "user", + content: [{ type: "text", text: "first transcript v2 after compaction" }], + }, + })}\n`, + ); + await fs.writeFile( + secondSessionPath, + `${JSON.stringify({ + type: "message", + message: { + role: "user", + content: [{ type: "text", text: "second transcript v2 should stay untouched" }], + }, + })}\n`, + ); + + await manager.sync?.({ + reason: "post-compaction", + force: true, + sessionFiles: [firstSessionPath], + }); + + expect(getSessionHash("sessions/targeted-first.jsonl")).not.toBe(firstOriginalHash); + expect(getSessionHash("sessions/targeted-second.jsonl")).toBe(secondOriginalHash); + await manager.close?.(); + } finally { + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + it("reindexes when the embedding model changes", async () => { const base = createCfg({ storePath: indexModelPath }); const baseAgents = base.agents!; diff --git a/src/memory/manager-sync-ops.ts b/src/memory/manager-sync-ops.ts index 6fd3e6bb9c0..ec420e8796a 100644 --- a/src/memory/manager-sync-ops.ts +++ b/src/memory/manager-sync-ops.ts @@ -151,6 +151,7 @@ export abstract class MemoryManagerSyncOps { protected abstract sync(params?: { reason?: string; force?: boolean; + sessionFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }): Promise; protected abstract withTimeout( @@ -611,6 +612,24 @@ export abstract class MemoryManagerSyncOps { return resolvedFile.startsWith(`${resolvedDir}${path.sep}`); } + private normalizeTargetSessionFiles(sessionFiles?: string[]): Set | null { + if (!sessionFiles || sessionFiles.length === 0) { + return null; + } + const normalized = new Set(); + for (const sessionFile of sessionFiles) { + const trimmed = sessionFile.trim(); + if (!trimmed) { + continue; + } + const resolved = path.resolve(trimmed); + if (this.isSessionFileForAgent(resolved)) { + normalized.add(resolved); + } + } + return normalized.size > 0 ? normalized : null; + } + protected ensureIntervalSync() { const minutes = this.settings.sync.intervalMinutes; if (!minutes || minutes <= 0 || this.intervalTimer) { @@ -640,12 +659,15 @@ export abstract class MemoryManagerSyncOps { } private shouldSyncSessions( - params?: { reason?: string; force?: boolean }, + params?: { reason?: string; force?: boolean; sessionFiles?: string[] }, needsFullReindex = false, ) { if (!this.sources.has("sessions")) { return false; } + if (params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) { + return true; + } if (params?.force) { return true; } @@ -752,6 +774,7 @@ export abstract class MemoryManagerSyncOps { private async syncSessionFiles(params: { needsFullReindex: boolean; + targetSessionFiles?: string[]; progress?: MemorySyncProgressState; }) { // FTS-only mode: skip embedding sync (no provider) @@ -760,13 +783,22 @@ export abstract class MemoryManagerSyncOps { return; } - const files = await listSessionFilesForAgent(this.agentId); - const activePaths = new Set(files.map((file) => sessionPathForFile(file))); - const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0; + const targetSessionFiles = params.needsFullReindex + ? null + : this.normalizeTargetSessionFiles(params.targetSessionFiles); + const files = targetSessionFiles + ? Array.from(targetSessionFiles) + : await listSessionFilesForAgent(this.agentId); + const activePaths = targetSessionFiles + ? null + : new Set(files.map((file) => sessionPathForFile(file))); + const indexAll = + params.needsFullReindex || targetSessionFiles !== null || this.sessionsDirtyFiles.size === 0; log.debug("memory sync: indexing session files", { files: files.length, indexAll, dirtyFiles: this.sessionsDirtyFiles.size, + targetedFiles: targetSessionFiles?.size ?? 0, batch: this.batch.enabled, concurrency: this.getIndexConcurrency(), }); @@ -827,6 +859,12 @@ export abstract class MemoryManagerSyncOps { }); await runWithConcurrency(tasks, this.getIndexConcurrency()); + if (activePaths === null) { + // Targeted syncs only refresh the requested transcripts and should not + // prune unrelated session rows without a full directory enumeration. + return; + } + const staleRows = this.db .prepare(`SELECT path FROM files WHERE source = ?`) .all("sessions") as Array<{ path: string }>; @@ -899,8 +937,10 @@ export abstract class MemoryManagerSyncOps { const meta = this.readMeta(); const configuredSources = this.resolveConfiguredSourcesForMeta(); const configuredScopeHash = this.resolveConfiguredScopeHash(); + const targetSessionFiles = this.normalizeTargetSessionFiles(params?.sessionFiles); + const hasTargetSessionFiles = targetSessionFiles !== null; const needsFullReindex = - params?.force || + (params?.force && !hasTargetSessionFiles) || !meta || (this.provider && meta.model !== this.provider.model) || (this.provider && meta.provider !== this.provider.id) || @@ -932,7 +972,8 @@ export abstract class MemoryManagerSyncOps { } const shouldSyncMemory = - this.sources.has("memory") && (params?.force || needsFullReindex || this.dirty); + this.sources.has("memory") && + ((!hasTargetSessionFiles && params?.force) || needsFullReindex || this.dirty); const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex); if (shouldSyncMemory) { @@ -941,7 +982,11 @@ export abstract class MemoryManagerSyncOps { } if (shouldSyncSessions) { - await this.syncSessionFiles({ needsFullReindex, progress: progress ?? undefined }); + await this.syncSessionFiles({ + needsFullReindex, + targetSessionFiles: targetSessionFiles ? Array.from(targetSessionFiles) : undefined, + progress: progress ?? undefined, + }); this.sessionsDirty = false; this.sessionsDirtyFiles.clear(); } else if (this.sessionsDirtyFiles.size > 0) { diff --git a/src/memory/manager.ts b/src/memory/manager.ts index e79f83c570a..00ac6f0b453 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -452,6 +452,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem async sync(params?: { reason?: string; force?: boolean; + sessionFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }): Promise { if (this.closed) { @@ -518,6 +519,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem private async runSyncWithReadonlyRecovery(params?: { reason?: string; force?: boolean; + sessionFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }): Promise { try { diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 7efe8f10af5..36067ff88a6 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -867,6 +867,7 @@ export class QmdMemoryManager implements MemorySearchManager { async sync(params?: { reason?: string; force?: boolean; + sessionFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }): Promise { if (params?.progress) { diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index ea581b5d6da..6cc8d9f20a4 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -181,6 +181,7 @@ class FallbackMemoryManager implements MemorySearchManager { async sync(params?: { reason?: string; force?: boolean; + sessionFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }) { if (!this.primaryFailed) { diff --git a/src/memory/types.ts b/src/memory/types.ts index 287ee6ac5a6..880384df71a 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -72,6 +72,7 @@ export interface MemorySearchManager { sync?(params?: { reason?: string; force?: boolean; + sessionFiles?: string[]; progress?: (update: MemorySyncProgressUpdate) => void; }): Promise; probeEmbeddingAvailability(): Promise;