mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-19 11:18:37 +00:00
Compaction Runner: wire post-compaction memory sync (#25561)
Merged via squash.
Prepared head SHA: 6d2bc02cc1
Co-authored-by: rodrigouroz <384037+rodrigouroz@users.noreply.github.com>
Co-authored-by: jalehman <550978+jalehman@users.noreply.github.com>
Reviewed-by: @jalehman
This commit is contained in:
@@ -55,6 +55,7 @@ Docs: https://docs.openclaw.ai
|
|||||||
- Gateway/session stores: regenerate the Swift push-test protocol models and align Windows native session-store realpath handling so protocol checks and sync session discovery stop drifting on Windows. (#44266) thanks @jalehman.
|
- Gateway/session stores: regenerate the Swift push-test protocol models and align Windows native session-store realpath handling so protocol checks and sync session discovery stop drifting on Windows. (#44266) thanks @jalehman.
|
||||||
- Context engine/session routing: forward optional `sessionKey` through context-engine lifecycle calls so plugins can see structured routing metadata during bootstrap, assembly, post-turn ingestion, and compaction. (#44157) thanks @jalehman.
|
- Context engine/session routing: forward optional `sessionKey` through context-engine lifecycle calls so plugins can see structured routing metadata during bootstrap, assembly, post-turn ingestion, and compaction. (#44157) thanks @jalehman.
|
||||||
- Agents/failover: classify z.ai `network_error` stop reasons as retryable timeouts so provider connectivity failures trigger fallback instead of surfacing raw unhandled-stop-reason errors. (#43884) Thanks @hougangdev.
|
- Agents/failover: classify z.ai `network_error` stop reasons as retryable timeouts so provider connectivity failures trigger fallback instead of surfacing raw unhandled-stop-reason errors. (#43884) Thanks @hougangdev.
|
||||||
|
- Memory/session sync: add mode-aware post-compaction session reindexing with `agents.defaults.compaction.postIndexSync` plus `agents.defaults.memorySearch.sync.sessions.postCompactionForce`, so compacted session memory can refresh immediately without forcing every deployment into synchronous reindexing. (#25561) thanks @rodrigouroz.
|
||||||
|
|
||||||
## 2026.3.11
|
## 2026.3.11
|
||||||
|
|
||||||
|
|||||||
@@ -1106,7 +1106,6 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
public let tokensuffix: String
|
public let tokensuffix: String
|
||||||
public let topic: String
|
public let topic: String
|
||||||
public let environment: String
|
public let environment: String
|
||||||
public let transport: String
|
|
||||||
|
|
||||||
public init(
|
public init(
|
||||||
ok: Bool,
|
ok: Bool,
|
||||||
@@ -1115,8 +1114,7 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
reason: String?,
|
reason: String?,
|
||||||
tokensuffix: String,
|
tokensuffix: String,
|
||||||
topic: String,
|
topic: String,
|
||||||
environment: String,
|
environment: String)
|
||||||
transport: String)
|
|
||||||
{
|
{
|
||||||
self.ok = ok
|
self.ok = ok
|
||||||
self.status = status
|
self.status = status
|
||||||
@@ -1125,7 +1123,6 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
self.tokensuffix = tokensuffix
|
self.tokensuffix = tokensuffix
|
||||||
self.topic = topic
|
self.topic = topic
|
||||||
self.environment = environment
|
self.environment = environment
|
||||||
self.transport = transport
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private enum CodingKeys: String, CodingKey {
|
private enum CodingKeys: String, CodingKey {
|
||||||
@@ -1136,7 +1133,6 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
case tokensuffix = "tokenSuffix"
|
case tokensuffix = "tokenSuffix"
|
||||||
case topic
|
case topic
|
||||||
case environment
|
case environment
|
||||||
case transport
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1106,7 +1106,6 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
public let tokensuffix: String
|
public let tokensuffix: String
|
||||||
public let topic: String
|
public let topic: String
|
||||||
public let environment: String
|
public let environment: String
|
||||||
public let transport: String
|
|
||||||
|
|
||||||
public init(
|
public init(
|
||||||
ok: Bool,
|
ok: Bool,
|
||||||
@@ -1115,8 +1114,7 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
reason: String?,
|
reason: String?,
|
||||||
tokensuffix: String,
|
tokensuffix: String,
|
||||||
topic: String,
|
topic: String,
|
||||||
environment: String,
|
environment: String)
|
||||||
transport: String)
|
|
||||||
{
|
{
|
||||||
self.ok = ok
|
self.ok = ok
|
||||||
self.status = status
|
self.status = status
|
||||||
@@ -1125,7 +1123,6 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
self.tokensuffix = tokensuffix
|
self.tokensuffix = tokensuffix
|
||||||
self.topic = topic
|
self.topic = topic
|
||||||
self.environment = environment
|
self.environment = environment
|
||||||
self.transport = transport
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private enum CodingKeys: String, CodingKey {
|
private enum CodingKeys: String, CodingKey {
|
||||||
@@ -1136,7 +1133,6 @@ public struct PushTestResult: Codable, Sendable {
|
|||||||
case tokensuffix = "tokenSuffix"
|
case tokensuffix = "tokenSuffix"
|
||||||
case topic
|
case topic
|
||||||
case environment
|
case environment
|
||||||
case transport
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -284,6 +284,7 @@ describe("memory search config", () => {
|
|||||||
expect(resolved?.sync.sessions).toEqual({
|
expect(resolved?.sync.sessions).toEqual({
|
||||||
deltaBytes: 100000,
|
deltaBytes: 100000,
|
||||||
deltaMessages: 50,
|
deltaMessages: 50,
|
||||||
|
postCompactionForce: true,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ export type ResolvedMemorySearchConfig = {
|
|||||||
sessions: {
|
sessions: {
|
||||||
deltaBytes: number;
|
deltaBytes: number;
|
||||||
deltaMessages: number;
|
deltaMessages: number;
|
||||||
|
postCompactionForce: boolean;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
query: {
|
query: {
|
||||||
@@ -248,6 +249,10 @@ function mergeConfig(
|
|||||||
overrides?.sync?.sessions?.deltaMessages ??
|
overrides?.sync?.sessions?.deltaMessages ??
|
||||||
defaults?.sync?.sessions?.deltaMessages ??
|
defaults?.sync?.sessions?.deltaMessages ??
|
||||||
DEFAULT_SESSION_DELTA_MESSAGES,
|
DEFAULT_SESSION_DELTA_MESSAGES,
|
||||||
|
postCompactionForce:
|
||||||
|
overrides?.sync?.sessions?.postCompactionForce ??
|
||||||
|
defaults?.sync?.sessions?.postCompactionForce ??
|
||||||
|
true,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
const query = {
|
const query = {
|
||||||
@@ -315,6 +320,7 @@ function mergeConfig(
|
|||||||
);
|
);
|
||||||
const deltaBytes = clampInt(sync.sessions.deltaBytes, 0, Number.MAX_SAFE_INTEGER);
|
const deltaBytes = clampInt(sync.sessions.deltaBytes, 0, Number.MAX_SAFE_INTEGER);
|
||||||
const deltaMessages = clampInt(sync.sessions.deltaMessages, 0, Number.MAX_SAFE_INTEGER);
|
const deltaMessages = clampInt(sync.sessions.deltaMessages, 0, Number.MAX_SAFE_INTEGER);
|
||||||
|
const postCompactionForce = sync.sessions.postCompactionForce;
|
||||||
return {
|
return {
|
||||||
enabled,
|
enabled,
|
||||||
sources,
|
sources,
|
||||||
@@ -336,6 +342,7 @@ function mergeConfig(
|
|||||||
sessions: {
|
sessions: {
|
||||||
deltaBytes,
|
deltaBytes,
|
||||||
deltaMessages,
|
deltaMessages,
|
||||||
|
postCompactionForce,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
query: {
|
query: {
|
||||||
|
|||||||
@@ -4,41 +4,67 @@ import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js";
|
|||||||
const {
|
const {
|
||||||
hookRunner,
|
hookRunner,
|
||||||
ensureRuntimePluginsLoaded,
|
ensureRuntimePluginsLoaded,
|
||||||
|
resolveContextEngineMock,
|
||||||
resolveModelMock,
|
resolveModelMock,
|
||||||
sessionCompactImpl,
|
sessionCompactImpl,
|
||||||
triggerInternalHook,
|
triggerInternalHook,
|
||||||
sanitizeSessionHistoryMock,
|
sanitizeSessionHistoryMock,
|
||||||
contextEngineCompactMock,
|
contextEngineCompactMock,
|
||||||
} = vi.hoisted(() => ({
|
getMemorySearchManagerMock,
|
||||||
hookRunner: {
|
resolveMemorySearchConfigMock,
|
||||||
hasHooks: vi.fn(),
|
resolveSessionAgentIdMock,
|
||||||
runBeforeCompaction: vi.fn(),
|
} = vi.hoisted(() => {
|
||||||
runAfterCompaction: vi.fn(),
|
const contextEngineCompactMock = vi.fn(async () => ({
|
||||||
},
|
|
||||||
ensureRuntimePluginsLoaded: vi.fn(),
|
|
||||||
resolveModelMock: vi.fn(() => ({
|
|
||||||
model: { provider: "openai", api: "responses", id: "fake", input: [] },
|
|
||||||
error: null,
|
|
||||||
authStorage: { setRuntimeApiKey: vi.fn() },
|
|
||||||
modelRegistry: {},
|
|
||||||
})),
|
|
||||||
sessionCompactImpl: vi.fn(async () => ({
|
|
||||||
summary: "summary",
|
|
||||||
firstKeptEntryId: "entry-1",
|
|
||||||
tokensBefore: 120,
|
|
||||||
details: { ok: true },
|
|
||||||
})),
|
|
||||||
triggerInternalHook: vi.fn(),
|
|
||||||
sanitizeSessionHistoryMock: vi.fn(async (params: { messages: unknown[] }) => params.messages),
|
|
||||||
contextEngineCompactMock: vi.fn(async () => ({
|
|
||||||
ok: true as boolean,
|
ok: true as boolean,
|
||||||
compacted: true as boolean,
|
compacted: true as boolean,
|
||||||
reason: undefined as string | undefined,
|
reason: undefined as string | undefined,
|
||||||
result: { summary: "engine-summary", tokensAfter: 50 } as
|
result: { summary: "engine-summary", tokensAfter: 50 } as
|
||||||
| { summary: string; tokensAfter: number }
|
| { summary: string; tokensAfter: number }
|
||||||
| undefined,
|
| undefined,
|
||||||
})),
|
}));
|
||||||
}));
|
|
||||||
|
return {
|
||||||
|
hookRunner: {
|
||||||
|
hasHooks: vi.fn(),
|
||||||
|
runBeforeCompaction: vi.fn(),
|
||||||
|
runAfterCompaction: vi.fn(),
|
||||||
|
},
|
||||||
|
ensureRuntimePluginsLoaded: vi.fn(),
|
||||||
|
resolveContextEngineMock: vi.fn(async () => ({
|
||||||
|
info: { ownsCompaction: true },
|
||||||
|
compact: contextEngineCompactMock,
|
||||||
|
})),
|
||||||
|
resolveModelMock: vi.fn(() => ({
|
||||||
|
model: { provider: "openai", api: "responses", id: "fake", input: [] },
|
||||||
|
error: null,
|
||||||
|
authStorage: { setRuntimeApiKey: vi.fn() },
|
||||||
|
modelRegistry: {},
|
||||||
|
})),
|
||||||
|
sessionCompactImpl: vi.fn(async () => ({
|
||||||
|
summary: "summary",
|
||||||
|
firstKeptEntryId: "entry-1",
|
||||||
|
tokensBefore: 120,
|
||||||
|
details: { ok: true },
|
||||||
|
})),
|
||||||
|
triggerInternalHook: vi.fn(),
|
||||||
|
sanitizeSessionHistoryMock: vi.fn(async (params: { messages: unknown[] }) => params.messages),
|
||||||
|
contextEngineCompactMock,
|
||||||
|
getMemorySearchManagerMock: vi.fn(async () => ({
|
||||||
|
manager: {
|
||||||
|
sync: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
resolveMemorySearchConfigMock: vi.fn(() => ({
|
||||||
|
sources: ["sessions"],
|
||||||
|
sync: {
|
||||||
|
sessions: {
|
||||||
|
postCompactionForce: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
resolveSessionAgentIdMock: vi.fn(() => "main"),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock("../../plugins/hook-runner-global.js", () => ({
|
vi.mock("../../plugins/hook-runner-global.js", () => ({
|
||||||
getGlobalHookRunner: () => hookRunner,
|
getGlobalHookRunner: () => hookRunner,
|
||||||
@@ -135,10 +161,7 @@ vi.mock("../session-write-lock.js", () => ({
|
|||||||
|
|
||||||
vi.mock("../../context-engine/index.js", () => ({
|
vi.mock("../../context-engine/index.js", () => ({
|
||||||
ensureContextEnginesInitialized: vi.fn(),
|
ensureContextEnginesInitialized: vi.fn(),
|
||||||
resolveContextEngine: vi.fn(async () => ({
|
resolveContextEngine: resolveContextEngineMock,
|
||||||
info: { ownsCompaction: true },
|
|
||||||
compact: contextEngineCompactMock,
|
|
||||||
})),
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("../../process/command-queue.js", () => ({
|
vi.mock("../../process/command-queue.js", () => ({
|
||||||
@@ -211,9 +234,18 @@ vi.mock("../agent-paths.js", () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("../agent-scope.js", () => ({
|
vi.mock("../agent-scope.js", () => ({
|
||||||
|
resolveSessionAgentId: resolveSessionAgentIdMock,
|
||||||
resolveSessionAgentIds: vi.fn(() => ({ defaultAgentId: "main", sessionAgentId: "main" })),
|
resolveSessionAgentIds: vi.fn(() => ({ defaultAgentId: "main", sessionAgentId: "main" })),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock("../memory-search.js", () => ({
|
||||||
|
resolveMemorySearchConfig: resolveMemorySearchConfigMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../../memory/index.js", () => ({
|
||||||
|
getMemorySearchManager: getMemorySearchManagerMock,
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock("../date-time.js", () => ({
|
vi.mock("../date-time.js", () => ({
|
||||||
formatUserTime: vi.fn(() => ""),
|
formatUserTime: vi.fn(() => ""),
|
||||||
resolveUserTimeFormat: vi.fn(() => ""),
|
resolveUserTimeFormat: vi.fn(() => ""),
|
||||||
@@ -314,6 +346,23 @@ describe("compactEmbeddedPiSessionDirect hooks", () => {
|
|||||||
sanitizeSessionHistoryMock.mockImplementation(async (params: { messages: unknown[] }) => {
|
sanitizeSessionHistoryMock.mockImplementation(async (params: { messages: unknown[] }) => {
|
||||||
return params.messages;
|
return params.messages;
|
||||||
});
|
});
|
||||||
|
getMemorySearchManagerMock.mockReset();
|
||||||
|
getMemorySearchManagerMock.mockResolvedValue({
|
||||||
|
manager: {
|
||||||
|
sync: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
resolveMemorySearchConfigMock.mockReset();
|
||||||
|
resolveMemorySearchConfigMock.mockReturnValue({
|
||||||
|
sources: ["sessions"],
|
||||||
|
sync: {
|
||||||
|
sessions: {
|
||||||
|
postCompactionForce: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
resolveSessionAgentIdMock.mockReset();
|
||||||
|
resolveSessionAgentIdMock.mockReturnValue("main");
|
||||||
unregisterApiProviders(getCustomApiRegistrySourceId("ollama"));
|
unregisterApiProviders(getCustomApiRegistrySourceId("ollama"));
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -452,6 +501,161 @@ describe("compactEmbeddedPiSessionDirect hooks", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("skips sync in await mode when postCompactionForce is false", async () => {
|
||||||
|
const sync = vi.fn(async () => {});
|
||||||
|
getMemorySearchManagerMock.mockResolvedValue({ manager: { sync } });
|
||||||
|
resolveMemorySearchConfigMock.mockReturnValue({
|
||||||
|
sources: ["sessions"],
|
||||||
|
sync: {
|
||||||
|
sessions: {
|
||||||
|
postCompactionForce: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await compactEmbeddedPiSessionDirect({
|
||||||
|
sessionId: "session-1",
|
||||||
|
sessionKey: "agent:main:session-1",
|
||||||
|
sessionFile: "/tmp/session.jsonl",
|
||||||
|
workspaceDir: "/tmp",
|
||||||
|
customInstructions: "focus on decisions",
|
||||||
|
config: {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
compaction: {
|
||||||
|
postIndexSync: "await",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(resolveSessionAgentIdMock).toHaveBeenCalledWith({
|
||||||
|
sessionKey: "agent:main:session-1",
|
||||||
|
config: expect.any(Object),
|
||||||
|
});
|
||||||
|
expect(getMemorySearchManagerMock).not.toHaveBeenCalled();
|
||||||
|
expect(sync).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("awaits post-compaction memory sync in await mode when postCompactionForce is true", async () => {
|
||||||
|
let releaseSync: (() => void) | undefined;
|
||||||
|
const syncGate = new Promise<void>((resolve) => {
|
||||||
|
releaseSync = resolve;
|
||||||
|
});
|
||||||
|
const sync = vi.fn(() => syncGate);
|
||||||
|
getMemorySearchManagerMock.mockResolvedValue({ manager: { sync } });
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const resultPromise = compactEmbeddedPiSessionDirect({
|
||||||
|
sessionId: "session-1",
|
||||||
|
sessionKey: "agent:main:session-1",
|
||||||
|
sessionFile: "/tmp/session.jsonl",
|
||||||
|
workspaceDir: "/tmp",
|
||||||
|
customInstructions: "focus on decisions",
|
||||||
|
config: {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
compaction: {
|
||||||
|
postIndexSync: "await",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
void resultPromise.then(() => {
|
||||||
|
settled = true;
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(sync).toHaveBeenCalledWith({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: ["/tmp/session.jsonl"],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
expect(settled).toBe(false);
|
||||||
|
releaseSync?.();
|
||||||
|
const result = await resultPromise;
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(settled).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips post-compaction memory sync when the mode is off", async () => {
|
||||||
|
const sync = vi.fn(async () => {});
|
||||||
|
getMemorySearchManagerMock.mockResolvedValue({ manager: { sync } });
|
||||||
|
|
||||||
|
const result = await compactEmbeddedPiSessionDirect({
|
||||||
|
sessionId: "session-1",
|
||||||
|
sessionKey: "agent:main:session-1",
|
||||||
|
sessionFile: "/tmp/session.jsonl",
|
||||||
|
workspaceDir: "/tmp",
|
||||||
|
customInstructions: "focus on decisions",
|
||||||
|
config: {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
compaction: {
|
||||||
|
postIndexSync: "off",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(resolveSessionAgentIdMock).not.toHaveBeenCalled();
|
||||||
|
expect(getMemorySearchManagerMock).not.toHaveBeenCalled();
|
||||||
|
expect(sync).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fires post-compaction memory sync without awaiting it in async mode", async () => {
|
||||||
|
const sync = vi.fn(async () => {});
|
||||||
|
let resolveManager: ((value: { manager: { sync: typeof sync } }) => void) | undefined;
|
||||||
|
const managerGate = new Promise<{ manager: { sync: typeof sync } }>((resolve) => {
|
||||||
|
resolveManager = resolve;
|
||||||
|
});
|
||||||
|
getMemorySearchManagerMock.mockImplementation(() => managerGate);
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const resultPromise = compactEmbeddedPiSessionDirect({
|
||||||
|
sessionId: "session-1",
|
||||||
|
sessionKey: "agent:main:session-1",
|
||||||
|
sessionFile: "/tmp/session.jsonl",
|
||||||
|
workspaceDir: "/tmp",
|
||||||
|
customInstructions: "focus on decisions",
|
||||||
|
config: {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
compaction: {
|
||||||
|
postIndexSync: "async",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
void resultPromise.then(() => {
|
||||||
|
settled = true;
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(settled).toBe(true);
|
||||||
|
});
|
||||||
|
expect(sync).not.toHaveBeenCalled();
|
||||||
|
resolveManager?.({ manager: { sync } });
|
||||||
|
await managerGate;
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(sync).toHaveBeenCalledWith({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: ["/tmp/session.jsonl"],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const result = await resultPromise;
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("registers the Ollama api provider before compaction", async () => {
|
it("registers the Ollama api provider before compaction", async () => {
|
||||||
resolveModelMock.mockReturnValue({
|
resolveModelMock.mockReturnValue({
|
||||||
model: {
|
model: {
|
||||||
@@ -493,6 +697,11 @@ describe("compactEmbeddedPiSession hooks (ownsCompaction engine)", () => {
|
|||||||
hookRunner.hasHooks.mockReset();
|
hookRunner.hasHooks.mockReset();
|
||||||
hookRunner.runBeforeCompaction.mockReset();
|
hookRunner.runBeforeCompaction.mockReset();
|
||||||
hookRunner.runAfterCompaction.mockReset();
|
hookRunner.runAfterCompaction.mockReset();
|
||||||
|
resolveContextEngineMock.mockReset();
|
||||||
|
resolveContextEngineMock.mockResolvedValue({
|
||||||
|
info: { ownsCompaction: true },
|
||||||
|
compact: contextEngineCompactMock,
|
||||||
|
});
|
||||||
contextEngineCompactMock.mockReset();
|
contextEngineCompactMock.mockReset();
|
||||||
contextEngineCompactMock.mockResolvedValue({
|
contextEngineCompactMock.mockResolvedValue({
|
||||||
ok: true,
|
ok: true,
|
||||||
@@ -546,8 +755,47 @@ describe("compactEmbeddedPiSession hooks (ownsCompaction engine)", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("emits a transcript update and post-compaction memory sync on the engine-owned path", async () => {
|
||||||
|
const listener = vi.fn();
|
||||||
|
const cleanup = onSessionTranscriptUpdate(listener);
|
||||||
|
const sync = vi.fn(async () => {});
|
||||||
|
getMemorySearchManagerMock.mockResolvedValue({ manager: { sync } });
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await compactEmbeddedPiSession({
|
||||||
|
sessionId: "session-1",
|
||||||
|
sessionKey: "agent:main:session-1",
|
||||||
|
sessionFile: " /tmp/session.jsonl ",
|
||||||
|
workspaceDir: "/tmp",
|
||||||
|
customInstructions: "focus on decisions",
|
||||||
|
enqueue: (task) => task(),
|
||||||
|
config: {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
compaction: {
|
||||||
|
postIndexSync: "await",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(listener).toHaveBeenCalledTimes(1);
|
||||||
|
expect(listener).toHaveBeenCalledWith({ sessionFile: "/tmp/session.jsonl" });
|
||||||
|
expect(sync).toHaveBeenCalledWith({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: ["/tmp/session.jsonl"],
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("does not fire after_compaction when compaction fails", async () => {
|
it("does not fire after_compaction when compaction fails", async () => {
|
||||||
hookRunner.hasHooks.mockReturnValue(true);
|
hookRunner.hasHooks.mockReturnValue(true);
|
||||||
|
const sync = vi.fn(async () => {});
|
||||||
|
getMemorySearchManagerMock.mockResolvedValue({ manager: { sync } });
|
||||||
contextEngineCompactMock.mockResolvedValue({
|
contextEngineCompactMock.mockResolvedValue({
|
||||||
ok: false,
|
ok: false,
|
||||||
compacted: false,
|
compacted: false,
|
||||||
@@ -567,6 +815,44 @@ describe("compactEmbeddedPiSession hooks (ownsCompaction engine)", () => {
|
|||||||
expect(result.ok).toBe(false);
|
expect(result.ok).toBe(false);
|
||||||
expect(hookRunner.runBeforeCompaction).toHaveBeenCalled();
|
expect(hookRunner.runBeforeCompaction).toHaveBeenCalled();
|
||||||
expect(hookRunner.runAfterCompaction).not.toHaveBeenCalled();
|
expect(hookRunner.runAfterCompaction).not.toHaveBeenCalled();
|
||||||
|
expect(sync).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not duplicate transcript updates or sync in the wrapper when the engine delegates compaction", async () => {
|
||||||
|
const listener = vi.fn();
|
||||||
|
const cleanup = onSessionTranscriptUpdate(listener);
|
||||||
|
const sync = vi.fn(async () => {});
|
||||||
|
getMemorySearchManagerMock.mockResolvedValue({ manager: { sync } });
|
||||||
|
resolveContextEngineMock.mockResolvedValue({
|
||||||
|
info: { ownsCompaction: false },
|
||||||
|
compact: contextEngineCompactMock,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await compactEmbeddedPiSession({
|
||||||
|
sessionId: "session-1",
|
||||||
|
sessionKey: "agent:main:session-1",
|
||||||
|
sessionFile: "/tmp/session.jsonl",
|
||||||
|
workspaceDir: "/tmp",
|
||||||
|
customInstructions: "focus on decisions",
|
||||||
|
enqueue: (task) => task(),
|
||||||
|
config: {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
compaction: {
|
||||||
|
postIndexSync: "await",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(listener).not.toHaveBeenCalled();
|
||||||
|
expect(sync).not.toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("catches and logs hook exceptions without aborting compaction", async () => {
|
it("catches and logs hook exceptions without aborting compaction", async () => {
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import {
|
|||||||
import { createInternalHookEvent, triggerInternalHook } from "../../hooks/internal-hooks.js";
|
import { createInternalHookEvent, triggerInternalHook } from "../../hooks/internal-hooks.js";
|
||||||
import { getMachineDisplayName } from "../../infra/machine-name.js";
|
import { getMachineDisplayName } from "../../infra/machine-name.js";
|
||||||
import { generateSecureToken } from "../../infra/secure-random.js";
|
import { generateSecureToken } from "../../infra/secure-random.js";
|
||||||
|
import { getMemorySearchManager } from "../../memory/index.js";
|
||||||
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
||||||
import { type enqueueCommand, enqueueCommandInLane } from "../../process/command-queue.js";
|
import { type enqueueCommand, enqueueCommandInLane } from "../../process/command-queue.js";
|
||||||
import { isCronSessionKey, isSubagentSessionKey } from "../../routing/session-key.js";
|
import { isCronSessionKey, isSubagentSessionKey } from "../../routing/session-key.js";
|
||||||
@@ -30,7 +31,7 @@ import { resolveUserPath } from "../../utils.js";
|
|||||||
import { normalizeMessageChannel } from "../../utils/message-channel.js";
|
import { normalizeMessageChannel } from "../../utils/message-channel.js";
|
||||||
import { isReasoningTagProvider } from "../../utils/provider-utils.js";
|
import { isReasoningTagProvider } from "../../utils/provider-utils.js";
|
||||||
import { resolveOpenClawAgentDir } from "../agent-paths.js";
|
import { resolveOpenClawAgentDir } from "../agent-paths.js";
|
||||||
import { resolveSessionAgentIds } from "../agent-scope.js";
|
import { resolveSessionAgentId, resolveSessionAgentIds } from "../agent-scope.js";
|
||||||
import type { ExecElevatedDefaults } from "../bash-tools.js";
|
import type { ExecElevatedDefaults } from "../bash-tools.js";
|
||||||
import { makeBootstrapWarn, resolveBootstrapContextForRun } from "../bootstrap-files.js";
|
import { makeBootstrapWarn, resolveBootstrapContextForRun } from "../bootstrap-files.js";
|
||||||
import { listChannelSupportedActions, resolveChannelMessageToolHints } from "../channel-tools.js";
|
import { listChannelSupportedActions, resolveChannelMessageToolHints } from "../channel-tools.js";
|
||||||
@@ -39,6 +40,7 @@ import { ensureCustomApiRegistered } from "../custom-api-registry.js";
|
|||||||
import { formatUserTime, resolveUserTimeFormat, resolveUserTimezone } from "../date-time.js";
|
import { formatUserTime, resolveUserTimeFormat, resolveUserTimezone } from "../date-time.js";
|
||||||
import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, DEFAULT_PROVIDER } from "../defaults.js";
|
import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, DEFAULT_PROVIDER } from "../defaults.js";
|
||||||
import { resolveOpenClawDocsPath } from "../docs-path.js";
|
import { resolveOpenClawDocsPath } from "../docs-path.js";
|
||||||
|
import { resolveMemorySearchConfig } from "../memory-search.js";
|
||||||
import { getApiKeyForModel, resolveModelAuthMode } from "../model-auth.js";
|
import { getApiKeyForModel, resolveModelAuthMode } from "../model-auth.js";
|
||||||
import { supportsModelTools } from "../model-tool-support.js";
|
import { supportsModelTools } from "../model-tool-support.js";
|
||||||
import { ensureOpenClawModelsJson } from "../models-config.js";
|
import { ensureOpenClawModelsJson } from "../models-config.js";
|
||||||
@@ -268,6 +270,95 @@ function classifyCompactionReason(reason?: string): string {
|
|||||||
return "unknown";
|
return "unknown";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolvePostCompactionIndexSyncMode(config?: OpenClawConfig): "off" | "async" | "await" {
|
||||||
|
const mode = config?.agents?.defaults?.compaction?.postIndexSync;
|
||||||
|
if (mode === "off" || mode === "async" || mode === "await") {
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
return "async";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runPostCompactionSessionMemorySync(params: {
|
||||||
|
config?: OpenClawConfig;
|
||||||
|
sessionKey?: string;
|
||||||
|
sessionFile: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
if (!params.config) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const sessionFile = params.sessionFile.trim();
|
||||||
|
if (!sessionFile) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const agentId = resolveSessionAgentId({
|
||||||
|
sessionKey: params.sessionKey,
|
||||||
|
config: params.config,
|
||||||
|
});
|
||||||
|
const resolvedMemory = resolveMemorySearchConfig(params.config, agentId);
|
||||||
|
if (!resolvedMemory || !resolvedMemory.sources.includes("sessions")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!resolvedMemory.sync.sessions.postCompactionForce) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { manager } = await getMemorySearchManager({
|
||||||
|
cfg: params.config,
|
||||||
|
agentId,
|
||||||
|
});
|
||||||
|
if (!manager?.sync) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const syncTask = manager.sync({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: [sessionFile],
|
||||||
|
});
|
||||||
|
await syncTask;
|
||||||
|
} catch (err) {
|
||||||
|
log.warn(`memory sync skipped (post-compaction): ${String(err)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function syncPostCompactionSessionMemory(params: {
|
||||||
|
config?: OpenClawConfig;
|
||||||
|
sessionKey?: string;
|
||||||
|
sessionFile: string;
|
||||||
|
mode: "off" | "async" | "await";
|
||||||
|
}): Promise<void> {
|
||||||
|
if (params.mode === "off" || !params.config) {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
|
||||||
|
const syncTask = runPostCompactionSessionMemorySync({
|
||||||
|
config: params.config,
|
||||||
|
sessionKey: params.sessionKey,
|
||||||
|
sessionFile: params.sessionFile,
|
||||||
|
});
|
||||||
|
if (params.mode === "await") {
|
||||||
|
return syncTask;
|
||||||
|
}
|
||||||
|
void syncTask;
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runPostCompactionSideEffects(params: {
|
||||||
|
config?: OpenClawConfig;
|
||||||
|
sessionKey?: string;
|
||||||
|
sessionFile: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
const sessionFile = params.sessionFile.trim();
|
||||||
|
if (!sessionFile) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
emitSessionTranscriptUpdate(sessionFile);
|
||||||
|
await syncPostCompactionSessionMemory({
|
||||||
|
config: params.config,
|
||||||
|
sessionKey: params.sessionKey,
|
||||||
|
sessionFile,
|
||||||
|
mode: resolvePostCompactionIndexSyncMode(params.config),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Core compaction logic without lane queueing.
|
* Core compaction logic without lane queueing.
|
||||||
* Use this when already inside a session/global lane to avoid deadlocks.
|
* Use this when already inside a session/global lane to avoid deadlocks.
|
||||||
@@ -809,7 +900,11 @@ export async function compactEmbeddedPiSessionDirect(
|
|||||||
const result = await compactWithSafetyTimeout(() =>
|
const result = await compactWithSafetyTimeout(() =>
|
||||||
session.compact(params.customInstructions),
|
session.compact(params.customInstructions),
|
||||||
);
|
);
|
||||||
emitSessionTranscriptUpdate(params.sessionFile);
|
await runPostCompactionSideEffects({
|
||||||
|
config: params.config,
|
||||||
|
sessionKey: params.sessionKey,
|
||||||
|
sessionFile: params.sessionFile,
|
||||||
|
});
|
||||||
// Estimate tokens after compaction by summing token estimates for remaining messages
|
// Estimate tokens after compaction by summing token estimates for remaining messages
|
||||||
let tokensAfter: number | undefined;
|
let tokensAfter: number | undefined;
|
||||||
try {
|
try {
|
||||||
@@ -999,6 +1094,13 @@ export async function compactEmbeddedPiSession(
|
|||||||
force: params.trigger === "manual",
|
force: params.trigger === "manual",
|
||||||
runtimeContext: params as Record<string, unknown>,
|
runtimeContext: params as Record<string, unknown>,
|
||||||
});
|
});
|
||||||
|
if (engineOwnsCompaction && result.ok && result.compacted) {
|
||||||
|
await runPostCompactionSideEffects({
|
||||||
|
config: params.config,
|
||||||
|
sessionKey: params.sessionKey,
|
||||||
|
sessionFile: params.sessionFile,
|
||||||
|
});
|
||||||
|
}
|
||||||
if (result.ok && result.compacted && hookRunner?.hasHooks("after_compaction")) {
|
if (result.ok && result.compacted && hookRunner?.hasHooks("after_compaction")) {
|
||||||
try {
|
try {
|
||||||
await hookRunner.runAfterCompaction(
|
await hookRunner.runAfterCompaction(
|
||||||
|
|||||||
@@ -930,6 +930,8 @@ export const FIELD_HELP: Record<string, string> = {
|
|||||||
"Requires at least this many newly appended bytes before session transcript changes trigger reindex (default: 100000). Increase to reduce frequent small reindexes, or lower for faster transcript freshness.",
|
"Requires at least this many newly appended bytes before session transcript changes trigger reindex (default: 100000). Increase to reduce frequent small reindexes, or lower for faster transcript freshness.",
|
||||||
"agents.defaults.memorySearch.sync.sessions.deltaMessages":
|
"agents.defaults.memorySearch.sync.sessions.deltaMessages":
|
||||||
"Requires at least this many appended transcript messages before reindex is triggered (default: 50). Lower this for near-real-time transcript recall, or raise it to reduce indexing churn.",
|
"Requires at least this many appended transcript messages before reindex is triggered (default: 50). Lower this for near-real-time transcript recall, or raise it to reduce indexing churn.",
|
||||||
|
"agents.defaults.memorySearch.sync.sessions.postCompactionForce":
|
||||||
|
"Forces a session memory-search reindex after compaction-triggered transcript updates (default: true). Keep enabled when compacted summaries must be immediately searchable, or disable to reduce write-time indexing pressure.",
|
||||||
ui: "UI presentation settings for accenting and assistant identity shown in control surfaces. Use this for branding and readability customization without changing runtime behavior.",
|
ui: "UI presentation settings for accenting and assistant identity shown in control surfaces. Use this for branding and readability customization without changing runtime behavior.",
|
||||||
"ui.seamColor":
|
"ui.seamColor":
|
||||||
"Primary accent/seam color used by UI surfaces for emphasis, badges, and visual identity cues. Use high-contrast values that remain readable across light/dark themes.",
|
"Primary accent/seam color used by UI surfaces for emphasis, badges, and visual identity cues. Use high-contrast values that remain readable across light/dark themes.",
|
||||||
@@ -1033,6 +1035,8 @@ export const FIELD_HELP: Record<string, string> = {
|
|||||||
"Enables summary quality audits and regeneration retries for safeguard compaction. Default: false, so safeguard mode alone does not turn on retry behavior.",
|
"Enables summary quality audits and regeneration retries for safeguard compaction. Default: false, so safeguard mode alone does not turn on retry behavior.",
|
||||||
"agents.defaults.compaction.qualityGuard.maxRetries":
|
"agents.defaults.compaction.qualityGuard.maxRetries":
|
||||||
"Maximum number of regeneration retries after a failed safeguard summary quality audit. Use small values to bound extra latency and token cost.",
|
"Maximum number of regeneration retries after a failed safeguard summary quality audit. Use small values to bound extra latency and token cost.",
|
||||||
|
"agents.defaults.compaction.postIndexSync":
|
||||||
|
'Controls post-compaction session memory reindex mode: "off", "async", or "await" (default: "async"). Use "await" for strongest freshness, "async" for lower compaction latency, and "off" only when session-memory sync is handled elsewhere.',
|
||||||
"agents.defaults.compaction.postCompactionSections":
|
"agents.defaults.compaction.postCompactionSections":
|
||||||
'AGENTS.md H2/H3 section names re-injected after compaction so the agent reruns critical startup guidance. Leave unset to use "Session Startup"/"Red Lines" with legacy fallback to "Every Session"/"Safety"; set to [] to disable reinjection entirely.',
|
'AGENTS.md H2/H3 section names re-injected after compaction so the agent reruns critical startup guidance. Leave unset to use "Session Startup"/"Red Lines" with legacy fallback to "Every Session"/"Safety"; set to [] to disable reinjection entirely.',
|
||||||
"agents.defaults.compaction.model":
|
"agents.defaults.compaction.model":
|
||||||
|
|||||||
@@ -354,6 +354,8 @@ export const FIELD_LABELS: Record<string, string> = {
|
|||||||
"agents.defaults.memorySearch.sync.watchDebounceMs": "Memory Watch Debounce (ms)",
|
"agents.defaults.memorySearch.sync.watchDebounceMs": "Memory Watch Debounce (ms)",
|
||||||
"agents.defaults.memorySearch.sync.sessions.deltaBytes": "Session Delta Bytes",
|
"agents.defaults.memorySearch.sync.sessions.deltaBytes": "Session Delta Bytes",
|
||||||
"agents.defaults.memorySearch.sync.sessions.deltaMessages": "Session Delta Messages",
|
"agents.defaults.memorySearch.sync.sessions.deltaMessages": "Session Delta Messages",
|
||||||
|
"agents.defaults.memorySearch.sync.sessions.postCompactionForce":
|
||||||
|
"Force Reindex After Compaction",
|
||||||
"agents.defaults.memorySearch.query.maxResults": "Memory Search Max Results",
|
"agents.defaults.memorySearch.query.maxResults": "Memory Search Max Results",
|
||||||
"agents.defaults.memorySearch.query.minScore": "Memory Search Min Score",
|
"agents.defaults.memorySearch.query.minScore": "Memory Search Min Score",
|
||||||
"agents.defaults.memorySearch.query.hybrid.enabled": "Memory Search Hybrid",
|
"agents.defaults.memorySearch.query.hybrid.enabled": "Memory Search Hybrid",
|
||||||
@@ -468,6 +470,7 @@ export const FIELD_LABELS: Record<string, string> = {
|
|||||||
"agents.defaults.compaction.qualityGuard": "Compaction Quality Guard",
|
"agents.defaults.compaction.qualityGuard": "Compaction Quality Guard",
|
||||||
"agents.defaults.compaction.qualityGuard.enabled": "Compaction Quality Guard Enabled",
|
"agents.defaults.compaction.qualityGuard.enabled": "Compaction Quality Guard Enabled",
|
||||||
"agents.defaults.compaction.qualityGuard.maxRetries": "Compaction Quality Guard Max Retries",
|
"agents.defaults.compaction.qualityGuard.maxRetries": "Compaction Quality Guard Max Retries",
|
||||||
|
"agents.defaults.compaction.postIndexSync": "Compaction Post-Index Sync",
|
||||||
"agents.defaults.compaction.postCompactionSections": "Post-Compaction Context Sections",
|
"agents.defaults.compaction.postCompactionSections": "Post-Compaction Context Sections",
|
||||||
"agents.defaults.compaction.model": "Compaction Model Override",
|
"agents.defaults.compaction.model": "Compaction Model Override",
|
||||||
"agents.defaults.compaction.memoryFlush": "Compaction Memory Flush",
|
"agents.defaults.compaction.memoryFlush": "Compaction Memory Flush",
|
||||||
|
|||||||
@@ -287,6 +287,7 @@ export type AgentDefaultsConfig = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export type AgentCompactionMode = "default" | "safeguard";
|
export type AgentCompactionMode = "default" | "safeguard";
|
||||||
|
export type AgentCompactionPostIndexSyncMode = "off" | "async" | "await";
|
||||||
export type AgentCompactionIdentifierPolicy = "strict" | "off" | "custom";
|
export type AgentCompactionIdentifierPolicy = "strict" | "off" | "custom";
|
||||||
export type AgentCompactionQualityGuardConfig = {
|
export type AgentCompactionQualityGuardConfig = {
|
||||||
/** Enable compaction summary quality audits and regeneration retries. Default: false. */
|
/** Enable compaction summary quality audits and regeneration retries. Default: false. */
|
||||||
@@ -314,6 +315,8 @@ export type AgentCompactionConfig = {
|
|||||||
identifierInstructions?: string;
|
identifierInstructions?: string;
|
||||||
/** Optional quality-audit retries for safeguard compaction summaries. */
|
/** Optional quality-audit retries for safeguard compaction summaries. */
|
||||||
qualityGuard?: AgentCompactionQualityGuardConfig;
|
qualityGuard?: AgentCompactionQualityGuardConfig;
|
||||||
|
/** Post-compaction session memory index sync mode. */
|
||||||
|
postIndexSync?: AgentCompactionPostIndexSyncMode;
|
||||||
/** Pre-compaction memory flush (agentic turn). Default: enabled. */
|
/** Pre-compaction memory flush (agentic turn). Default: enabled. */
|
||||||
memoryFlush?: AgentCompactionMemoryFlushConfig;
|
memoryFlush?: AgentCompactionMemoryFlushConfig;
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -402,6 +402,8 @@ export type MemorySearchConfig = {
|
|||||||
deltaBytes?: number;
|
deltaBytes?: number;
|
||||||
/** Minimum appended JSONL lines before session transcripts are reindexed. */
|
/** Minimum appended JSONL lines before session transcripts are reindexed. */
|
||||||
deltaMessages?: number;
|
deltaMessages?: number;
|
||||||
|
/** Force session reindex after compaction-triggered transcript updates (default: true). */
|
||||||
|
postCompactionForce?: boolean;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
/** Query behavior. */
|
/** Query behavior. */
|
||||||
|
|||||||
@@ -103,6 +103,7 @@ export const AgentDefaultsSchema = z
|
|||||||
})
|
})
|
||||||
.strict()
|
.strict()
|
||||||
.optional(),
|
.optional(),
|
||||||
|
postIndexSync: z.enum(["off", "async", "await"]).optional(),
|
||||||
postCompactionSections: z.array(z.string()).optional(),
|
postCompactionSections: z.array(z.string()).optional(),
|
||||||
model: z.string().optional(),
|
model: z.string().optional(),
|
||||||
memoryFlush: z
|
memoryFlush: z
|
||||||
|
|||||||
@@ -649,6 +649,7 @@ export const MemorySearchSchema = z
|
|||||||
.object({
|
.object({
|
||||||
deltaBytes: z.number().int().nonnegative().optional(),
|
deltaBytes: z.number().int().nonnegative().optional(),
|
||||||
deltaMessages: z.number().int().nonnegative().optional(),
|
deltaMessages: z.number().int().nonnegative().optional(),
|
||||||
|
postCompactionForce: z.boolean().optional(),
|
||||||
})
|
})
|
||||||
.strict()
|
.strict()
|
||||||
.optional(),
|
.optional(),
|
||||||
|
|||||||
@@ -461,6 +461,391 @@ describe("memory index", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("targets explicit session files during post-compaction sync", async () => {
|
||||||
|
const stateDir = path.join(fixtureRoot, `state-targeted-${randomUUID()}`);
|
||||||
|
const sessionDir = path.join(stateDir, "agents", "main", "sessions");
|
||||||
|
const firstSessionPath = path.join(sessionDir, "targeted-first.jsonl");
|
||||||
|
const secondSessionPath = path.join(sessionDir, "targeted-second.jsonl");
|
||||||
|
const storePath = path.join(workspaceDir, `index-targeted-${randomUUID()}.sqlite`);
|
||||||
|
const previousStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||||
|
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||||
|
|
||||||
|
await fs.mkdir(sessionDir, { recursive: true });
|
||||||
|
await fs.writeFile(
|
||||||
|
firstSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: { role: "user", content: [{ type: "text", text: "first transcript v1" }] },
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
await fs.writeFile(
|
||||||
|
secondSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: { role: "user", content: [{ type: "text", text: "second transcript v1" }] },
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await getMemorySearchManager({
|
||||||
|
cfg: createCfg({
|
||||||
|
storePath,
|
||||||
|
sources: ["sessions"],
|
||||||
|
sessionMemory: true,
|
||||||
|
}),
|
||||||
|
agentId: "main",
|
||||||
|
});
|
||||||
|
const manager = requireManager(result);
|
||||||
|
await manager.sync?.({ reason: "test" });
|
||||||
|
|
||||||
|
const db = (
|
||||||
|
manager as unknown as {
|
||||||
|
db: {
|
||||||
|
prepare: (sql: string) => {
|
||||||
|
get: (path: string, source: string) => { hash: string } | undefined;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
).db;
|
||||||
|
const getSessionHash = (sessionPath: string) =>
|
||||||
|
db
|
||||||
|
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
||||||
|
.get(sessionPath, "sessions")?.hash;
|
||||||
|
|
||||||
|
const firstOriginalHash = getSessionHash("sessions/targeted-first.jsonl");
|
||||||
|
const secondOriginalHash = getSessionHash("sessions/targeted-second.jsonl");
|
||||||
|
|
||||||
|
await fs.writeFile(
|
||||||
|
firstSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: {
|
||||||
|
role: "user",
|
||||||
|
content: [{ type: "text", text: "first transcript v2 after compaction" }],
|
||||||
|
},
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
await fs.writeFile(
|
||||||
|
secondSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: {
|
||||||
|
role: "user",
|
||||||
|
content: [{ type: "text", text: "second transcript v2 should stay untouched" }],
|
||||||
|
},
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await manager.sync?.({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: [firstSessionPath],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getSessionHash("sessions/targeted-first.jsonl")).not.toBe(firstOriginalHash);
|
||||||
|
expect(getSessionHash("sessions/targeted-second.jsonl")).toBe(secondOriginalHash);
|
||||||
|
await manager.close?.();
|
||||||
|
} finally {
|
||||||
|
if (previousStateDir === undefined) {
|
||||||
|
delete process.env.OPENCLAW_STATE_DIR;
|
||||||
|
} else {
|
||||||
|
process.env.OPENCLAW_STATE_DIR = previousStateDir;
|
||||||
|
}
|
||||||
|
await fs.rm(stateDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves unrelated dirty sessions after targeted post-compaction sync", async () => {
|
||||||
|
const stateDir = path.join(fixtureRoot, `state-targeted-dirty-${randomUUID()}`);
|
||||||
|
const sessionDir = path.join(stateDir, "agents", "main", "sessions");
|
||||||
|
const firstSessionPath = path.join(sessionDir, "targeted-dirty-first.jsonl");
|
||||||
|
const secondSessionPath = path.join(sessionDir, "targeted-dirty-second.jsonl");
|
||||||
|
const storePath = path.join(workspaceDir, `index-targeted-dirty-${randomUUID()}.sqlite`);
|
||||||
|
const previousStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||||
|
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||||
|
|
||||||
|
await fs.mkdir(sessionDir, { recursive: true });
|
||||||
|
await fs.writeFile(
|
||||||
|
firstSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: { role: "user", content: [{ type: "text", text: "first transcript v1" }] },
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
await fs.writeFile(
|
||||||
|
secondSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: { role: "user", content: [{ type: "text", text: "second transcript v1" }] },
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const manager = requireManager(
|
||||||
|
await getMemorySearchManager({
|
||||||
|
cfg: createCfg({
|
||||||
|
storePath,
|
||||||
|
sources: ["sessions"],
|
||||||
|
sessionMemory: true,
|
||||||
|
}),
|
||||||
|
agentId: "main",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await manager.sync({ reason: "test" });
|
||||||
|
|
||||||
|
const db = (
|
||||||
|
manager as unknown as {
|
||||||
|
db: {
|
||||||
|
prepare: (sql: string) => {
|
||||||
|
get: (path: string, source: string) => { hash: string } | undefined;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
).db;
|
||||||
|
const getSessionHash = (sessionPath: string) =>
|
||||||
|
db
|
||||||
|
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
||||||
|
.get(sessionPath, "sessions")?.hash;
|
||||||
|
|
||||||
|
const firstOriginalHash = getSessionHash("sessions/targeted-dirty-first.jsonl");
|
||||||
|
const secondOriginalHash = getSessionHash("sessions/targeted-dirty-second.jsonl");
|
||||||
|
|
||||||
|
await fs.writeFile(
|
||||||
|
firstSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: {
|
||||||
|
role: "user",
|
||||||
|
content: [{ type: "text", text: "first transcript v2 after compaction" }],
|
||||||
|
},
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
await fs.writeFile(
|
||||||
|
secondSessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: {
|
||||||
|
role: "user",
|
||||||
|
content: [{ type: "text", text: "second transcript v2 still pending" }],
|
||||||
|
},
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const internal = manager as unknown as {
|
||||||
|
sessionsDirty: boolean;
|
||||||
|
sessionsDirtyFiles: Set<string>;
|
||||||
|
};
|
||||||
|
internal.sessionsDirty = true;
|
||||||
|
internal.sessionsDirtyFiles.add(secondSessionPath);
|
||||||
|
|
||||||
|
await manager.sync({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: [firstSessionPath],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getSessionHash("sessions/targeted-dirty-first.jsonl")).not.toBe(firstOriginalHash);
|
||||||
|
expect(getSessionHash("sessions/targeted-dirty-second.jsonl")).toBe(secondOriginalHash);
|
||||||
|
expect(internal.sessionsDirtyFiles.has(secondSessionPath)).toBe(true);
|
||||||
|
expect(internal.sessionsDirty).toBe(true);
|
||||||
|
|
||||||
|
await manager.sync({ reason: "test" });
|
||||||
|
|
||||||
|
expect(getSessionHash("sessions/targeted-dirty-second.jsonl")).not.toBe(secondOriginalHash);
|
||||||
|
await manager.close?.();
|
||||||
|
} finally {
|
||||||
|
if (previousStateDir === undefined) {
|
||||||
|
delete process.env.OPENCLAW_STATE_DIR;
|
||||||
|
} else {
|
||||||
|
process.env.OPENCLAW_STATE_DIR = previousStateDir;
|
||||||
|
}
|
||||||
|
await fs.rm(stateDir, { recursive: true, force: true });
|
||||||
|
await fs.rm(storePath, { force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("queues targeted session sync when another sync is already in progress", async () => {
|
||||||
|
const stateDir = path.join(fixtureRoot, `state-targeted-queued-${randomUUID()}`);
|
||||||
|
const sessionDir = path.join(stateDir, "agents", "main", "sessions");
|
||||||
|
const sessionPath = path.join(sessionDir, "targeted-queued.jsonl");
|
||||||
|
const storePath = path.join(workspaceDir, `index-targeted-queued-${randomUUID()}.sqlite`);
|
||||||
|
const previousStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||||
|
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||||
|
|
||||||
|
await fs.mkdir(sessionDir, { recursive: true });
|
||||||
|
await fs.writeFile(
|
||||||
|
sessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: { role: "user", content: [{ type: "text", text: "queued transcript v1" }] },
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const manager = requireManager(
|
||||||
|
await getMemorySearchManager({
|
||||||
|
cfg: createCfg({
|
||||||
|
storePath,
|
||||||
|
sources: ["sessions"],
|
||||||
|
sessionMemory: true,
|
||||||
|
}),
|
||||||
|
agentId: "main",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await manager.sync({ reason: "test" });
|
||||||
|
|
||||||
|
const db = (
|
||||||
|
manager as unknown as {
|
||||||
|
db: {
|
||||||
|
prepare: (sql: string) => {
|
||||||
|
get: (path: string, source: string) => { hash: string } | undefined;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
).db;
|
||||||
|
const getSessionHash = (sessionRelPath: string) =>
|
||||||
|
db
|
||||||
|
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
||||||
|
.get(sessionRelPath, "sessions")?.hash;
|
||||||
|
const originalHash = getSessionHash("sessions/targeted-queued.jsonl");
|
||||||
|
|
||||||
|
const internal = manager as unknown as {
|
||||||
|
runSyncWithReadonlyRecovery: (params?: {
|
||||||
|
reason?: string;
|
||||||
|
sessionFiles?: string[];
|
||||||
|
}) => Promise<void>;
|
||||||
|
};
|
||||||
|
const originalRunSync = internal.runSyncWithReadonlyRecovery.bind(manager);
|
||||||
|
let releaseBusySync: (() => void) | undefined;
|
||||||
|
const busyGate = new Promise<void>((resolve) => {
|
||||||
|
releaseBusySync = resolve;
|
||||||
|
});
|
||||||
|
internal.runSyncWithReadonlyRecovery = async (params) => {
|
||||||
|
if (params?.reason === "busy-sync") {
|
||||||
|
await busyGate;
|
||||||
|
}
|
||||||
|
return await originalRunSync(params);
|
||||||
|
};
|
||||||
|
|
||||||
|
const busySyncPromise = manager.sync({ reason: "busy-sync" });
|
||||||
|
await fs.writeFile(
|
||||||
|
sessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: {
|
||||||
|
role: "user",
|
||||||
|
content: [{ type: "text", text: "queued transcript v2 after compaction" }],
|
||||||
|
},
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const targetedSyncPromise = manager.sync({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: [sessionPath],
|
||||||
|
});
|
||||||
|
|
||||||
|
releaseBusySync?.();
|
||||||
|
await Promise.all([busySyncPromise, targetedSyncPromise]);
|
||||||
|
|
||||||
|
expect(getSessionHash("sessions/targeted-queued.jsonl")).not.toBe(originalHash);
|
||||||
|
await manager.close?.();
|
||||||
|
} finally {
|
||||||
|
if (previousStateDir === undefined) {
|
||||||
|
delete process.env.OPENCLAW_STATE_DIR;
|
||||||
|
} else {
|
||||||
|
process.env.OPENCLAW_STATE_DIR = previousStateDir;
|
||||||
|
}
|
||||||
|
await fs.rm(stateDir, { recursive: true, force: true });
|
||||||
|
await fs.rm(storePath, { force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("runs a full reindex after fallback activates during targeted sync", async () => {
|
||||||
|
const stateDir = path.join(fixtureRoot, `state-targeted-fallback-${randomUUID()}`);
|
||||||
|
const sessionDir = path.join(stateDir, "agents", "main", "sessions");
|
||||||
|
const sessionPath = path.join(sessionDir, "targeted-fallback.jsonl");
|
||||||
|
const storePath = path.join(workspaceDir, `index-targeted-fallback-${randomUUID()}.sqlite`);
|
||||||
|
const previousStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||||
|
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||||
|
|
||||||
|
await fs.mkdir(sessionDir, { recursive: true });
|
||||||
|
await fs.writeFile(
|
||||||
|
sessionPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "message",
|
||||||
|
message: { role: "user", content: [{ type: "text", text: "fallback transcript v1" }] },
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const manager = requireManager(
|
||||||
|
await getMemorySearchManager({
|
||||||
|
cfg: createCfg({
|
||||||
|
storePath,
|
||||||
|
sources: ["sessions"],
|
||||||
|
sessionMemory: true,
|
||||||
|
}),
|
||||||
|
agentId: "main",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await manager.sync({ reason: "test" });
|
||||||
|
|
||||||
|
const internal = manager as unknown as {
|
||||||
|
syncSessionFiles: (params: {
|
||||||
|
targetSessionFiles?: string[];
|
||||||
|
needsFullReindex: boolean;
|
||||||
|
}) => Promise<void>;
|
||||||
|
shouldFallbackOnError: (message: string) => boolean;
|
||||||
|
activateFallbackProvider: (reason: string) => Promise<boolean>;
|
||||||
|
runUnsafeReindex: (params: {
|
||||||
|
reason?: string;
|
||||||
|
force?: boolean;
|
||||||
|
progress?: unknown;
|
||||||
|
}) => Promise<void>;
|
||||||
|
};
|
||||||
|
const originalSyncSessionFiles = internal.syncSessionFiles.bind(manager);
|
||||||
|
const originalShouldFallbackOnError = internal.shouldFallbackOnError.bind(manager);
|
||||||
|
const originalActivateFallbackProvider = internal.activateFallbackProvider.bind(manager);
|
||||||
|
const originalRunUnsafeReindex = internal.runUnsafeReindex.bind(manager);
|
||||||
|
|
||||||
|
internal.syncSessionFiles = async (params) => {
|
||||||
|
if (params.targetSessionFiles?.length) {
|
||||||
|
throw new Error("embedding backend failed");
|
||||||
|
}
|
||||||
|
return await originalSyncSessionFiles(params);
|
||||||
|
};
|
||||||
|
internal.shouldFallbackOnError = () => true;
|
||||||
|
const activateFallbackProvider = vi.fn(async () => true);
|
||||||
|
internal.activateFallbackProvider = activateFallbackProvider;
|
||||||
|
const runUnsafeReindex = vi.fn(async () => {});
|
||||||
|
internal.runUnsafeReindex = runUnsafeReindex;
|
||||||
|
|
||||||
|
await manager.sync({
|
||||||
|
reason: "post-compaction",
|
||||||
|
sessionFiles: [sessionPath],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(activateFallbackProvider).toHaveBeenCalledWith("embedding backend failed");
|
||||||
|
expect(runUnsafeReindex).toHaveBeenCalledWith({
|
||||||
|
reason: "post-compaction",
|
||||||
|
force: true,
|
||||||
|
progress: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
internal.syncSessionFiles = originalSyncSessionFiles;
|
||||||
|
internal.shouldFallbackOnError = originalShouldFallbackOnError;
|
||||||
|
internal.activateFallbackProvider = originalActivateFallbackProvider;
|
||||||
|
internal.runUnsafeReindex = originalRunUnsafeReindex;
|
||||||
|
await manager.close?.();
|
||||||
|
} finally {
|
||||||
|
if (previousStateDir === undefined) {
|
||||||
|
delete process.env.OPENCLAW_STATE_DIR;
|
||||||
|
} else {
|
||||||
|
process.env.OPENCLAW_STATE_DIR = previousStateDir;
|
||||||
|
}
|
||||||
|
await fs.rm(stateDir, { recursive: true, force: true });
|
||||||
|
await fs.rm(storePath, { force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("reindexes when the embedding model changes", async () => {
|
it("reindexes when the embedding model changes", async () => {
|
||||||
const base = createCfg({ storePath: indexModelPath });
|
const base = createCfg({ storePath: indexModelPath });
|
||||||
const baseAgents = base.agents!;
|
const baseAgents = base.agents!;
|
||||||
|
|||||||
@@ -151,6 +151,8 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
protected abstract sync(params?: {
|
protected abstract sync(params?: {
|
||||||
reason?: string;
|
reason?: string;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
forceSessions?: boolean;
|
||||||
|
sessionFile?: string;
|
||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}): Promise<void>;
|
}): Promise<void>;
|
||||||
protected abstract withTimeout<T>(
|
protected abstract withTimeout<T>(
|
||||||
@@ -611,6 +613,35 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
return resolvedFile.startsWith(`${resolvedDir}${path.sep}`);
|
return resolvedFile.startsWith(`${resolvedDir}${path.sep}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private normalizeTargetSessionFiles(sessionFiles?: string[]): Set<string> | null {
|
||||||
|
if (!sessionFiles || sessionFiles.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const normalized = new Set<string>();
|
||||||
|
for (const sessionFile of sessionFiles) {
|
||||||
|
const trimmed = sessionFile.trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const resolved = path.resolve(trimmed);
|
||||||
|
if (this.isSessionFileForAgent(resolved)) {
|
||||||
|
normalized.add(resolved);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return normalized.size > 0 ? normalized : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearSyncedSessionFiles(targetSessionFiles?: Iterable<string> | null) {
|
||||||
|
if (!targetSessionFiles) {
|
||||||
|
this.sessionsDirtyFiles.clear();
|
||||||
|
} else {
|
||||||
|
for (const targetSessionFile of targetSessionFiles) {
|
||||||
|
this.sessionsDirtyFiles.delete(targetSessionFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.sessionsDirty = this.sessionsDirtyFiles.size > 0;
|
||||||
|
}
|
||||||
|
|
||||||
protected ensureIntervalSync() {
|
protected ensureIntervalSync() {
|
||||||
const minutes = this.settings.sync.intervalMinutes;
|
const minutes = this.settings.sync.intervalMinutes;
|
||||||
if (!minutes || minutes <= 0 || this.intervalTimer) {
|
if (!minutes || minutes <= 0 || this.intervalTimer) {
|
||||||
@@ -640,12 +671,15 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private shouldSyncSessions(
|
private shouldSyncSessions(
|
||||||
params?: { reason?: string; force?: boolean },
|
params?: { reason?: string; force?: boolean; sessionFiles?: string[] },
|
||||||
needsFullReindex = false,
|
needsFullReindex = false,
|
||||||
) {
|
) {
|
||||||
if (!this.sources.has("sessions")) {
|
if (!this.sources.has("sessions")) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if (params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if (params?.force) {
|
if (params?.force) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -752,6 +786,7 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
|
|
||||||
private async syncSessionFiles(params: {
|
private async syncSessionFiles(params: {
|
||||||
needsFullReindex: boolean;
|
needsFullReindex: boolean;
|
||||||
|
targetSessionFiles?: string[];
|
||||||
progress?: MemorySyncProgressState;
|
progress?: MemorySyncProgressState;
|
||||||
}) {
|
}) {
|
||||||
// FTS-only mode: skip embedding sync (no provider)
|
// FTS-only mode: skip embedding sync (no provider)
|
||||||
@@ -760,13 +795,22 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const files = await listSessionFilesForAgent(this.agentId);
|
const targetSessionFiles = params.needsFullReindex
|
||||||
const activePaths = new Set(files.map((file) => sessionPathForFile(file)));
|
? null
|
||||||
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
|
: this.normalizeTargetSessionFiles(params.targetSessionFiles);
|
||||||
|
const files = targetSessionFiles
|
||||||
|
? Array.from(targetSessionFiles)
|
||||||
|
: await listSessionFilesForAgent(this.agentId);
|
||||||
|
const activePaths = targetSessionFiles
|
||||||
|
? null
|
||||||
|
: new Set(files.map((file) => sessionPathForFile(file)));
|
||||||
|
const indexAll =
|
||||||
|
params.needsFullReindex || Boolean(targetSessionFiles) || this.sessionsDirtyFiles.size === 0;
|
||||||
log.debug("memory sync: indexing session files", {
|
log.debug("memory sync: indexing session files", {
|
||||||
files: files.length,
|
files: files.length,
|
||||||
indexAll,
|
indexAll,
|
||||||
dirtyFiles: this.sessionsDirtyFiles.size,
|
dirtyFiles: this.sessionsDirtyFiles.size,
|
||||||
|
targetedFiles: targetSessionFiles?.size ?? 0,
|
||||||
batch: this.batch.enabled,
|
batch: this.batch.enabled,
|
||||||
concurrency: this.getIndexConcurrency(),
|
concurrency: this.getIndexConcurrency(),
|
||||||
});
|
});
|
||||||
@@ -827,6 +871,12 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
});
|
});
|
||||||
await runWithConcurrency(tasks, this.getIndexConcurrency());
|
await runWithConcurrency(tasks, this.getIndexConcurrency());
|
||||||
|
|
||||||
|
if (activePaths === null) {
|
||||||
|
// Targeted syncs only refresh the requested transcripts and should not
|
||||||
|
// prune unrelated session rows without a full directory enumeration.
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const staleRows = this.db
|
const staleRows = this.db
|
||||||
.prepare(`SELECT path FROM files WHERE source = ?`)
|
.prepare(`SELECT path FROM files WHERE source = ?`)
|
||||||
.all("sessions") as Array<{ path: string }>;
|
.all("sessions") as Array<{ path: string }>;
|
||||||
@@ -885,6 +935,7 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
protected async runSync(params?: {
|
protected async runSync(params?: {
|
||||||
reason?: string;
|
reason?: string;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
sessionFiles?: string[];
|
||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}) {
|
}) {
|
||||||
const progress = params?.progress ? this.createSyncProgress(params.progress) : undefined;
|
const progress = params?.progress ? this.createSyncProgress(params.progress) : undefined;
|
||||||
@@ -899,8 +950,47 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
const meta = this.readMeta();
|
const meta = this.readMeta();
|
||||||
const configuredSources = this.resolveConfiguredSourcesForMeta();
|
const configuredSources = this.resolveConfiguredSourcesForMeta();
|
||||||
const configuredScopeHash = this.resolveConfiguredScopeHash();
|
const configuredScopeHash = this.resolveConfiguredScopeHash();
|
||||||
|
const targetSessionFiles = this.normalizeTargetSessionFiles(params?.sessionFiles);
|
||||||
|
const hasTargetSessionFiles = targetSessionFiles !== null;
|
||||||
|
if (hasTargetSessionFiles && targetSessionFiles && this.sources.has("sessions")) {
|
||||||
|
// Post-compaction refreshes should only update the explicit transcript files and
|
||||||
|
// leave broader reindex/dirty-work decisions to the regular sync path.
|
||||||
|
try {
|
||||||
|
await this.syncSessionFiles({
|
||||||
|
needsFullReindex: false,
|
||||||
|
targetSessionFiles: Array.from(targetSessionFiles),
|
||||||
|
progress: progress ?? undefined,
|
||||||
|
});
|
||||||
|
this.clearSyncedSessionFiles(targetSessionFiles);
|
||||||
|
} catch (err) {
|
||||||
|
const reason = err instanceof Error ? err.message : String(err);
|
||||||
|
const activated =
|
||||||
|
this.shouldFallbackOnError(reason) && (await this.activateFallbackProvider(reason));
|
||||||
|
if (activated) {
|
||||||
|
if (
|
||||||
|
process.env.OPENCLAW_TEST_FAST === "1" &&
|
||||||
|
process.env.OPENCLAW_TEST_MEMORY_UNSAFE_REINDEX === "1"
|
||||||
|
) {
|
||||||
|
await this.runUnsafeReindex({
|
||||||
|
reason: params?.reason,
|
||||||
|
force: true,
|
||||||
|
progress: progress ?? undefined,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
await this.runSafeReindex({
|
||||||
|
reason: params?.reason,
|
||||||
|
force: true,
|
||||||
|
progress: progress ?? undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
const needsFullReindex =
|
const needsFullReindex =
|
||||||
params?.force ||
|
(params?.force && !hasTargetSessionFiles) ||
|
||||||
!meta ||
|
!meta ||
|
||||||
(this.provider && meta.model !== this.provider.model) ||
|
(this.provider && meta.model !== this.provider.model) ||
|
||||||
(this.provider && meta.provider !== this.provider.id) ||
|
(this.provider && meta.provider !== this.provider.id) ||
|
||||||
@@ -932,7 +1022,8 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const shouldSyncMemory =
|
const shouldSyncMemory =
|
||||||
this.sources.has("memory") && (params?.force || needsFullReindex || this.dirty);
|
this.sources.has("memory") &&
|
||||||
|
((!hasTargetSessionFiles && params?.force) || needsFullReindex || this.dirty);
|
||||||
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
|
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
|
||||||
|
|
||||||
if (shouldSyncMemory) {
|
if (shouldSyncMemory) {
|
||||||
@@ -941,7 +1032,11 @@ export abstract class MemoryManagerSyncOps {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (shouldSyncSessions) {
|
if (shouldSyncSessions) {
|
||||||
await this.syncSessionFiles({ needsFullReindex, progress: progress ?? undefined });
|
await this.syncSessionFiles({
|
||||||
|
needsFullReindex,
|
||||||
|
targetSessionFiles: targetSessionFiles ? Array.from(targetSessionFiles) : undefined,
|
||||||
|
progress: progress ?? undefined,
|
||||||
|
});
|
||||||
this.sessionsDirty = false;
|
this.sessionsDirty = false;
|
||||||
this.sessionsDirtyFiles.clear();
|
this.sessionsDirtyFiles.clear();
|
||||||
} else if (this.sessionsDirtyFiles.size > 0) {
|
} else if (this.sessionsDirtyFiles.size > 0) {
|
||||||
|
|||||||
@@ -125,6 +125,8 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
|||||||
>();
|
>();
|
||||||
private sessionWarm = new Set<string>();
|
private sessionWarm = new Set<string>();
|
||||||
private syncing: Promise<void> | null = null;
|
private syncing: Promise<void> | null = null;
|
||||||
|
private queuedSessionFiles = new Set<string>();
|
||||||
|
private queuedSessionSync: Promise<void> | null = null;
|
||||||
private readonlyRecoveryAttempts = 0;
|
private readonlyRecoveryAttempts = 0;
|
||||||
private readonlyRecoverySuccesses = 0;
|
private readonlyRecoverySuccesses = 0;
|
||||||
private readonlyRecoveryFailures = 0;
|
private readonlyRecoveryFailures = 0;
|
||||||
@@ -452,12 +454,16 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
|||||||
async sync(params?: {
|
async sync(params?: {
|
||||||
reason?: string;
|
reason?: string;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
sessionFiles?: string[];
|
||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
if (this.closed) {
|
if (this.closed) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (this.syncing) {
|
if (this.syncing) {
|
||||||
|
if (params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) {
|
||||||
|
return this.enqueueTargetedSessionSync(params.sessionFiles);
|
||||||
|
}
|
||||||
return this.syncing;
|
return this.syncing;
|
||||||
}
|
}
|
||||||
this.syncing = this.runSyncWithReadonlyRecovery(params).finally(() => {
|
this.syncing = this.runSyncWithReadonlyRecovery(params).finally(() => {
|
||||||
@@ -466,6 +472,36 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
|||||||
return this.syncing ?? Promise.resolve();
|
return this.syncing ?? Promise.resolve();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private enqueueTargetedSessionSync(sessionFiles?: string[]): Promise<void> {
|
||||||
|
for (const sessionFile of sessionFiles ?? []) {
|
||||||
|
const trimmed = sessionFile.trim();
|
||||||
|
if (trimmed) {
|
||||||
|
this.queuedSessionFiles.add(trimmed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.queuedSessionFiles.size === 0) {
|
||||||
|
return this.syncing ?? Promise.resolve();
|
||||||
|
}
|
||||||
|
if (!this.queuedSessionSync) {
|
||||||
|
this.queuedSessionSync = (async () => {
|
||||||
|
try {
|
||||||
|
await this.syncing?.catch(() => undefined);
|
||||||
|
while (!this.closed && this.queuedSessionFiles.size > 0) {
|
||||||
|
const queuedSessionFiles = Array.from(this.queuedSessionFiles);
|
||||||
|
this.queuedSessionFiles.clear();
|
||||||
|
await this.sync({
|
||||||
|
reason: "queued-session-files",
|
||||||
|
sessionFiles: queuedSessionFiles,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
this.queuedSessionSync = null;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
return this.queuedSessionSync;
|
||||||
|
}
|
||||||
|
|
||||||
private isReadonlyDbError(err: unknown): boolean {
|
private isReadonlyDbError(err: unknown): boolean {
|
||||||
const readonlyPattern =
|
const readonlyPattern =
|
||||||
/attempt to write a readonly database|database is read-only|SQLITE_READONLY/i;
|
/attempt to write a readonly database|database is read-only|SQLITE_READONLY/i;
|
||||||
@@ -518,6 +554,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
|||||||
private async runSyncWithReadonlyRecovery(params?: {
|
private async runSyncWithReadonlyRecovery(params?: {
|
||||||
reason?: string;
|
reason?: string;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
sessionFiles?: string[];
|
||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -867,8 +867,12 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
async sync(params?: {
|
async sync(params?: {
|
||||||
reason?: string;
|
reason?: string;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
sessionFiles?: string[];
|
||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
|
if (params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) {
|
||||||
|
log.debug("qmd sync ignoring targeted sessionFiles hint; running regular update");
|
||||||
|
}
|
||||||
if (params?.progress) {
|
if (params?.progress) {
|
||||||
params.progress({ completed: 0, total: 1, label: "Updating QMD index…" });
|
params.progress({ completed: 0, total: 1, label: "Updating QMD index…" });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -181,6 +181,7 @@ class FallbackMemoryManager implements MemorySearchManager {
|
|||||||
async sync(params?: {
|
async sync(params?: {
|
||||||
reason?: string;
|
reason?: string;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
sessionFiles?: string[];
|
||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}) {
|
}) {
|
||||||
if (!this.primaryFailed) {
|
if (!this.primaryFailed) {
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ export interface MemorySearchManager {
|
|||||||
sync?(params?: {
|
sync?(params?: {
|
||||||
reason?: string;
|
reason?: string;
|
||||||
force?: boolean;
|
force?: boolean;
|
||||||
|
sessionFiles?: string[];
|
||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}): Promise<void>;
|
}): Promise<void>;
|
||||||
probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult>;
|
probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult>;
|
||||||
|
|||||||
Reference in New Issue
Block a user