diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index a6c3ef28401..66194ef5e0e 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -28,6 +28,19 @@ The default workspace layout uses two memory layers: These files live under the workspace (`agents.defaults.workspace`, default `~/.openclaw/workspace`). See [Agent workspace](/concepts/agent-workspace) for the full layout. +## Memory tools + +OpenClaw exposes two agent-facing tools for these Markdown files: + +- `memory_search` — semantic recall over indexed snippets. +- `memory_get` — targeted read of a specific Markdown file/line range. + +`memory_get` now **degrades gracefully when a file doesn't exist** (for example, +today's daily log before the first write). Both the builtin manager and the QMD +backend return `{ text: "", path }` instead of throwing `ENOENT`, so agents can +handle "nothing recorded yet" and continue their workflow without wrapping the +tool call in try/catch logic. + ## When to write memory - Decisions, preferences, and durable facts go to `MEMORY.md`. diff --git a/src/memory/fs-utils.ts b/src/memory/fs-utils.ts new file mode 100644 index 00000000000..81107c7ef3d --- /dev/null +++ b/src/memory/fs-utils.ts @@ -0,0 +1,31 @@ +import type { Stats } from "node:fs"; +import fs from "node:fs/promises"; + +export type RegularFileStatResult = { missing: true } | { missing: false; stat: Stats }; + +export function isFileMissingError( + err: unknown, +): err is NodeJS.ErrnoException & { code: "ENOENT" } { + return Boolean( + err && + typeof err === "object" && + "code" in err && + (err as Partial).code === "ENOENT", + ); +} + +export async function statRegularFile(absPath: string): Promise { + let stat: Stats; + try { + stat = await fs.lstat(absPath); + } catch (err) { + if (isFileMissingError(err)) { + return { missing: true }; + } + throw err; + } + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error("path required"); + } + return { missing: false, stat }; +} diff --git a/src/memory/internal.test.ts b/src/memory/internal.test.ts index 6c0e55f4bb4..0c3b199ca51 100644 --- a/src/memory/internal.test.ts +++ b/src/memory/internal.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { + buildFileEntry, chunkMarkdown, listMemoryFiles, normalizeExtraMemoryPaths, @@ -116,6 +117,35 @@ describe("listMemoryFiles", () => { }); }); +describe("buildFileEntry", () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-build-entry-")); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it("returns null when the file disappears before reading", async () => { + const target = path.join(tmpDir, "ghost.md"); + await fs.writeFile(target, "ghost", "utf-8"); + await fs.rm(target); + const entry = await buildFileEntry(target, tmpDir); + expect(entry).toBeNull(); + }); + + it("returns metadata when the file exists", async () => { + const target = path.join(tmpDir, "note.md"); + await fs.writeFile(target, "hello", "utf-8"); + const entry = await buildFileEntry(target, tmpDir); + expect(entry).not.toBeNull(); + expect(entry?.path).toBe("note.md"); + expect(entry?.size).toBeGreaterThan(0); + }); +}); + describe("chunkMarkdown", () => { it("splits overly long lines into max-sized chunks", () => { const chunkTokens = 400; diff --git a/src/memory/internal.ts b/src/memory/internal.ts index 04afeb8c8a8..d39e355d2c0 100644 --- a/src/memory/internal.ts +++ b/src/memory/internal.ts @@ -3,6 +3,7 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js"; +import { isFileMissingError } from "./fs-utils.js"; export type MemoryFileEntry = { path: string; @@ -151,9 +152,25 @@ export function hashText(value: string): string { export async function buildFileEntry( absPath: string, workspaceDir: string, -): Promise { - const stat = await fs.stat(absPath); - const content = await fs.readFile(absPath, "utf-8"); +): Promise { + let stat; + try { + stat = await fs.stat(absPath); + } catch (err) { + if (isFileMissingError(err)) { + return null; + } + throw err; + } + let content: string; + try { + content = await fs.readFile(absPath, "utf-8"); + } catch (err) { + if (isFileMissingError(err)) { + return null; + } + throw err; + } const hash = hashText(content); return { path: path.relative(workspaceDir, absPath).replace(/\\/g, "/"), diff --git a/src/memory/manager-sync-ops.ts b/src/memory/manager-sync-ops.ts index c5c1d71a2e5..61b479b3d29 100644 --- a/src/memory/manager-sync-ops.ts +++ b/src/memory/manager-sync-ops.ts @@ -21,6 +21,7 @@ import { type OpenAiEmbeddingClient, type VoyageEmbeddingClient, } from "./embeddings.js"; +import { isFileMissingError } from "./fs-utils.js"; import { buildFileEntry, ensureDir, @@ -522,7 +523,15 @@ export abstract class MemoryManagerSyncOps { if (end <= start) { return 0; } - const handle = await fs.open(absPath, "r"); + let handle; + try { + handle = await fs.open(absPath, "r"); + } catch (err) { + if (isFileMissingError(err)) { + return 0; + } + throw err; + } try { let offset = start; let count = 0; @@ -625,9 +634,9 @@ export abstract class MemoryManagerSyncOps { } const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths); - const fileEntries = await Promise.all( - files.map(async (file) => buildFileEntry(file, this.workspaceDir)), - ); + const fileEntries = ( + await Promise.all(files.map(async (file) => buildFileEntry(file, this.workspaceDir))) + ).filter((entry): entry is MemoryFileEntry => entry !== null); log.debug("memory sync: indexing memory files", { files: fileEntries.length, needsFullReindex: params.needsFullReindex, diff --git a/src/memory/manager.read-file.test.ts b/src/memory/manager.read-file.test.ts index da9ce370c91..b5d2e6c18ad 100644 --- a/src/memory/manager.read-file.test.ts +++ b/src/memory/manager.read-file.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { resetEmbeddingMocks } from "./embedding.test-mocks.js"; import type { MemoryIndexManager } from "./index.js"; @@ -74,4 +74,51 @@ describe("MemoryIndexManager.readFile", () => { const result = await manager.readFile({ relPath, from: 2, lines: 1 }); expect(result).toEqual({ text: "line 2", path: relPath }); }); + + it("returns empty text when the requested slice is past EOF", async () => { + const relPath = "memory/window.md"; + const absPath = path.join(workspaceDir, relPath); + await fs.mkdir(path.dirname(absPath), { recursive: true }); + await fs.writeFile(absPath, ["alpha", "beta"].join("\n"), "utf-8"); + + manager = await getRequiredMemoryIndexManager({ + cfg: createMemorySearchCfg({ workspaceDir, indexPath }), + agentId: "main", + }); + + const result = await manager.readFile({ relPath, from: 10, lines: 5 }); + expect(result).toEqual({ text: "", path: relPath }); + }); + + it("returns empty text when the file disappears after stat", async () => { + const relPath = "memory/transient.md"; + const absPath = path.join(workspaceDir, relPath); + await fs.mkdir(path.dirname(absPath), { recursive: true }); + await fs.writeFile(absPath, "first\nsecond", "utf-8"); + + manager = await getRequiredMemoryIndexManager({ + cfg: createMemorySearchCfg({ workspaceDir, indexPath }), + agentId: "main", + }); + + const realReadFile = fs.readFile; + let injected = false; + const readSpy = vi + .spyOn(fs, "readFile") + .mockImplementation(async (...args: Parameters) => { + const [target, options] = args; + if (!injected && typeof target === "string" && path.resolve(target) === absPath) { + injected = true; + const err = new Error("missing") as NodeJS.ErrnoException; + err.code = "ENOENT"; + throw err; + } + return realReadFile(target, options); + }); + + const result = await manager.readFile({ relPath }); + expect(result).toEqual({ text: "", path: relPath }); + + readSpy.mockRestore(); + }); }); diff --git a/src/memory/manager.ts b/src/memory/manager.ts index bfada289940..1082e5ee52a 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -1,4 +1,3 @@ -import type { Stats } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; @@ -16,6 +15,7 @@ import { type OpenAiEmbeddingClient, type VoyageEmbeddingClient, } from "./embeddings.js"; +import { isFileMissingError, statRegularFile } from "./fs-utils.js"; import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js"; import { isMemoryPath, normalizeExtraMemoryPaths } from "./internal.js"; import { MemoryManagerEmbeddingOps } from "./manager-embedding-ops.js"; @@ -37,15 +37,6 @@ const BATCH_FAILURE_LIMIT = 2; const log = createSubsystemLogger("memory"); -function isFileMissingError(err: unknown): err is NodeJS.ErrnoException & { code: "ENOENT" } { - return Boolean( - err && - typeof err === "object" && - "code" in err && - (err as Partial).code === "ENOENT", - ); -} - const INDEX_CACHE = new Map(); export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements MemorySearchManager { @@ -447,17 +438,9 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem if (!absPath.endsWith(".md")) { throw new Error("path required"); } - let stat: Stats; - try { - stat = await fs.lstat(absPath); - } catch (err) { - if (isFileMissingError(err)) { - return { text: "", path: relPath }; - } - throw err; - } - if (stat.isSymbolicLink() || !stat.isFile()) { - throw new Error("path required"); + const statResult = await statRegularFile(absPath); + if (statResult.missing) { + return { text: "", path: relPath }; } let content: string; try { diff --git a/src/memory/manager.vector-dedupe.test.ts b/src/memory/manager.vector-dedupe.test.ts index 14f2e2f8f3f..699f6c67ec4 100644 --- a/src/memory/manager.vector-dedupe.test.ts +++ b/src/memory/manager.vector-dedupe.test.ts @@ -81,6 +81,9 @@ describe("memory vector dedupe", () => { ).ensureVectorReady = async () => true; const entry = await buildFileEntry(path.join(workspaceDir, "MEMORY.md"), workspaceDir); + if (!entry) { + throw new Error("entry missing"); + } await ( manager as unknown as { indexFile: (entry: unknown, options: { source: "memory" }) => Promise; diff --git a/src/memory/qmd-manager.test.ts b/src/memory/qmd-manager.test.ts index 8ff974362aa..d38c569bdfb 100644 --- a/src/memory/qmd-manager.test.ts +++ b/src/memory/qmd-manager.test.ts @@ -1079,6 +1079,42 @@ describe("QmdMemoryManager", () => { readFileSpy.mockRestore(); }); + it("returns empty text when a qmd workspace file does not exist", async () => { + const { manager } = await createManager(); + const result = await manager.readFile({ relPath: "ghost.md" }); + expect(result).toEqual({ text: "", path: "ghost.md" }); + await manager.close(); + }); + + it("returns empty text when a qmd file disappears before partial read", async () => { + const relPath = "qmd-window.md"; + const absPath = path.join(workspaceDir, relPath); + await fs.writeFile(absPath, "one\ntwo\nthree", "utf-8"); + + const { manager } = await createManager(); + + const realOpen = fs.open; + let injected = false; + const openSpy = vi + .spyOn(fs, "open") + .mockImplementation(async (...args: Parameters) => { + const [target, options] = args; + if (!injected && typeof target === "string" && path.resolve(target) === absPath) { + injected = true; + const err = new Error("gone") as NodeJS.ErrnoException; + err.code = "ENOENT"; + throw err; + } + return realOpen(target, options); + }); + + const result = await manager.readFile({ relPath, from: 2, lines: 1 }); + expect(result).toEqual({ text: "", path: relPath }); + + openSpy.mockRestore(); + await manager.close(); + }); + it("reuses exported session markdown files when inputs are unchanged", async () => { const writeFileSpy = vi.spyOn(fs, "writeFile"); const sessionsDir = path.join(stateDir, "agents", agentId, "sessions"); diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 380f4175c99..342b93ad45f 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -7,6 +7,7 @@ import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import type { OpenClawConfig } from "../config/config.js"; import { resolveStateDir } from "../config/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; +import { isFileMissingError, statRegularFile } from "./fs-utils.js"; import { deriveQmdScopeChannel, deriveQmdScopeChatType, isQmdScopeAllowed } from "./qmd-scope.js"; import { listSessionFilesForAgent, @@ -493,19 +494,25 @@ export class QmdMemoryManager implements MemorySearchManager { if (!absPath.endsWith(".md")) { throw new Error("path required"); } - const stat = await fs.lstat(absPath); - if (stat.isSymbolicLink() || !stat.isFile()) { - throw new Error("path required"); + const statResult = await statRegularFile(absPath); + if (statResult.missing) { + return { text: "", path: relPath }; } if (params.from !== undefined || params.lines !== undefined) { - const text = await this.readPartialText(absPath, params.from, params.lines); - return { text, path: relPath }; + const partial = await this.readPartialText(absPath, params.from, params.lines); + if (partial.missing) { + return { text: "", path: relPath }; + } + return { text: partial.text, path: relPath }; + } + const full = await this.readFullText(absPath); + if (full.missing) { + return { text: "", path: relPath }; } - const content = await fs.readFile(absPath, "utf-8"); if (!params.from && !params.lines) { - return { text: content, path: relPath }; + return { text: full.text, path: relPath }; } - const lines = content.split("\n"); + const lines = full.text.split("\n"); const start = Math.max(1, params.from ?? 1); const count = Math.max(1, params.lines ?? lines.length); const slice = lines.slice(start - 1, start - 1 + count); @@ -764,10 +771,22 @@ export class QmdMemoryManager implements MemorySearchManager { }); } - private async readPartialText(absPath: string, from?: number, lines?: number): Promise { + private async readPartialText( + absPath: string, + from?: number, + lines?: number, + ): Promise<{ missing: true } | { missing: false; text: string }> { const start = Math.max(1, from ?? 1); const count = Math.max(1, lines ?? Number.POSITIVE_INFINITY); - const handle = await fs.open(absPath); + let handle; + try { + handle = await fs.open(absPath); + } catch (err) { + if (isFileMissingError(err)) { + return { missing: true }; + } + throw err; + } const stream = handle.createReadStream({ encoding: "utf-8" }); const rl = readline.createInterface({ input: stream, @@ -790,7 +809,21 @@ export class QmdMemoryManager implements MemorySearchManager { rl.close(); await handle.close(); } - return selected.slice(0, count).join("\n"); + return { missing: false, text: selected.slice(0, count).join("\n") }; + } + + private async readFullText( + absPath: string, + ): Promise<{ missing: true } | { missing: false; text: string }> { + try { + const text = await fs.readFile(absPath, "utf-8"); + return { missing: false, text }; + } catch (err) { + if (isFileMissingError(err)) { + return { missing: true }; + } + throw err; + } } private ensureDb(): SqliteDatabase { diff --git a/src/memory/sync-memory-files.ts b/src/memory/sync-memory-files.ts index 182bae6d0dd..ac081839a97 100644 --- a/src/memory/sync-memory-files.ts +++ b/src/memory/sync-memory-files.ts @@ -25,9 +25,9 @@ export async function syncMemoryFiles(params: { model: string; }) { const files = await listMemoryFiles(params.workspaceDir, params.extraPaths); - const fileEntries = await Promise.all( - files.map(async (file) => buildFileEntry(file, params.workspaceDir)), - ); + const fileEntries = ( + await Promise.all(files.map(async (file) => buildFileEntry(file, params.workspaceDir))) + ).filter((entry): entry is MemoryFileEntry => entry !== null); log.debug("memory sync: indexing memory files", { files: fileEntries.length,