mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-09 19:14:33 +00:00
Memory/QMD: optimize qmd readFile for line-window reads
This commit is contained in:
@@ -71,6 +71,7 @@ Docs: https://docs.openclaw.ai
|
|||||||
- Memory/QMD: query QMD index using exact docid matches before falling back to prefix lookup for better recall correctness and index efficiency.
|
- Memory/QMD: query QMD index using exact docid matches before falling back to prefix lookup for better recall correctness and index efficiency.
|
||||||
- Memory/QMD: make QMD result JSON parsing resilient to noisy command output by extracting the first JSON array from noisy `stdout`.
|
- Memory/QMD: make QMD result JSON parsing resilient to noisy command output by extracting the first JSON array from noisy `stdout`.
|
||||||
- Memory/QMD: pass result limits to `search`/`vsearch` commands so QMD can cap results earlier.
|
- Memory/QMD: pass result limits to `search`/`vsearch` commands so QMD can cap results earlier.
|
||||||
|
- Memory/QMD: avoid reading full markdown files when a `from/lines` window is requested in QMD reads.
|
||||||
- Models/CLI: guard `models status` string trimming paths to prevent crashes from malformed non-string config values. (#16395) Thanks @BinHPdev.
|
- Models/CLI: guard `models status` string trimming paths to prevent crashes from malformed non-string config values. (#16395) Thanks @BinHPdev.
|
||||||
|
|
||||||
## 2026.2.14
|
## 2026.2.14
|
||||||
|
|||||||
@@ -789,6 +789,26 @@ describe("QmdMemoryManager", () => {
|
|||||||
await manager.close();
|
await manager.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reads only requested line ranges without loading the whole file", async () => {
|
||||||
|
const readFileSpy = vi.spyOn(fs, "readFile");
|
||||||
|
const text = Array.from({ length: 50 }, (_, index) => `line-${index + 1}`).join("\n");
|
||||||
|
await fs.writeFile(path.join(workspaceDir, "window.md"), text, "utf-8");
|
||||||
|
|
||||||
|
const resolved = resolveMemoryBackendConfig({ cfg, agentId });
|
||||||
|
const manager = await QmdMemoryManager.create({ cfg, agentId, resolved });
|
||||||
|
expect(manager).toBeTruthy();
|
||||||
|
if (!manager) {
|
||||||
|
throw new Error("manager missing");
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await manager.readFile({ relPath: "window.md", from: 10, lines: 3 });
|
||||||
|
expect(result.text).toBe("line-10\nline-11\nline-12");
|
||||||
|
expect(readFileSpy).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
await manager.close();
|
||||||
|
readFileSpy.mockRestore();
|
||||||
|
});
|
||||||
|
|
||||||
it("throws when sqlite index is busy", async () => {
|
it("throws when sqlite index is busy", async () => {
|
||||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId });
|
const resolved = resolveMemoryBackendConfig({ cfg, agentId });
|
||||||
const manager = await QmdMemoryManager.create({ cfg, agentId, resolved });
|
const manager = await QmdMemoryManager.create({ cfg, agentId, resolved });
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { spawn } from "node:child_process";
|
|||||||
import fs from "node:fs/promises";
|
import fs from "node:fs/promises";
|
||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
import readline from "node:readline";
|
||||||
import type { OpenClawConfig } from "../config/config.js";
|
import type { OpenClawConfig } from "../config/config.js";
|
||||||
import type {
|
import type {
|
||||||
MemoryEmbeddingProbeResult,
|
MemoryEmbeddingProbeResult,
|
||||||
@@ -353,6 +354,10 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||||
throw new Error("path required");
|
throw new Error("path required");
|
||||||
}
|
}
|
||||||
|
if (params.from !== undefined || params.lines !== undefined) {
|
||||||
|
const text = await this.readPartialText(absPath, params.from, params.lines);
|
||||||
|
return { text, path: relPath };
|
||||||
|
}
|
||||||
const content = await fs.readFile(absPath, "utf-8");
|
const content = await fs.readFile(absPath, "utf-8");
|
||||||
if (!params.from && !params.lines) {
|
if (!params.from && !params.lines) {
|
||||||
return { text: content, path: relPath };
|
return { text: content, path: relPath };
|
||||||
@@ -609,6 +614,35 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async readPartialText(absPath: string, from?: number, lines?: number): Promise<string> {
|
||||||
|
const start = Math.max(1, from ?? 1);
|
||||||
|
const count = Math.max(1, lines ?? Number.POSITIVE_INFINITY);
|
||||||
|
const handle = await fs.open(absPath);
|
||||||
|
const stream = handle.createReadStream({ encoding: "utf-8" });
|
||||||
|
const rl = readline.createInterface({
|
||||||
|
input: stream,
|
||||||
|
crlfDelay: Infinity,
|
||||||
|
});
|
||||||
|
const selected: string[] = [];
|
||||||
|
let index = 0;
|
||||||
|
try {
|
||||||
|
for await (const line of rl) {
|
||||||
|
index += 1;
|
||||||
|
if (index < start) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (selected.length >= count) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
selected.push(line);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
rl.close();
|
||||||
|
await handle.close();
|
||||||
|
}
|
||||||
|
return selected.slice(0, count).join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
private ensureDb(): SqliteDatabase {
|
private ensureDb(): SqliteDatabase {
|
||||||
if (this.db) {
|
if (this.db) {
|
||||||
return this.db;
|
return this.db;
|
||||||
|
|||||||
Reference in New Issue
Block a user