fix(doctor): use gateway health status for memory search key check (#22327)

Merged via /review-pr -> /prepare-pr -> /merge-pr.

Prepared head SHA: 2f02ec9403
Co-authored-by: therk <901920+therk@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras
This commit is contained in:
Ruslan Kharitonov
2026-02-23 14:07:16 -05:00
committed by GitHub
parent bf373eeb43
commit 8d69251475
11 changed files with 338 additions and 11 deletions

View File

@@ -17,6 +17,7 @@ Docs: https://docs.openclaw.ai
- Agents/Context pruning: extend `cache-ttl` eligibility to Moonshot/Kimi and ZAI/GLM providers (including OpenRouter model refs), so `contextPruning.mode: "cache-ttl"` is no longer silently skipped for those sessions. (#24497) Thanks @lailoo. - Agents/Context pruning: extend `cache-ttl` eligibility to Moonshot/Kimi and ZAI/GLM providers (including OpenRouter model refs), so `contextPruning.mode: "cache-ttl"` is no longer silently skipped for those sessions. (#24497) Thanks @lailoo.
- Tools/web_search: add `provider: "kimi"` (Moonshot) support with key/config schema wiring and a corrected two-step `$web_search` tool flow that echoes tool results before final synthesis, including citation extraction from search results. (#16616, #18822) Thanks @adshine. - Tools/web_search: add `provider: "kimi"` (Moonshot) support with key/config schema wiring and a corrected two-step `$web_search` tool flow that echoes tool results before final synthesis, including citation extraction from search results. (#16616, #18822) Thanks @adshine.
- Media understanding/Video: add a native Moonshot video provider and include Moonshot in auto video key detection, plus refactor video execution to honor `entry/config/provider` baseUrl+header precedence (matching audio behavior). (#12063) Thanks @xiaoyaner0201. - Media understanding/Video: add a native Moonshot video provider and include Moonshot in auto video key detection, plus refactor video execution to honor `entry/config/provider` baseUrl+header precedence (matching audio behavior). (#12063) Thanks @xiaoyaner0201.
- Doctor/Memory: query gateway-side default-agent memory embedding readiness during `openclaw doctor` (instead of inferring from generic gateway health), and warn when the gateway memory probe is unavailable or not ready while keeping `openclaw configure` remediation guidance. (#22327) thanks @therk.
- Sessions/Store: canonicalize inbound mixed-case session keys for metadata and route updates, and migrate legacy case-variant entries to a single lowercase key to prevent duplicate sessions and missing TUI/WebUI history. (#9561) Thanks @hillghost86. - Sessions/Store: canonicalize inbound mixed-case session keys for metadata and route updates, and migrate legacy case-variant entries to a single lowercase key to prevent duplicate sessions and missing TUI/WebUI history. (#9561) Thanks @hillghost86.
- Telegram/Reactions: soft-fail reaction action errors (policy/token/emoji/API), accept snake_case `message_id`, and fallback to inbound message-id context when explicit `messageId` is omitted so DM reactions stay stable without regeneration loops. (#20236, #21001) Thanks @PeterShanxin and @vincentkoc. - Telegram/Reactions: soft-fail reaction action errors (policy/token/emoji/API), accept snake_case `message_id`, and fallback to inbound message-id context when explicit `messageId` is omitted so DM reactions stay stable without regeneration loops. (#20236, #21001) Thanks @PeterShanxin and @vincentkoc.
- Telegram/Polling: scope persisted polling offsets to bot identity and reuse a single awaited runner-stop path on abort/retry, preventing cross-token offset bleed and overlapping pollers during restart/error recovery. (#10850, #11347) Thanks @talhaorak, @anooprdawar, and @vincentkoc. - Telegram/Polling: scope persisted polling offsets to bot identity and reuse a single awaited runner-stop path on abort/retry, preventing cross-token offset bleed and overlapping pollers during restart/error recovery. (#10850, #11347) Thanks @talhaorak, @anooprdawar, and @vincentkoc.

View File

@@ -1,11 +1,18 @@
import type { OpenClawConfig } from "../config/config.js"; import type { OpenClawConfig } from "../config/config.js";
import { buildGatewayConnectionDetails, callGateway } from "../gateway/call.js"; import { buildGatewayConnectionDetails, callGateway } from "../gateway/call.js";
import type { DoctorMemoryStatusPayload } from "../gateway/server-methods/doctor.js";
import { collectChannelStatusIssues } from "../infra/channels-status-issues.js"; import { collectChannelStatusIssues } from "../infra/channels-status-issues.js";
import type { RuntimeEnv } from "../runtime.js"; import type { RuntimeEnv } from "../runtime.js";
import { note } from "../terminal/note.js"; import { note } from "../terminal/note.js";
import { formatHealthCheckFailure } from "./health-format.js"; import { formatHealthCheckFailure } from "./health-format.js";
import { healthCommand } from "./health.js"; import { healthCommand } from "./health.js";
export type GatewayMemoryProbe = {
checked: boolean;
ready: boolean;
error?: string;
};
export async function checkGatewayHealth(params: { export async function checkGatewayHealth(params: {
runtime: RuntimeEnv; runtime: RuntimeEnv;
cfg: OpenClawConfig; cfg: OpenClawConfig;
@@ -56,3 +63,30 @@ export async function checkGatewayHealth(params: {
return { healthOk }; return { healthOk };
} }
export async function probeGatewayMemoryStatus(params: {
cfg: OpenClawConfig;
timeoutMs?: number;
}): Promise<GatewayMemoryProbe> {
const timeoutMs =
typeof params.timeoutMs === "number" && params.timeoutMs > 0 ? params.timeoutMs : 8_000;
try {
const payload = await callGateway<DoctorMemoryStatusPayload>({
method: "doctor.memory.status",
timeoutMs,
config: params.cfg,
});
return {
checked: true,
ready: payload.embedding.ok,
error: payload.embedding.error,
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
return {
checked: true,
ready: false,
error: `gateway memory probe unavailable: ${message}`,
};
}
}

View File

@@ -43,7 +43,7 @@ describe("noteMemorySearchHealth", () => {
remote: { apiKey: "from-config" }, remote: { apiKey: "from-config" },
}); });
await noteMemorySearchHealth(cfg); await noteMemorySearchHealth(cfg, {});
expect(note).not.toHaveBeenCalled(); expect(note).not.toHaveBeenCalled();
expect(resolveApiKeyForProvider).not.toHaveBeenCalled(); expect(resolveApiKeyForProvider).not.toHaveBeenCalled();
@@ -53,9 +53,10 @@ describe("noteMemorySearchHealth", () => {
note.mockClear(); note.mockClear();
resolveDefaultAgentId.mockClear(); resolveDefaultAgentId.mockClear();
resolveAgentDir.mockClear(); resolveAgentDir.mockClear();
resolveMemorySearchConfig.mockClear(); resolveMemorySearchConfig.mockReset();
resolveApiKeyForProvider.mockClear(); resolveApiKeyForProvider.mockReset();
resolveMemoryBackendConfig.mockClear(); resolveApiKeyForProvider.mockRejectedValue(new Error("missing key"));
resolveMemoryBackendConfig.mockReset();
resolveMemoryBackendConfig.mockReturnValue({ backend: "builtin", citations: "auto" }); resolveMemoryBackendConfig.mockReturnValue({ backend: "builtin", citations: "auto" });
}); });
@@ -70,7 +71,7 @@ describe("noteMemorySearchHealth", () => {
remote: {}, remote: {},
}); });
await noteMemorySearchHealth(cfg); await noteMemorySearchHealth(cfg, {});
expect(note).not.toHaveBeenCalled(); expect(note).not.toHaveBeenCalled();
}); });
@@ -95,7 +96,7 @@ describe("noteMemorySearchHealth", () => {
mode: "api-key", mode: "api-key",
}); });
await noteMemorySearchHealth(cfg); await noteMemorySearchHealth(cfg, {});
expect(resolveApiKeyForProvider).toHaveBeenCalledWith({ expect(resolveApiKeyForProvider).toHaveBeenCalledWith({
provider: "google", provider: "google",
@@ -126,6 +127,42 @@ describe("noteMemorySearchHealth", () => {
}); });
expect(note).not.toHaveBeenCalled(); expect(note).not.toHaveBeenCalled();
}); });
it("notes when gateway probe reports embeddings ready and CLI API key is missing", async () => {
resolveMemorySearchConfig.mockReturnValue({
provider: "gemini",
local: {},
remote: {},
});
await noteMemorySearchHealth(cfg, {
gatewayMemoryProbe: { checked: true, ready: true },
});
const message = note.mock.calls[0]?.[0] as string;
expect(message).toContain("reports memory embeddings are ready");
});
it("uses configure hint when gateway probe is unavailable and API key is missing", async () => {
resolveMemorySearchConfig.mockReturnValue({
provider: "gemini",
local: {},
remote: {},
});
await noteMemorySearchHealth(cfg, {
gatewayMemoryProbe: {
checked: true,
ready: false,
error: "gateway memory probe unavailable: timeout",
},
});
const message = note.mock.calls[0]?.[0] as string;
expect(message).toContain("Gateway memory probe for default agent is not ready");
expect(message).toContain("openclaw configure");
expect(message).not.toContain("auth add");
});
}); });
describe("detectLegacyWorkspaceDirs", () => { describe("detectLegacyWorkspaceDirs", () => {

View File

@@ -12,7 +12,16 @@ import { resolveUserPath } from "../utils.js";
* Check whether memory search has a usable embedding provider. * Check whether memory search has a usable embedding provider.
* Runs as part of `openclaw doctor` — config-only, no network calls. * Runs as part of `openclaw doctor` — config-only, no network calls.
*/ */
export async function noteMemorySearchHealth(cfg: OpenClawConfig): Promise<void> { export async function noteMemorySearchHealth(
cfg: OpenClawConfig,
opts?: {
gatewayMemoryProbe?: {
checked: boolean;
ready: boolean;
error?: string;
};
},
): Promise<void> {
const agentId = resolveDefaultAgentId(cfg); const agentId = resolveDefaultAgentId(cfg);
const agentDir = resolveAgentDir(cfg, agentId); const agentDir = resolveAgentDir(cfg, agentId);
const resolved = resolveMemorySearchConfig(cfg, agentId); const resolved = resolveMemorySearchConfig(cfg, agentId);
@@ -54,15 +63,28 @@ export async function noteMemorySearchHealth(cfg: OpenClawConfig): Promise<void>
if (hasRemoteApiKey || (await hasApiKeyForProvider(resolved.provider, cfg, agentDir))) { if (hasRemoteApiKey || (await hasApiKeyForProvider(resolved.provider, cfg, agentDir))) {
return; return;
} }
if (opts?.gatewayMemoryProbe?.checked && opts.gatewayMemoryProbe.ready) {
note(
[
`Memory search provider is set to "${resolved.provider}" but the API key was not found in the CLI environment.`,
"The running gateway reports memory embeddings are ready for the default agent.",
`Verify: ${formatCliCommand("openclaw memory status --deep")}`,
].join("\n"),
"Memory search",
);
return;
}
const gatewayProbeWarning = buildGatewayProbeWarning(opts?.gatewayMemoryProbe);
const envVar = providerEnvVar(resolved.provider); const envVar = providerEnvVar(resolved.provider);
note( note(
[ [
`Memory search provider is set to "${resolved.provider}" but no API key was found.`, `Memory search provider is set to "${resolved.provider}" but no API key was found.`,
`Semantic recall will not work without a valid API key.`, `Semantic recall will not work without a valid API key.`,
gatewayProbeWarning ? gatewayProbeWarning : null,
"", "",
"Fix (pick one):", "Fix (pick one):",
`- Set ${envVar} in your environment`, `- Set ${envVar} in your environment`,
`- Add credentials: ${formatCliCommand(`openclaw auth add --provider ${resolved.provider}`)}`, `- Configure credentials: ${formatCliCommand("openclaw configure")}`,
`- To disable: ${formatCliCommand("openclaw config set agents.defaults.memorySearch.enabled false")}`, `- To disable: ${formatCliCommand("openclaw config set agents.defaults.memorySearch.enabled false")}`,
"", "",
`Verify: ${formatCliCommand("openclaw memory status --deep")}`, `Verify: ${formatCliCommand("openclaw memory status --deep")}`,
@@ -82,14 +104,28 @@ export async function noteMemorySearchHealth(cfg: OpenClawConfig): Promise<void>
} }
} }
if (opts?.gatewayMemoryProbe?.checked && opts.gatewayMemoryProbe.ready) {
note(
[
'Memory search provider is set to "auto" but the API key was not found in the CLI environment.',
"The running gateway reports memory embeddings are ready for the default agent.",
`Verify: ${formatCliCommand("openclaw memory status --deep")}`,
].join("\n"),
"Memory search",
);
return;
}
const gatewayProbeWarning = buildGatewayProbeWarning(opts?.gatewayMemoryProbe);
note( note(
[ [
"Memory search is enabled but no embedding provider is configured.", "Memory search is enabled but no embedding provider is configured.",
"Semantic recall will not work without an embedding provider.", "Semantic recall will not work without an embedding provider.",
gatewayProbeWarning ? gatewayProbeWarning : null,
"", "",
"Fix (pick one):", "Fix (pick one):",
"- Set OPENAI_API_KEY, GEMINI_API_KEY, VOYAGE_API_KEY, or MISTRAL_API_KEY in your environment", "- Set OPENAI_API_KEY, GEMINI_API_KEY, VOYAGE_API_KEY, or MISTRAL_API_KEY in your environment",
`- Add credentials: ${formatCliCommand("openclaw auth add --provider openai")}`, `- Configure credentials: ${formatCliCommand("openclaw configure")}`,
`- For local embeddings: configure agents.defaults.memorySearch.provider and local model path`, `- For local embeddings: configure agents.defaults.memorySearch.provider and local model path`,
`- To disable: ${formatCliCommand("openclaw config set agents.defaults.memorySearch.enabled false")}`, `- To disable: ${formatCliCommand("openclaw config set agents.defaults.memorySearch.enabled false")}`,
"", "",
@@ -145,3 +181,21 @@ function providerEnvVar(provider: string): string {
return `${provider.toUpperCase()}_API_KEY`; return `${provider.toUpperCase()}_API_KEY`;
} }
} }
function buildGatewayProbeWarning(
probe:
| {
checked: boolean;
ready: boolean;
error?: string;
}
| undefined,
): string | null {
if (!probe?.checked || probe.ready) {
return null;
}
const detail = probe.error?.trim();
return detail
? `Gateway memory probe for default agent is not ready: ${detail}`
: "Gateway memory probe for default agent is not ready.";
}

View File

@@ -10,6 +10,7 @@ vi.mock("./doctor-gateway-daemon-flow.js", () => ({
vi.mock("./doctor-gateway-health.js", () => ({ vi.mock("./doctor-gateway-health.js", () => ({
checkGatewayHealth: vi.fn().mockResolvedValue({ healthOk: false }), checkGatewayHealth: vi.fn().mockResolvedValue({ healthOk: false }),
probeGatewayMemoryStatus: vi.fn().mockResolvedValue({ checked: false, ready: false }),
})); }));
vi.mock("./doctor-memory-search.js", () => ({ vi.mock("./doctor-memory-search.js", () => ({

View File

@@ -29,7 +29,7 @@ import {
import { doctorShellCompletion } from "./doctor-completion.js"; import { doctorShellCompletion } from "./doctor-completion.js";
import { loadAndMaybeMigrateDoctorConfig } from "./doctor-config-flow.js"; import { loadAndMaybeMigrateDoctorConfig } from "./doctor-config-flow.js";
import { maybeRepairGatewayDaemon } from "./doctor-gateway-daemon-flow.js"; import { maybeRepairGatewayDaemon } from "./doctor-gateway-daemon-flow.js";
import { checkGatewayHealth } from "./doctor-gateway-health.js"; import { checkGatewayHealth, probeGatewayMemoryStatus } from "./doctor-gateway-health.js";
import { import {
maybeRepairGatewayServiceConfig, maybeRepairGatewayServiceConfig,
maybeScanExtraGatewayServices, maybeScanExtraGatewayServices,
@@ -264,7 +264,6 @@ export async function doctorCommand(
} }
noteWorkspaceStatus(cfg); noteWorkspaceStatus(cfg);
await noteMemorySearchHealth(cfg);
// Check and fix shell completion // Check and fix shell completion
await doctorShellCompletion(runtime, prompter, { await doctorShellCompletion(runtime, prompter, {
@@ -276,6 +275,13 @@ export async function doctorCommand(
cfg, cfg,
timeoutMs: options.nonInteractive === true ? 3000 : 10_000, timeoutMs: options.nonInteractive === true ? 3000 : 10_000,
}); });
const gatewayMemoryProbe = healthOk
? await probeGatewayMemoryStatus({
cfg,
timeoutMs: options.nonInteractive === true ? 3000 : 10_000,
})
: { checked: false, ready: false };
await noteMemorySearchHealth(cfg, { gatewayMemoryProbe });
await maybeRepairGatewayDaemon({ await maybeRepairGatewayDaemon({
cfg, cfg,
runtime, runtime,

View File

@@ -43,6 +43,7 @@ const METHOD_SCOPE_GROUPS: Record<OperatorScope, readonly string[]> = {
], ],
[READ_SCOPE]: [ [READ_SCOPE]: [
"health", "health",
"doctor.memory.status",
"logs.tail", "logs.tail",
"channels.status", "channels.status",
"status", "status",

View File

@@ -3,6 +3,7 @@ import { GATEWAY_EVENT_UPDATE_AVAILABLE } from "./events.js";
const BASE_METHODS = [ const BASE_METHODS = [
"health", "health",
"doctor.memory.status",
"logs.tail", "logs.tail",
"channels.status", "channels.status",
"channels.logout", "channels.logout",

View File

@@ -12,6 +12,7 @@ import { configHandlers } from "./server-methods/config.js";
import { connectHandlers } from "./server-methods/connect.js"; import { connectHandlers } from "./server-methods/connect.js";
import { cronHandlers } from "./server-methods/cron.js"; import { cronHandlers } from "./server-methods/cron.js";
import { deviceHandlers } from "./server-methods/devices.js"; import { deviceHandlers } from "./server-methods/devices.js";
import { doctorHandlers } from "./server-methods/doctor.js";
import { execApprovalsHandlers } from "./server-methods/exec-approvals.js"; import { execApprovalsHandlers } from "./server-methods/exec-approvals.js";
import { healthHandlers } from "./server-methods/health.js"; import { healthHandlers } from "./server-methods/health.js";
import { logsHandlers } from "./server-methods/logs.js"; import { logsHandlers } from "./server-methods/logs.js";
@@ -71,6 +72,7 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
...chatHandlers, ...chatHandlers,
...cronHandlers, ...cronHandlers,
...deviceHandlers, ...deviceHandlers,
...doctorHandlers,
...execApprovalsHandlers, ...execApprovalsHandlers,
...webHandlers, ...webHandlers,
...modelsHandlers, ...modelsHandlers,

View File

@@ -0,0 +1,128 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
const loadConfig = vi.hoisted(() => vi.fn(() => ({}) as OpenClawConfig));
const resolveDefaultAgentId = vi.hoisted(() => vi.fn(() => "main"));
const getMemorySearchManager = vi.hoisted(() => vi.fn());
vi.mock("../../config/config.js", () => ({
loadConfig,
}));
vi.mock("../../agents/agent-scope.js", () => ({
resolveDefaultAgentId,
}));
vi.mock("../../memory/index.js", () => ({
getMemorySearchManager,
}));
import { doctorHandlers } from "./doctor.js";
describe("doctor.memory.status", () => {
beforeEach(() => {
loadConfig.mockClear();
resolveDefaultAgentId.mockClear();
getMemorySearchManager.mockReset();
});
it("returns gateway embedding probe status for the default agent", async () => {
const close = vi.fn().mockResolvedValue(undefined);
getMemorySearchManager.mockResolvedValue({
manager: {
status: () => ({ provider: "gemini" }),
probeEmbeddingAvailability: vi.fn().mockResolvedValue({ ok: true }),
close,
},
});
const respond = vi.fn();
await doctorHandlers["doctor.memory.status"]({
req: {} as never,
params: {} as never,
respond: respond as never,
context: {} as never,
client: null,
isWebchatConnect: () => false,
});
expect(getMemorySearchManager).toHaveBeenCalledWith({
cfg: expect.any(Object),
agentId: "main",
purpose: "status",
});
expect(respond).toHaveBeenCalledWith(
true,
{
agentId: "main",
provider: "gemini",
embedding: { ok: true },
},
undefined,
);
expect(close).toHaveBeenCalled();
});
it("returns unavailable when memory manager is missing", async () => {
getMemorySearchManager.mockResolvedValue({
manager: null,
error: "memory search unavailable",
});
const respond = vi.fn();
await doctorHandlers["doctor.memory.status"]({
req: {} as never,
params: {} as never,
respond: respond as never,
context: {} as never,
client: null,
isWebchatConnect: () => false,
});
expect(respond).toHaveBeenCalledWith(
true,
{
agentId: "main",
embedding: {
ok: false,
error: "memory search unavailable",
},
},
undefined,
);
});
it("returns probe failure when manager probe throws", async () => {
const close = vi.fn().mockResolvedValue(undefined);
getMemorySearchManager.mockResolvedValue({
manager: {
status: () => ({ provider: "openai" }),
probeEmbeddingAvailability: vi.fn().mockRejectedValue(new Error("timeout")),
close,
},
});
const respond = vi.fn();
await doctorHandlers["doctor.memory.status"]({
req: {} as never,
params: {} as never,
respond: respond as never,
context: {} as never,
client: null,
isWebchatConnect: () => false,
});
expect(respond).toHaveBeenCalledWith(
true,
{
agentId: "main",
embedding: {
ok: false,
error: "gateway memory probe failed: timeout",
},
},
undefined,
);
expect(close).toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,62 @@
import { resolveDefaultAgentId } from "../../agents/agent-scope.js";
import { loadConfig } from "../../config/config.js";
import { getMemorySearchManager } from "../../memory/index.js";
import { formatError } from "../server-utils.js";
import type { GatewayRequestHandlers } from "./types.js";
export type DoctorMemoryStatusPayload = {
agentId: string;
provider?: string;
embedding: {
ok: boolean;
error?: string;
};
};
export const doctorHandlers: GatewayRequestHandlers = {
"doctor.memory.status": async ({ respond }) => {
const cfg = loadConfig();
const agentId = resolveDefaultAgentId(cfg);
const { manager, error } = await getMemorySearchManager({
cfg,
agentId,
purpose: "status",
});
if (!manager) {
const payload: DoctorMemoryStatusPayload = {
agentId,
embedding: {
ok: false,
error: error ?? "memory search unavailable",
},
};
respond(true, payload, undefined);
return;
}
try {
const status = manager.status();
let embedding = await manager.probeEmbeddingAvailability();
if (!embedding.ok && !embedding.error) {
embedding = { ok: false, error: "memory embeddings unavailable" };
}
const payload: DoctorMemoryStatusPayload = {
agentId,
provider: status.provider,
embedding,
};
respond(true, payload, undefined);
} catch (err) {
const payload: DoctorMemoryStatusPayload = {
agentId,
embedding: {
ok: false,
error: `gateway memory probe failed: ${formatError(err)}`,
},
};
respond(true, payload, undefined);
} finally {
await manager.close?.().catch(() => {});
}
},
};