mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-10 06:32:43 +00:00
fix: enable FTS fallback when no embedding provider available (#17725)
When no embedding provider is available (e.g., OAuth mode without API keys), memory_search now falls back to FTS-only mode instead of returning disabled: true. Changes: - embeddings.ts: return null provider with reason instead of throwing - manager.ts: handle null provider, use FTS-only search mode - manager-search.ts: allow searching all models when provider is undefined - memory-tool.ts: expose search mode in results The search results now include a 'mode' field indicating 'hybrid' or 'fts-only'.
This commit is contained in:
@@ -81,12 +81,14 @@ export function createMemorySearchTool(options: {
|
|||||||
status.backend === "qmd"
|
status.backend === "qmd"
|
||||||
? clampResultsByInjectedChars(decorated, resolved.qmd?.limits.maxInjectedChars)
|
? clampResultsByInjectedChars(decorated, resolved.qmd?.limits.maxInjectedChars)
|
||||||
: decorated;
|
: decorated;
|
||||||
|
const searchMode = (status.custom as { searchMode?: string } | undefined)?.searchMode;
|
||||||
return jsonResult({
|
return jsonResult({
|
||||||
results,
|
results,
|
||||||
provider: status.provider,
|
provider: status.provider,
|
||||||
model: status.model,
|
model: status.model,
|
||||||
fallback: status.fallback,
|
fallback: status.fallback,
|
||||||
citations: citationsMode,
|
citations: citationsMode,
|
||||||
|
mode: searchMode,
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
|||||||
@@ -432,3 +432,63 @@ describe("local embedding normalization", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("FTS-only fallback when no provider available", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.resetAllMocks();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null provider with reason when auto mode finds no providers", async () => {
|
||||||
|
vi.mocked(authModule.resolveApiKeyForProvider).mockRejectedValue(
|
||||||
|
new Error('No API key found for provider "openai"'),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await createEmbeddingProvider({
|
||||||
|
config: {} as never,
|
||||||
|
provider: "auto",
|
||||||
|
model: "",
|
||||||
|
fallback: "none",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.provider).toBeNull();
|
||||||
|
expect(result.requestedProvider).toBe("auto");
|
||||||
|
expect(result.providerUnavailableReason).toBeDefined();
|
||||||
|
expect(result.providerUnavailableReason).toContain("No API key");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null provider when explicit provider fails with missing API key", async () => {
|
||||||
|
vi.mocked(authModule.resolveApiKeyForProvider).mockRejectedValue(
|
||||||
|
new Error('No API key found for provider "openai"'),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await createEmbeddingProvider({
|
||||||
|
config: {} as never,
|
||||||
|
provider: "openai",
|
||||||
|
model: "text-embedding-3-small",
|
||||||
|
fallback: "none",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.provider).toBeNull();
|
||||||
|
expect(result.requestedProvider).toBe("openai");
|
||||||
|
expect(result.providerUnavailableReason).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null provider when both primary and fallback fail with missing API keys", async () => {
|
||||||
|
vi.mocked(authModule.resolveApiKeyForProvider).mockRejectedValue(
|
||||||
|
new Error("No API key found for provider"),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await createEmbeddingProvider({
|
||||||
|
config: {} as never,
|
||||||
|
provider: "openai",
|
||||||
|
model: "text-embedding-3-small",
|
||||||
|
fallback: "gemini",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.provider).toBeNull();
|
||||||
|
expect(result.requestedProvider).toBe("openai");
|
||||||
|
expect(result.fallbackFrom).toBe("openai");
|
||||||
|
expect(result.providerUnavailableReason).toContain("Fallback to gemini failed");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -36,10 +36,11 @@ export type EmbeddingProviderFallback = EmbeddingProviderId | "none";
|
|||||||
const REMOTE_EMBEDDING_PROVIDER_IDS = ["openai", "gemini", "voyage"] as const;
|
const REMOTE_EMBEDDING_PROVIDER_IDS = ["openai", "gemini", "voyage"] as const;
|
||||||
|
|
||||||
export type EmbeddingProviderResult = {
|
export type EmbeddingProviderResult = {
|
||||||
provider: EmbeddingProvider;
|
provider: EmbeddingProvider | null;
|
||||||
requestedProvider: EmbeddingProviderRequest;
|
requestedProvider: EmbeddingProviderRequest;
|
||||||
fallbackFrom?: EmbeddingProviderId;
|
fallbackFrom?: EmbeddingProviderId;
|
||||||
fallbackReason?: string;
|
fallbackReason?: string;
|
||||||
|
providerUnavailableReason?: string;
|
||||||
openAi?: OpenAiEmbeddingClient;
|
openAi?: OpenAiEmbeddingClient;
|
||||||
gemini?: GeminiEmbeddingClient;
|
gemini?: GeminiEmbeddingClient;
|
||||||
voyage?: VoyageEmbeddingClient;
|
voyage?: VoyageEmbeddingClient;
|
||||||
@@ -183,15 +184,19 @@ export async function createEmbeddingProvider(
|
|||||||
missingKeyErrors.push(message);
|
missingKeyErrors.push(message);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
// Non-auth errors (e.g., network) are still fatal
|
||||||
throw new Error(message, { cause: err });
|
throw new Error(message, { cause: err });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// All providers failed due to missing API keys - return null provider for FTS-only mode
|
||||||
const details = [...missingKeyErrors, localError].filter(Boolean) as string[];
|
const details = [...missingKeyErrors, localError].filter(Boolean) as string[];
|
||||||
if (details.length > 0) {
|
const reason = details.length > 0 ? details.join("\n\n") : "No embeddings provider available.";
|
||||||
throw new Error(details.join("\n\n"));
|
return {
|
||||||
}
|
provider: null,
|
||||||
throw new Error("No embeddings provider available.");
|
requestedProvider,
|
||||||
|
providerUnavailableReason: reason,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -209,13 +214,31 @@ export async function createEmbeddingProvider(
|
|||||||
fallbackReason: reason,
|
fallbackReason: reason,
|
||||||
};
|
};
|
||||||
} catch (fallbackErr) {
|
} catch (fallbackErr) {
|
||||||
// oxlint-disable-next-line preserve-caught-error
|
// Both primary and fallback failed - check if it's auth-related
|
||||||
throw new Error(
|
const fallbackReason = formatErrorMessage(fallbackErr);
|
||||||
`${reason}\n\nFallback to ${fallback} failed: ${formatErrorMessage(fallbackErr)}`,
|
const combinedReason = `${reason}\n\nFallback to ${fallback} failed: ${fallbackReason}`;
|
||||||
{ cause: fallbackErr },
|
if (isMissingApiKeyError(primaryErr) && isMissingApiKeyError(fallbackErr)) {
|
||||||
);
|
// Both failed due to missing API keys - return null for FTS-only mode
|
||||||
|
return {
|
||||||
|
provider: null,
|
||||||
|
requestedProvider,
|
||||||
|
fallbackFrom: requestedProvider,
|
||||||
|
fallbackReason: reason,
|
||||||
|
providerUnavailableReason: combinedReason,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
// Non-auth errors are still fatal
|
||||||
|
throw new Error(combinedReason, { cause: fallbackErr });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// No fallback configured - check if we should degrade to FTS-only
|
||||||
|
if (isMissingApiKeyError(primaryErr)) {
|
||||||
|
return {
|
||||||
|
provider: null,
|
||||||
|
requestedProvider,
|
||||||
|
providerUnavailableReason: reason,
|
||||||
|
};
|
||||||
|
}
|
||||||
throw new Error(reason, { cause: primaryErr });
|
throw new Error(reason, { cause: primaryErr });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -202,6 +202,10 @@ class MemoryManagerEmbeddingOps {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private computeProviderKey(): string {
|
private computeProviderKey(): string {
|
||||||
|
// FTS-only mode: no provider, use a constant key
|
||||||
|
if (!this.provider) {
|
||||||
|
return hashText(JSON.stringify({ provider: "none", model: "fts-only" }));
|
||||||
|
}
|
||||||
if (this.provider.id === "openai" && this.openAi) {
|
if (this.provider.id === "openai" && this.openAi) {
|
||||||
const entries = Object.entries(this.openAi.headers)
|
const entries = Object.entries(this.openAi.headers)
|
||||||
.filter(([key]) => key.toLowerCase() !== "authorization")
|
.filter(([key]) => key.toLowerCase() !== "authorization")
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ export function listChunks(params: {
|
|||||||
export async function searchKeyword(params: {
|
export async function searchKeyword(params: {
|
||||||
db: DatabaseSync;
|
db: DatabaseSync;
|
||||||
ftsTable: string;
|
ftsTable: string;
|
||||||
providerModel: string;
|
providerModel: string | undefined;
|
||||||
query: string;
|
query: string;
|
||||||
limit: number;
|
limit: number;
|
||||||
snippetMaxChars: number;
|
snippetMaxChars: number;
|
||||||
@@ -152,16 +152,20 @@ export async function searchKeyword(params: {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// When providerModel is undefined (FTS-only mode), search all models
|
||||||
|
const modelClause = params.providerModel ? " AND model = ?" : "";
|
||||||
|
const modelParams = params.providerModel ? [params.providerModel] : [];
|
||||||
|
|
||||||
const rows = params.db
|
const rows = params.db
|
||||||
.prepare(
|
.prepare(
|
||||||
`SELECT id, path, source, start_line, end_line, text,\n` +
|
`SELECT id, path, source, start_line, end_line, text,\n` +
|
||||||
` bm25(${params.ftsTable}) AS rank\n` +
|
` bm25(${params.ftsTable}) AS rank\n` +
|
||||||
` FROM ${params.ftsTable}\n` +
|
` FROM ${params.ftsTable}\n` +
|
||||||
` WHERE ${params.ftsTable} MATCH ? AND model = ?${params.sourceFilter.sql}\n` +
|
` WHERE ${params.ftsTable} MATCH ?${modelClause}${params.sourceFilter.sql}\n` +
|
||||||
` ORDER BY rank ASC\n` +
|
` ORDER BY rank ASC\n` +
|
||||||
` LIMIT ?`,
|
` LIMIT ?`,
|
||||||
)
|
)
|
||||||
.all(ftsQuery, params.providerModel, ...params.sourceFilter.params, params.limit) as Array<{
|
.all(ftsQuery, ...modelParams, ...params.sourceFilter.params, params.limit) as Array<{
|
||||||
id: string;
|
id: string;
|
||||||
path: string;
|
path: string;
|
||||||
source: SearchSource;
|
source: SearchSource;
|
||||||
|
|||||||
@@ -46,10 +46,11 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
private readonly agentId: string;
|
private readonly agentId: string;
|
||||||
private readonly workspaceDir: string;
|
private readonly workspaceDir: string;
|
||||||
private readonly settings: ResolvedMemorySearchConfig;
|
private readonly settings: ResolvedMemorySearchConfig;
|
||||||
private provider: EmbeddingProvider;
|
private provider: EmbeddingProvider | null;
|
||||||
private readonly requestedProvider: "openai" | "local" | "gemini" | "voyage" | "auto";
|
private readonly requestedProvider: "openai" | "local" | "gemini" | "voyage" | "auto";
|
||||||
private fallbackFrom?: "openai" | "local" | "gemini" | "voyage";
|
private fallbackFrom?: "openai" | "local" | "gemini" | "voyage";
|
||||||
private fallbackReason?: string;
|
private fallbackReason?: string;
|
||||||
|
private readonly providerUnavailableReason?: string;
|
||||||
private openAi?: OpenAiEmbeddingClient;
|
private openAi?: OpenAiEmbeddingClient;
|
||||||
private gemini?: GeminiEmbeddingClient;
|
private gemini?: GeminiEmbeddingClient;
|
||||||
private voyage?: VoyageEmbeddingClient;
|
private voyage?: VoyageEmbeddingClient;
|
||||||
@@ -154,6 +155,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
this.requestedProvider = params.providerResult.requestedProvider;
|
this.requestedProvider = params.providerResult.requestedProvider;
|
||||||
this.fallbackFrom = params.providerResult.fallbackFrom;
|
this.fallbackFrom = params.providerResult.fallbackFrom;
|
||||||
this.fallbackReason = params.providerResult.fallbackReason;
|
this.fallbackReason = params.providerResult.fallbackReason;
|
||||||
|
this.providerUnavailableReason = params.providerResult.providerUnavailableReason;
|
||||||
this.openAi = params.providerResult.openAi;
|
this.openAi = params.providerResult.openAi;
|
||||||
this.gemini = params.providerResult.gemini;
|
this.gemini = params.providerResult.gemini;
|
||||||
this.voyage = params.providerResult.voyage;
|
this.voyage = params.providerResult.voyage;
|
||||||
@@ -225,6 +227,16 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)),
|
Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// FTS-only mode: no embedding provider available
|
||||||
|
if (!this.provider) {
|
||||||
|
if (!this.fts.enabled || !this.fts.available) {
|
||||||
|
log.warn("memory search: no provider and FTS unavailable");
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const ftsResults = await this.searchKeyword(cleaned, candidates).catch(() => []);
|
||||||
|
return ftsResults.filter((entry) => entry.score >= minScore).slice(0, maxResults);
|
||||||
|
}
|
||||||
|
|
||||||
const keywordResults = hybrid.enabled
|
const keywordResults = hybrid.enabled
|
||||||
? await this.searchKeyword(cleaned, candidates).catch(() => [])
|
? await this.searchKeyword(cleaned, candidates).catch(() => [])
|
||||||
: [];
|
: [];
|
||||||
@@ -253,6 +265,10 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
queryVec: number[],
|
queryVec: number[],
|
||||||
limit: number,
|
limit: number,
|
||||||
): Promise<Array<MemorySearchResult & { id: string }>> {
|
): Promise<Array<MemorySearchResult & { id: string }>> {
|
||||||
|
// This method should never be called without a provider
|
||||||
|
if (!this.provider) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
const results = await searchVector({
|
const results = await searchVector({
|
||||||
db: this.db,
|
db: this.db,
|
||||||
vectorTable: VECTOR_TABLE,
|
vectorTable: VECTOR_TABLE,
|
||||||
@@ -279,10 +295,12 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
const sourceFilter = this.buildSourceFilter();
|
const sourceFilter = this.buildSourceFilter();
|
||||||
|
// In FTS-only mode (no provider), search all models; otherwise filter by current provider's model
|
||||||
|
const providerModel = this.provider?.model;
|
||||||
const results = await searchKeyword({
|
const results = await searchKeyword({
|
||||||
db: this.db,
|
db: this.db,
|
||||||
ftsTable: FTS_TABLE,
|
ftsTable: FTS_TABLE,
|
||||||
providerModel: this.provider.model,
|
providerModel,
|
||||||
query,
|
query,
|
||||||
limit,
|
limit,
|
||||||
snippetMaxChars: SNIPPET_MAX_CHARS,
|
snippetMaxChars: SNIPPET_MAX_CHARS,
|
||||||
@@ -446,6 +464,13 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
}
|
}
|
||||||
return sources.map((source) => Object.assign({ source }, bySource.get(source)!));
|
return sources.map((source) => Object.assign({ source }, bySource.get(source)!));
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
// Determine search mode: "fts-only" if no provider, "hybrid" otherwise
|
||||||
|
const searchMode = this.provider ? "hybrid" : "fts-only";
|
||||||
|
const providerInfo = this.provider
|
||||||
|
? { provider: this.provider.id, model: this.provider.model }
|
||||||
|
: { provider: "none", model: undefined };
|
||||||
|
|
||||||
return {
|
return {
|
||||||
backend: "builtin",
|
backend: "builtin",
|
||||||
files: files?.c ?? 0,
|
files: files?.c ?? 0,
|
||||||
@@ -453,8 +478,8 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
dirty: this.dirty || this.sessionsDirty,
|
dirty: this.dirty || this.sessionsDirty,
|
||||||
workspaceDir: this.workspaceDir,
|
workspaceDir: this.workspaceDir,
|
||||||
dbPath: this.settings.store.path,
|
dbPath: this.settings.store.path,
|
||||||
provider: this.provider.id,
|
provider: providerInfo.provider,
|
||||||
model: this.provider.model,
|
model: providerInfo.model,
|
||||||
requestedProvider: this.requestedProvider,
|
requestedProvider: this.requestedProvider,
|
||||||
sources: Array.from(this.sources),
|
sources: Array.from(this.sources),
|
||||||
extraPaths: this.settings.extraPaths,
|
extraPaths: this.settings.extraPaths,
|
||||||
@@ -497,10 +522,18 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
lastError: this.batchFailureLastError,
|
lastError: this.batchFailureLastError,
|
||||||
lastProvider: this.batchFailureLastProvider,
|
lastProvider: this.batchFailureLastProvider,
|
||||||
},
|
},
|
||||||
|
custom: {
|
||||||
|
searchMode,
|
||||||
|
providerUnavailableReason: this.providerUnavailableReason,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async probeVectorAvailability(): Promise<boolean> {
|
async probeVectorAvailability(): Promise<boolean> {
|
||||||
|
// FTS-only mode: vector search not available
|
||||||
|
if (!this.provider) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!this.vector.enabled) {
|
if (!this.vector.enabled) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -508,6 +541,13 @@ export class MemoryIndexManager implements MemorySearchManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult> {
|
async probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult> {
|
||||||
|
// FTS-only mode: embeddings not available but search still works
|
||||||
|
if (!this.provider) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
error: this.providerUnavailableReason ?? "No embedding provider available (FTS-only mode)",
|
||||||
|
};
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
await this.embedBatchWithRetry(["ping"]);
|
await this.embedBatchWithRetry(["ping"]);
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
|
|||||||
Reference in New Issue
Block a user