github/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-11 00:14:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: dedupe agent and reply runtimes

Browse Source
This commit is contained in:
Peter Steinberger
2026-03-02 19:47:30 +00:00
parent 8768487aee
commit 9617ac9dd5
53 changed files with 1828 additions and 1176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/agents/tools/discord-actions-guild.ts
View File

@@ -29,16 +29,7 @@ import {
readStringArrayParam,
readStringParam,
} from "./common.js";
function readParentIdParam(params: Record<string, unknown>): string | null | undefined {
if (params.clearParent === true) {
return null;
}
if (params.parentId === null) {
return null;
}
return readStringParam(params, "parentId");
}
import { readDiscordParentIdParam } from "./discord-actions-shared.js";
type DiscordRoleMutation = (params: {
guildId: string;
@@ -287,7 +278,7 @@ export async function handleDiscordGuildAction(
const guildId = readStringParam(params, "guildId", { required: true });
const name = readStringParam(params, "name", { required: true });
const type = readNumberParam(params, "type", { integer: true });
const parentId = readParentIdParam(params);
const parentId = readDiscordParentIdParam(params);
const topic = readStringParam(params, "topic");
const position = readNumberParam(params, "position", { integer: true });
const nsfw = params.nsfw as boolean | undefined;
@@ -325,7 +316,7 @@ export async function handleDiscordGuildAction(
const name = readStringParam(params, "name");
const topic = readStringParam(params, "topic");
const position = readNumberParam(params, "position", { integer: true });
const parentId = readParentIdParam(params);
const parentId = readDiscordParentIdParam(params);
const nsfw = params.nsfw as boolean | undefined;
const rateLimitPerUser = readNumberParam(params, "rateLimitPerUser", {
integer: true,
@@ -388,7 +379,7 @@ export async function handleDiscordGuildAction(
const channelId = readStringParam(params, "channelId", {
required: true,
});
const parentId = readParentIdParam(params);
const parentId = readDiscordParentIdParam(params);
const position = readNumberParam(params, "position", { integer: true });
if (accountId) {
await moveChannelDiscord(

13
src/agents/tools/discord-actions-shared.ts Normal file
View File

@@ -0,0 +1,13 @@
import { readStringParam } from "./common.js";
export function readDiscordParentIdParam(
params: Record<string, unknown>,
): string | null | undefined {
if (params.clearParent === true) {
return null;
}
if (params.parentId === null) {
return null;
}
return readStringParam(params, "parentId");
}

20
src/agents/tools/image-tool.test.ts
View File

@@ -8,6 +8,7 @@ import { withFetchPreconnect } from "../../test-utils/fetch-mock.js";
import { createOpenClawCodingTools } from "../pi-tools.js";
import { createHostSandboxFsBridge } from "../test-helpers/host-sandbox-fs-bridge.js";
import { createUnsafeMountedSandbox } from "../test-helpers/unsafe-mounted-sandbox.js";
import { makeZeroUsageSnapshot } from "../usage.js";
import { __testing, createImageTool, resolveImageModelConfigForTool } from "./image-tool.js";
async function writeAuthProfiles(agentDir: string, profiles: unknown) {
@@ -766,23 +767,6 @@ describe("image tool MiniMax VLM routing", () => {
});
describe("image tool response validation", () => {
function zeroUsage() {
return {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
total: 0,
},
};
}
function createAssistantMessage(
overrides: Partial<{
api: string;
@@ -800,7 +784,7 @@ describe("image tool response validation", () => {
model: "gpt-5-mini",
stopReason: "stop",
timestamp: Date.now(),
usage: zeroUsage(),
usage: makeZeroUsageSnapshot(),
content: [] as unknown[],
...overrides,
};

70
src/agents/tools/image-tool.ts
View File

@@ -1,8 +1,8 @@
import { type Api, type Context, complete, type Model } from "@mariozechner/pi-ai";
import { type Context, complete } from "@mariozechner/pi-ai";
import { Type } from "@sinclair/typebox";
import type { OpenClawConfig } from "../../config/config.js";
import { resolveUserPath } from "../../utils.js";
import { getDefaultLocalRoots, loadWebMedia } from "../../web/media.js";
import { loadWebMedia } from "../../web/media.js";
import { minimaxUnderstandImage } from "../minimax-vlm.js";
import {
coerceImageAssistantText,
@@ -11,15 +11,20 @@ import {
type ImageModelConfig,
resolveProviderVisionModelFromConfig,
} from "./image-tool.helpers.js";
import {
applyImageModelConfigDefaults,
buildTextToolResult,
resolveMediaToolLocalRoots,
resolveModelFromRegistry,
resolveModelRuntimeApiKey,
resolvePromptAndModelOverride,
} from "./media-tool-shared.js";
import { hasAuthForProvider, resolveDefaultModelRef } from "./model-config.helpers.js";
import {
createSandboxBridgeReadFile,
discoverAuthStorage,
discoverModels,
ensureOpenClawModelsJson,
getApiKeyForModel,
normalizeWorkspaceDir,
requireApiKey,
resolveSandboxedBridgeMediaPath,
runWithImageModelFallback,
type AnyAgentTool,
@@ -202,18 +207,7 @@ async function runImagePrompt(params: {
model: string;
attempts: Array<{ provider: string; model: string; error: string }>;
}> {
const effectiveCfg: OpenClawConfig | undefined = params.cfg
? {
...params.cfg,
agents: {
...params.cfg.agents,
defaults: {
...params.cfg.agents?.defaults,
imageModel: params.imageModelConfig,
},
},
}
: undefined;
const effectiveCfg = applyImageModelConfigDefaults(params.cfg, params.imageModelConfig);
await ensureOpenClawModelsJson(effectiveCfg, params.agentDir);
const authStorage = discoverAuthStorage(params.agentDir);
@@ -223,20 +217,16 @@ async function runImagePrompt(params: {
cfg: effectiveCfg,
modelOverride: params.modelOverride,
run: async (provider, modelId) => {
const model = modelRegistry.find(provider, modelId) as Model<Api> | null;
if (!model) {
throw new Error(`Unknown model: ${provider}/${modelId}`);
}
const model = resolveModelFromRegistry({ modelRegistry, provider, modelId });
if (!model.input?.includes("image")) {
throw new Error(`Model does not support images: ${provider}/${modelId}`);
}
const apiKeyInfo = await getApiKeyForModel({
const apiKey = await resolveModelRuntimeApiKey({
model,
cfg: effectiveCfg,
agentDir: params.agentDir,
authStorage,
});
const apiKey = requireApiKey(apiKeyInfo, model.provider);
authStorage.setRuntimeApiKey(model.provider, apiKey);
// MiniMax VLM only supports a single image; use the first one.
if (model.provider === "minimax") {
@@ -308,6 +298,7 @@ export function createImageTool(options?: {
? "Analyze one or more images with a vision model. Use image for a single path/URL, or images for multiple (up to 20). Only use this tool when images were NOT already provided in the user's message. Images mentioned in the prompt are automatically visible to you."
: "Analyze one or more images with the configured image model (agents.defaults.imageModel). Use image for a single path/URL, or images for multiple (up to 20). Provide a prompt describing what to analyze.";
<<<<<<< HEAD
const localRoots = (() => {
const workspaceDir = normalizeWorkspaceDir(options?.workspaceDir);
if (options?.fsPolicy?.workspaceOnly) {
@@ -319,6 +310,18 @@ export function createImageTool(options?: {
}
return Array.from(new Set([...roots, workspaceDir]));
})();
||||||| parent of 4a741746c (refactor: dedupe agent and reply runtimes)
const localRoots = (() => {
const roots = getDefaultLocalRoots();
const workspaceDir = normalizeWorkspaceDir(options?.workspaceDir);
if (!workspaceDir) {
return roots;
}
return Array.from(new Set([...roots, workspaceDir]));
})();
=======
const localRoots = resolveMediaToolLocalRoots(options?.workspaceDir);
>>>>>>> 4a741746c (refactor: dedupe agent and reply runtimes)
return {
label: "Image",
@@ -383,12 +386,10 @@ export function createImageTool(options?: {
};
}
const promptRaw =
typeof record.prompt === "string" && record.prompt.trim()
? record.prompt.trim()
: DEFAULT_PROMPT;
const modelOverride =
typeof record.model === "string" && record.model.trim() ? record.model.trim() : undefined;
const { prompt: promptRaw, modelOverride } = resolvePromptAndModelOverride(
record,
DEFAULT_PROMPT,
);
const maxBytesMb = typeof record.maxBytesMb === "number" ? record.maxBytesMb : undefined;
const maxBytes = pickMaxBytes(options?.config, maxBytesMb);
@@ -525,14 +526,7 @@ export function createImageTool(options?: {
})),
};
return {
content: [{ type: "text", text: result.text }],
details: {
model: `${result.provider}/${result.model}`,
...imageDetails,
attempts: result.attempts,
},
};
return buildTextToolResult(result, imageDetails);
},
};
}

107
src/agents/tools/media-tool-shared.ts Normal file
View File

@@ -0,0 +1,107 @@
import { type Api, type Model } from "@mariozechner/pi-ai";
import type { OpenClawConfig } from "../../config/config.js";
import { getDefaultLocalRoots } from "../../web/media.js";
import type { ImageModelConfig } from "./image-tool.helpers.js";
import { getApiKeyForModel, normalizeWorkspaceDir, requireApiKey } from "./tool-runtime.helpers.js";
type TextToolAttempt = {
provider: string;
model: string;
error: string;
};
type TextToolResult = {
text: string;
provider: string;
model: string;
attempts: TextToolAttempt[];
};
export function applyImageModelConfigDefaults(
cfg: OpenClawConfig | undefined,
imageModelConfig: ImageModelConfig,
): OpenClawConfig | undefined {
if (!cfg) {
return undefined;
}
return {
...cfg,
agents: {
...cfg.agents,
defaults: {
...cfg.agents?.defaults,
imageModel: imageModelConfig,
},
},
};
}
export function resolveMediaToolLocalRoots(workspaceDirRaw: string | undefined): string[] {
const roots = getDefaultLocalRoots();
const workspaceDir = normalizeWorkspaceDir(workspaceDirRaw);
if (!workspaceDir) {
return [...roots];
}
return Array.from(new Set([...roots, workspaceDir]));
}
export function resolvePromptAndModelOverride(
args: Record<string, unknown>,
defaultPrompt: string,
): {
prompt: string;
modelOverride?: string;
} {
const prompt =
typeof args.prompt === "string" && args.prompt.trim() ? args.prompt.trim() : defaultPrompt;
const modelOverride =
typeof args.model === "string" && args.model.trim() ? args.model.trim() : undefined;
return { prompt, modelOverride };
}
export function buildTextToolResult(
result: TextToolResult,
extraDetails: Record<string, unknown>,
): {
content: Array<{ type: "text"; text: string }>;
details: Record<string, unknown>;
} {
return {
content: [{ type: "text", text: result.text }],
details: {
model: `${result.provider}/${result.model}`,
...extraDetails,
attempts: result.attempts,
},
};
}
export function resolveModelFromRegistry(params: {
modelRegistry: { find: (provider: string, modelId: string) => unknown };
provider: string;
modelId: string;
}): Model<Api> {
const model = params.modelRegistry.find(params.provider, params.modelId) as Model<Api> | null;
if (!model) {
throw new Error(`Unknown model: ${params.provider}/${params.modelId}`);
}
return model;
}
export async function resolveModelRuntimeApiKey(params: {
model: Model<Api>;
cfg: OpenClawConfig | undefined;
agentDir: string;
authStorage: {
setRuntimeApiKey: (provider: string, apiKey: string) => void;
};
}): Promise<string> {
const apiKeyInfo = await getApiKeyForModel({
model: params.model,
cfg: params.cfg,
agentDir: params.agentDir,
});
const apiKey = requireApiKey(apiKeyInfo, params.model.provider);
params.authStorage.setRuntimeApiKey(params.model.provider, apiKey);
return apiKey;
}

72
src/agents/tools/pdf-tool.ts
View File

@@ -1,14 +1,22 @@
import { type Api, type Context, complete, type Model } from "@mariozechner/pi-ai";
import { type Context, complete } from "@mariozechner/pi-ai";
import { Type } from "@sinclair/typebox";
import type { OpenClawConfig } from "../../config/config.js";
import { extractPdfContent, type PdfExtractedContent } from "../../media/pdf-extract.js";
import { resolveUserPath } from "../../utils.js";
import { getDefaultLocalRoots, loadWebMediaRaw } from "../../web/media.js";
import { loadWebMediaRaw } from "../../web/media.js";
import {
coerceImageModelConfig,
type ImageModelConfig,
resolveProviderVisionModelFromConfig,
} from "./image-tool.helpers.js";
import {
applyImageModelConfigDefaults,
buildTextToolResult,
resolveMediaToolLocalRoots,
resolveModelFromRegistry,
resolveModelRuntimeApiKey,
resolvePromptAndModelOverride,
} from "./media-tool-shared.js";
import { hasAuthForProvider, resolveDefaultModelRef } from "./model-config.helpers.js";
import { anthropicAnalyzePdf, geminiAnalyzePdf } from "./pdf-native-providers.js";
import {
@@ -23,9 +31,6 @@ import {
discoverAuthStorage,
discoverModels,
ensureOpenClawModelsJson,
getApiKeyForModel,
normalizeWorkspaceDir,
requireApiKey,
resolveSandboxedBridgeMediaPath,
runWithImageModelFallback,
type AnyAgentTool,
@@ -176,18 +181,7 @@ async function runPdfPrompt(params: {
native: boolean;
attempts: Array<{ provider: string; model: string; error: string }>;
}> {
const effectiveCfg: OpenClawConfig | undefined = params.cfg
? {
...params.cfg,
agents: {
...params.cfg.agents,
defaults: {
...params.cfg.agents?.defaults,
imageModel: params.pdfModelConfig,
},
},
}
: undefined;
const effectiveCfg = applyImageModelConfigDefaults(params.cfg, params.pdfModelConfig);
await ensureOpenClawModelsJson(effectiveCfg, params.agentDir);
const authStorage = discoverAuthStorage(params.agentDir);
@@ -205,18 +199,13 @@ async function runPdfPrompt(params: {
cfg: effectiveCfg,
modelOverride: params.modelOverride,
run: async (provider, modelId) => {
const model = modelRegistry.find(provider, modelId) as Model<Api> | null;
if (!model) {
throw new Error(`Unknown model: ${provider}/${modelId}`);
}
const apiKeyInfo = await getApiKeyForModel({
const model = resolveModelFromRegistry({ modelRegistry, provider, modelId });
const apiKey = await resolveModelRuntimeApiKey({
model,
cfg: effectiveCfg,
agentDir: params.agentDir,
authStorage,
});
const apiKey = requireApiKey(apiKeyInfo, model.provider);
authStorage.setRuntimeApiKey(model.provider, apiKey);
if (providerSupportsNativePdf(provider)) {
if (params.pageNumbers && params.pageNumbers.length > 0) {
@@ -338,6 +327,7 @@ export function createPdfTool(options?: {
? Math.floor(maxPagesDefault)
: DEFAULT_MAX_PAGES;
<<<<<<< HEAD
const localRoots = (() => {
const workspaceDir = normalizeWorkspaceDir(options?.workspaceDir);
if (options?.fsPolicy?.workspaceOnly) {
@@ -349,6 +339,18 @@ export function createPdfTool(options?: {
}
return Array.from(new Set([...roots, workspaceDir]));
})();
||||||| parent of 4a741746c (refactor: dedupe agent and reply runtimes)
const localRoots = (() => {
const roots = getDefaultLocalRoots();
const workspaceDir = normalizeWorkspaceDir(options?.workspaceDir);
if (!workspaceDir) {
return roots;
}
return Array.from(new Set([...roots, workspaceDir]));
})();
=======
const localRoots = resolveMediaToolLocalRoots(options?.workspaceDir);
>>>>>>> 4a741746c (refactor: dedupe agent and reply runtimes)
const description =
"Analyze one or more PDF documents with a model. Supports native PDF analysis for Anthropic and Google models, with text/image extraction fallback for other providers. Use pdf for a single path/URL, or pdfs for multiple (up to 10). Provide a prompt describing what to analyze.";
@@ -412,12 +414,10 @@ export function createPdfTool(options?: {
};
}
const promptRaw =
typeof record.prompt === "string" && record.prompt.trim()
? record.prompt.trim()
: DEFAULT_PROMPT;
const modelOverride =
typeof record.model === "string" && record.model.trim() ? record.model.trim() : undefined;
const { prompt: promptRaw, modelOverride } = resolvePromptAndModelOverride(
record,
DEFAULT_PROMPT,
);
const maxBytesMbRaw = typeof record.maxBytesMb === "number" ? record.maxBytesMb : undefined;
const maxBytesMb =
typeof maxBytesMbRaw === "number" && Number.isFinite(maxBytesMbRaw) && maxBytesMbRaw > 0
@@ -573,15 +573,7 @@ export function createPdfTool(options?: {
})),
};
return {
content: [{ type: "text", text: result.text }],
details: {
model: `${result.provider}/${result.model}`,
native: result.native,
...pdfDetails,
attempts: result.attempts,
},
};
return buildTextToolResult(result, { native: result.native, ...pdfDetails });
},
};
}

1
src/agents/tools/sessions-helpers.ts
View File

@@ -23,6 +23,7 @@ export {
resolveInternalSessionKey,
resolveMainSessionAlias,
resolveSessionReference,
resolveVisibleSessionReference,
shouldResolveSessionIdInput,
shouldVerifyRequesterSpawnedSessionVisibility,
} from "./sessions-resolution.js";

22
src/agents/tools/sessions-history-tool.ts
View File

@@ -10,10 +10,10 @@ import { jsonResult, readStringParam } from "./common.js";
import {
createSessionVisibilityGuard,
createAgentToAgentPolicy,
isResolvedSessionVisibleToRequester,
resolveEffectiveSessionToolsVisibility,
resolveSessionReference,
resolveSandboxedSessionToolContext,
resolveVisibleSessionReference,
stripToolMessages,
} from "./sessions-helpers.js";
@@ -197,23 +197,21 @@ export function createSessionsHistoryTool(opts?: {
if (!resolvedSession.ok) {
return jsonResult({ status: resolvedSession.status, error: resolvedSession.error });
}
// From here on, use the canonical key (sessionId inputs already resolved).
const resolvedKey = resolvedSession.key;
const displayKey = resolvedSession.displayKey;
const resolvedViaSessionId = resolvedSession.resolvedViaSessionId;
const visible = await isResolvedSessionVisibleToRequester({
const visibleSession = await resolveVisibleSessionReference({
resolvedSession,
requesterSessionKey: effectiveRequesterKey,
targetSessionKey: resolvedKey,
restrictToSpawned,
resolvedViaSessionId,
visibilitySessionKey: sessionKeyParam,
});
if (!visible) {
if (!visibleSession.ok) {
return jsonResult({
status: "forbidden",
error: `Session not visible from this sandboxed agent session: ${sessionKeyParam}`,
status: visibleSession.status,
error: visibleSession.error,
});
}
// From here on, use the canonical key (sessionId inputs already resolved).
const resolvedKey = visibleSession.key;
const displayKey = visibleSession.displayKey;
const a2aPolicy = createAgentToAgentPolicy(cfg);
const visibility = resolveEffectiveSessionToolsVisibility({

38
src/agents/tools/sessions-resolution.ts
View File

@@ -159,6 +159,19 @@ export type SessionReferenceResolution =
}
| { ok: false; status: "error" | "forbidden"; error: string };
export type VisibleSessionReferenceResolution =
| {
ok: true;
key: string;
displayKey: string;
}
| {
ok: false;
status: "forbidden";
error: string;
displayKey: string;
};
async function resolveSessionKeyFromSessionId(params: {
sessionId: string;
alias: string;
@@ -289,6 +302,31 @@ export async function resolveSessionReference(params: {
return { ok: true, key: resolvedKey, displayKey, resolvedViaSessionId: false };
}
export async function resolveVisibleSessionReference(params: {
resolvedSession: Extract<SessionReferenceResolution, { ok: true }>;
requesterSessionKey: string;
restrictToSpawned: boolean;
visibilitySessionKey: string;
}): Promise<VisibleSessionReferenceResolution> {
const resolvedKey = params.resolvedSession.key;
const displayKey = params.resolvedSession.displayKey;
const visible = await isResolvedSessionVisibleToRequester({
requesterSessionKey: params.requesterSessionKey,
targetSessionKey: resolvedKey,
restrictToSpawned: params.restrictToSpawned,
resolvedViaSessionId: params.resolvedSession.resolvedViaSessionId,
});
if (!visible) {
return {
ok: false,
status: "forbidden",
error: `Session not visible from this sandboxed agent session: ${params.visibilitySessionKey}`,
displayKey,
};
}
return { ok: true, key: resolvedKey, displayKey };
}
export function normalizeOptionalKey(value?: string) {
return normalizeKey(value);
}

24
src/agents/tools/sessions-send-tool.ts
View File

@@ -15,10 +15,10 @@ import {
createSessionVisibilityGuard,
createAgentToAgentPolicy,
extractAssistantText,
isResolvedSessionVisibleToRequester,
resolveEffectiveSessionToolsVisibility,
resolveSessionReference,
resolveSandboxedSessionToolContext,
resolveVisibleSessionReference,
stripToolMessages,
} from "./sessions-helpers.js";
import { buildAgentToAgentMessageContext, resolvePingPongTurns } from "./sessions-send-helpers.js";
@@ -171,25 +171,23 @@ export function createSessionsSendTool(opts?: {
error: resolvedSession.error,
});
}
// Normalize sessionKey/sessionId input into a canonical session key.
const resolvedKey = resolvedSession.key;
const displayKey = resolvedSession.displayKey;
const resolvedViaSessionId = resolvedSession.resolvedViaSessionId;
const visible = await isResolvedSessionVisibleToRequester({
const visibleSession = await resolveVisibleSessionReference({
resolvedSession,
requesterSessionKey: effectiveRequesterKey,
targetSessionKey: resolvedKey,
restrictToSpawned,
resolvedViaSessionId,
visibilitySessionKey: sessionKey,
});
if (!visible) {
if (!visibleSession.ok) {
return jsonResult({
runId: crypto.randomUUID(),
status: "forbidden",
error: `Session not visible from this sandboxed agent session: ${sessionKey}`,
sessionKey: displayKey,
status: visibleSession.status,
error: visibleSession.error,
sessionKey: visibleSession.displayKey,
});
}
// Normalize sessionKey/sessionId input into a canonical session key.
const resolvedKey = visibleSession.key;
const displayKey = visibleSession.displayKey;
const timeoutSeconds =
typeof params.timeoutSeconds === "number" && Number.isFinite(params.timeoutSeconds)
? Math.max(0, Math.floor(params.timeoutSeconds))