feat: add image model config + tool

This commit is contained in:
Peter Steinberger
2026-01-04 19:35:00 +01:00
parent 0716a624a8
commit 78998dba9e
20 changed files with 856 additions and 144 deletions

View File

@@ -9,6 +9,13 @@ export {
modelsFallbacksListCommand,
modelsFallbacksRemoveCommand,
} from "./models/fallbacks.js";
export {
modelsImageFallbacksAddCommand,
modelsImageFallbacksClearCommand,
modelsImageFallbacksListCommand,
modelsImageFallbacksRemoveCommand,
} from "./models/image-fallbacks.js";
export { modelsListCommand, modelsStatusCommand } from "./models/list.js";
export { modelsScanCommand } from "./models/scan.js";
export { modelsSetCommand } from "./models/set.js";
export { modelsSetImageCommand } from "./models/set-image.js";

View File

@@ -0,0 +1,135 @@
import {
buildModelAliasIndex,
resolveModelRefFromString,
} from "../../agents/model-selection.js";
import { CONFIG_PATH_CLAWDBOT, loadConfig } from "../../config/config.js";
import type { RuntimeEnv } from "../../runtime.js";
import {
DEFAULT_PROVIDER,
ensureFlagCompatibility,
modelKey,
resolveModelTarget,
updateConfig,
} from "./shared.js";
export async function modelsImageFallbacksListCommand(
opts: { json?: boolean; plain?: boolean },
runtime: RuntimeEnv,
) {
ensureFlagCompatibility(opts);
const cfg = loadConfig();
const fallbacks = cfg.agent?.imageModelFallbacks ?? [];
if (opts.json) {
runtime.log(JSON.stringify({ fallbacks }, null, 2));
return;
}
if (opts.plain) {
for (const entry of fallbacks) runtime.log(entry);
return;
}
runtime.log(`Image fallbacks (${fallbacks.length}):`);
if (fallbacks.length === 0) {
runtime.log("- none");
return;
}
for (const entry of fallbacks) runtime.log(`- ${entry}`);
}
export async function modelsImageFallbacksAddCommand(
modelRaw: string,
runtime: RuntimeEnv,
) {
const updated = await updateConfig((cfg) => {
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const targetKey = modelKey(resolved.provider, resolved.model);
const aliasIndex = buildModelAliasIndex({
cfg,
defaultProvider: DEFAULT_PROVIDER,
});
const existing = cfg.agent?.imageModelFallbacks ?? [];
const existingKeys = existing
.map((entry) =>
resolveModelRefFromString({
raw: String(entry ?? ""),
defaultProvider: DEFAULT_PROVIDER,
aliasIndex,
}),
)
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
.map((entry) => modelKey(entry.ref.provider, entry.ref.model));
if (existingKeys.includes(targetKey)) return cfg;
return {
...cfg,
agent: {
...cfg.agent,
imageModelFallbacks: [...existing, targetKey],
},
};
});
runtime.log(`Updated ${CONFIG_PATH_CLAWDBOT}`);
runtime.log(
`Image fallbacks: ${(updated.agent?.imageModelFallbacks ?? []).join(", ")}`,
);
}
export async function modelsImageFallbacksRemoveCommand(
modelRaw: string,
runtime: RuntimeEnv,
) {
const updated = await updateConfig((cfg) => {
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const targetKey = modelKey(resolved.provider, resolved.model);
const aliasIndex = buildModelAliasIndex({
cfg,
defaultProvider: DEFAULT_PROVIDER,
});
const existing = cfg.agent?.imageModelFallbacks ?? [];
const filtered = existing.filter((entry) => {
const resolvedEntry = resolveModelRefFromString({
raw: String(entry ?? ""),
defaultProvider: DEFAULT_PROVIDER,
aliasIndex,
});
if (!resolvedEntry) return true;
return (
modelKey(resolvedEntry.ref.provider, resolvedEntry.ref.model) !==
targetKey
);
});
if (filtered.length === existing.length) {
throw new Error(`Image fallback not found: ${targetKey}`);
}
return {
...cfg,
agent: {
...cfg.agent,
imageModelFallbacks: filtered,
},
};
});
runtime.log(`Updated ${CONFIG_PATH_CLAWDBOT}`);
runtime.log(
`Image fallbacks: ${(updated.agent?.imageModelFallbacks ?? []).join(", ")}`,
);
}
export async function modelsImageFallbacksClearCommand(runtime: RuntimeEnv) {
await updateConfig((cfg) => ({
...cfg,
agent: {
...cfg.agent,
imageModelFallbacks: [],
},
}));
runtime.log(`Updated ${CONFIG_PATH_CLAWDBOT}`);
runtime.log("Image fallback list cleared.");
}

View File

@@ -120,6 +120,26 @@ const resolveConfiguredEntries = (cfg: ClawdbotConfig) => {
addEntry(resolved.ref, `fallback#${idx + 1}`);
});
const imageModelRaw = cfg.agent?.imageModel?.trim();
if (imageModelRaw) {
const resolved = resolveModelRefFromString({
raw: imageModelRaw,
defaultProvider: DEFAULT_PROVIDER,
aliasIndex,
});
if (resolved) addEntry(resolved.ref, "image");
}
(cfg.agent?.imageModelFallbacks ?? []).forEach((raw, idx) => {
const resolved = resolveModelRefFromString({
raw: String(raw ?? ""),
defaultProvider: DEFAULT_PROVIDER,
aliasIndex,
});
if (!resolved) return;
addEntry(resolved.ref, `img-fallback#${idx + 1}`);
});
(cfg.agent?.allowedModels ?? []).forEach((raw) => {
const parsed = parseModelRef(String(raw ?? ""), DEFAULT_PROVIDER);
if (!parsed) return;
@@ -375,6 +395,8 @@ export async function modelsStatusCommand(
const rawModel = cfg.agent?.model?.trim() ?? "";
const defaultLabel = rawModel || `${resolved.provider}/${resolved.model}`;
const fallbacks = cfg.agent?.modelFallbacks ?? [];
const imageModel = cfg.agent?.imageModel?.trim() ?? "";
const imageFallbacks = cfg.agent?.imageModelFallbacks ?? [];
const aliases = cfg.agent?.modelAliases ?? {};
const allowed = cfg.agent?.allowedModels ?? [];
@@ -386,6 +408,8 @@ export async function modelsStatusCommand(
defaultModel: defaultLabel,
resolvedDefault: `${resolved.provider}/${resolved.model}`,
fallbacks,
imageModel: imageModel || null,
imageFallbacks,
aliases,
allowed,
},
@@ -406,6 +430,12 @@ export async function modelsStatusCommand(
runtime.log(
`Fallbacks (${fallbacks.length || 0}): ${fallbacks.join(", ") || "-"}`,
);
runtime.log(`Image model: ${imageModel || "-"}`);
runtime.log(
`Image fallbacks (${imageFallbacks.length || 0}): ${
imageFallbacks.length ? imageFallbacks.join(", ") : "-"
}`,
);
runtime.log(
`Aliases (${Object.keys(aliases).length || 0}): ${
Object.keys(aliases).length

View File

@@ -49,6 +49,24 @@ function sortScanResults(results: ModelScanResult[]): ModelScanResult[] {
});
}
function sortImageResults(results: ModelScanResult[]): ModelScanResult[] {
return results.slice().sort((a, b) => {
const aLatency = a.image.latencyMs ?? Number.POSITIVE_INFINITY;
const bLatency = b.image.latencyMs ?? Number.POSITIVE_INFINITY;
if (aLatency !== bLatency) return aLatency - bLatency;
const aCtx = a.contextLength ?? 0;
const bCtx = b.contextLength ?? 0;
if (aCtx !== bCtx) return bCtx - aCtx;
const aParams = a.inferredParamB ?? 0;
const bParams = b.inferredParamB ?? 0;
if (aParams !== bParams) return bParams - aParams;
return a.modelRef.localeCompare(b.modelRef);
});
}
function buildScanHint(result: ModelScanResult): string {
const toolLabel = result.tool.ok
? `tool ${formatMs(result.tool.latencyMs)}`
@@ -71,8 +89,9 @@ function printScanSummary(results: ModelScanResult[], runtime: RuntimeEnv) {
const toolOk = results.filter((r) => r.tool.ok);
const imageOk = results.filter((r) => r.image.ok);
const toolImageOk = results.filter((r) => r.tool.ok && r.image.ok);
const imageOnly = imageOk.filter((r) => !r.tool.ok);
runtime.log(
`Scan results: tested ${results.length}, tool ok ${toolOk.length}, image ok ${imageOk.length}, tool+image ok ${toolImageOk.length}`,
`Scan results: tested ${results.length}, tool ok ${toolOk.length}, image ok ${imageOk.length}, tool+image ok ${toolImageOk.length}, image only ${imageOnly.length}`,
);
}
@@ -127,6 +146,7 @@ export async function modelsScanCommand(
yes?: boolean;
input?: boolean;
setDefault?: boolean;
setImage?: boolean;
json?: boolean;
},
runtime: RuntimeEnv,
@@ -177,12 +197,18 @@ export async function modelsScanCommand(
throw new Error("No tool-capable OpenRouter free models found.");
}
const sorted = sortScanResults(toolOk);
const imagePreferred = sorted.filter((entry) => entry.image.ok);
const preselectPool = imagePreferred.length > 0 ? imagePreferred : sorted;
const sorted = sortScanResults(results);
const toolSorted = sortScanResults(toolOk);
const imageOk = results.filter((entry) => entry.image.ok);
const imageSorted = sortImageResults(imageOk);
const imagePreferred = toolSorted.filter((entry) => entry.image.ok);
const preselectPool = imagePreferred.length > 0 ? imagePreferred : toolSorted;
const preselected = preselectPool
.slice(0, Math.floor(maxCandidates))
.map((entry) => entry.modelRef);
const imagePreselected = imageSorted
.slice(0, Math.floor(maxCandidates))
.map((entry) => entry.modelRef);
if (!opts.json) {
printScanSummary(results, runtime);
@@ -192,11 +218,12 @@ export async function modelsScanCommand(
const noInput = opts.input === false;
const canPrompt = process.stdin.isTTY && !opts.yes && !noInput && !opts.json;
let selected: string[] = preselected;
let selectedImages: string[] = imagePreselected;
if (canPrompt) {
const selection = await multiselect({
message: "Select fallback models (ordered)",
options: sorted.map((entry) => ({
options: toolSorted.map((entry) => ({
value: entry.modelRef,
label: entry.modelRef,
hint: buildScanHint(entry),
@@ -210,6 +237,24 @@ export async function modelsScanCommand(
}
selected = selection as string[];
if (imageSorted.length > 0) {
const imageSelection = await multiselect({
message: "Select image fallback models (ordered)",
options: imageSorted.map((entry) => ({
value: entry.modelRef,
label: entry.modelRef,
hint: buildScanHint(entry),
})),
initialValues: imagePreselected,
});
if (isCancel(imageSelection)) {
cancel("Model scan cancelled.");
runtime.exit(0);
}
selectedImages = imageSelection as string[];
}
} else if (!process.stdin.isTTY && !opts.yes && !noInput && !opts.json) {
throw new Error("Non-interactive scan: pass --yes to apply defaults.");
}
@@ -217,34 +262,58 @@ export async function modelsScanCommand(
if (selected.length === 0) {
throw new Error("No models selected for fallbacks.");
}
if (opts.setImage && selectedImages.length === 0) {
throw new Error("No image-capable models selected for image model.");
}
const updated = await updateConfig((cfg) => {
const next = {
const agent = {
...cfg.agent,
modelFallbacks: selected,
...(opts.setDefault ? { model: selected[0] } : {}),
...(opts.setImage && selectedImages.length > 0
? { imageModel: selectedImages[0] }
: {}),
} satisfies NonNullable<typeof cfg.agent>;
if (imageSorted.length > 0) {
agent.imageModelFallbacks = selectedImages;
}
return {
...cfg,
agent: {
...cfg.agent,
modelFallbacks: selected,
...(opts.setDefault ? { model: selected[0] } : {}),
},
agent,
};
return next;
});
const allowlist = buildAllowlistSet(updated);
const allowlistMissing =
allowlist.size > 0 ? selected.filter((entry) => !allowlist.has(entry)) : [];
const allowlistMissingImages =
allowlist.size > 0
? selectedImages.filter((entry) => !allowlist.has(entry))
: [];
if (opts.json) {
runtime.log(
JSON.stringify(
{
selected,
selectedImages,
setDefault: Boolean(opts.setDefault),
setImage: Boolean(opts.setImage),
results,
warnings:
allowlistMissing.length > 0
allowlistMissing.length > 0 || allowlistMissingImages.length > 0
? [
`Selected models not in agent.allowedModels: ${allowlistMissing.join(", ")}`,
...(allowlistMissing.length > 0
? [
`Selected models not in agent.allowedModels: ${allowlistMissing.join(", ")}`,
]
: []),
...(allowlistMissingImages.length > 0
? [
`Selected image models not in agent.allowedModels: ${allowlistMissingImages.join(", ")}`,
]
: []),
]
: [],
},
@@ -262,10 +331,23 @@ export async function modelsScanCommand(
),
);
}
if (allowlistMissingImages.length > 0) {
runtime.log(
warn(
`Warning: ${allowlistMissingImages.length} selected image models are not in agent.allowedModels and will be ignored by fallback: ${allowlistMissingImages.join(", ")}`,
),
);
}
runtime.log(`Updated ${CONFIG_PATH_CLAWDBOT}`);
runtime.log(`Fallbacks: ${selected.join(", ")}`);
if (selectedImages.length > 0) {
runtime.log(`Image fallbacks: ${selectedImages.join(", ")}`);
}
if (opts.setDefault) {
runtime.log(`Default model: ${selected[0]}`);
}
if (opts.setImage && selectedImages.length > 0) {
runtime.log(`Image model: ${selectedImages[0]}`);
}
}

View File

@@ -0,0 +1,34 @@
import { CONFIG_PATH_CLAWDBOT } from "../../config/config.js";
import type { RuntimeEnv } from "../../runtime.js";
import {
buildAllowlistSet,
modelKey,
resolveModelTarget,
updateConfig,
} from "./shared.js";
export async function modelsSetImageCommand(
modelRaw: string,
runtime: RuntimeEnv,
) {
const updated = await updateConfig((cfg) => {
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const allowlist = buildAllowlistSet(cfg);
if (allowlist.size > 0) {
const key = modelKey(resolved.provider, resolved.model);
if (!allowlist.has(key)) {
throw new Error(`Model ${key} is not in agent.allowedModels.`);
}
}
return {
...cfg,
agent: {
...cfg.agent,
imageModel: `${resolved.provider}/${resolved.model}`,
},
};
});
runtime.log(`Updated ${CONFIG_PATH_CLAWDBOT}`);
runtime.log(`Image model: ${updated.agent?.imageModel ?? modelRaw}`);
}