mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-12 07:41:10 +00:00
refactor(core): dedupe embedding imports and env parsing
This commit is contained in:
16
src/memory/batch-embedding-common.ts
Normal file
16
src/memory/batch-embedding-common.ts
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
export { extractBatchErrorMessage, formatUnavailableBatchError } from "./batch-error-utils.js";
|
||||||
|
export { postJsonWithRetry } from "./batch-http.js";
|
||||||
|
export { applyEmbeddingBatchOutputLine } from "./batch-output.js";
|
||||||
|
export {
|
||||||
|
EMBEDDING_BATCH_ENDPOINT,
|
||||||
|
type EmbeddingBatchStatus,
|
||||||
|
type ProviderBatchOutputLine,
|
||||||
|
} from "./batch-provider-common.js";
|
||||||
|
export {
|
||||||
|
buildEmbeddingBatchGroupOptions,
|
||||||
|
runEmbeddingBatchGroups,
|
||||||
|
type EmbeddingBatchExecutionParams,
|
||||||
|
} from "./batch-runner.js";
|
||||||
|
export { uploadBatchJsonlFile } from "./batch-upload.js";
|
||||||
|
export { buildBatchHeaders, normalizeBatchBaseUrl } from "./batch-utils.js";
|
||||||
|
export { withRemoteHttpResponse } from "./remote-http.js";
|
||||||
@@ -1,20 +1,20 @@
|
|||||||
import { extractBatchErrorMessage, formatUnavailableBatchError } from "./batch-error-utils.js";
|
|
||||||
import { postJsonWithRetry } from "./batch-http.js";
|
|
||||||
import { applyEmbeddingBatchOutputLine } from "./batch-output.js";
|
|
||||||
import {
|
|
||||||
EMBEDDING_BATCH_ENDPOINT,
|
|
||||||
type EmbeddingBatchStatus,
|
|
||||||
type ProviderBatchOutputLine,
|
|
||||||
} from "./batch-provider-common.js";
|
|
||||||
import {
|
import {
|
||||||
|
applyEmbeddingBatchOutputLine,
|
||||||
|
buildBatchHeaders,
|
||||||
buildEmbeddingBatchGroupOptions,
|
buildEmbeddingBatchGroupOptions,
|
||||||
|
EMBEDDING_BATCH_ENDPOINT,
|
||||||
|
extractBatchErrorMessage,
|
||||||
|
formatUnavailableBatchError,
|
||||||
|
normalizeBatchBaseUrl,
|
||||||
|
postJsonWithRetry,
|
||||||
runEmbeddingBatchGroups,
|
runEmbeddingBatchGroups,
|
||||||
type EmbeddingBatchExecutionParams,
|
type EmbeddingBatchExecutionParams,
|
||||||
} from "./batch-runner.js";
|
type EmbeddingBatchStatus,
|
||||||
import { uploadBatchJsonlFile } from "./batch-upload.js";
|
type ProviderBatchOutputLine,
|
||||||
import { buildBatchHeaders, normalizeBatchBaseUrl } from "./batch-utils.js";
|
uploadBatchJsonlFile,
|
||||||
|
withRemoteHttpResponse,
|
||||||
|
} from "./batch-embedding-common.js";
|
||||||
import type { OpenAiEmbeddingClient } from "./embeddings-openai.js";
|
import type { OpenAiEmbeddingClient } from "./embeddings-openai.js";
|
||||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
|
||||||
|
|
||||||
export type OpenAiBatchRequest = {
|
export type OpenAiBatchRequest = {
|
||||||
custom_id: string;
|
custom_id: string;
|
||||||
|
|||||||
@@ -1,22 +1,22 @@
|
|||||||
import { createInterface } from "node:readline";
|
import { createInterface } from "node:readline";
|
||||||
import { Readable } from "node:stream";
|
import { Readable } from "node:stream";
|
||||||
import { extractBatchErrorMessage, formatUnavailableBatchError } from "./batch-error-utils.js";
|
|
||||||
import { postJsonWithRetry } from "./batch-http.js";
|
|
||||||
import { applyEmbeddingBatchOutputLine } from "./batch-output.js";
|
|
||||||
import {
|
|
||||||
EMBEDDING_BATCH_ENDPOINT,
|
|
||||||
type EmbeddingBatchStatus,
|
|
||||||
type ProviderBatchOutputLine,
|
|
||||||
} from "./batch-provider-common.js";
|
|
||||||
import {
|
import {
|
||||||
|
applyEmbeddingBatchOutputLine,
|
||||||
|
buildBatchHeaders,
|
||||||
buildEmbeddingBatchGroupOptions,
|
buildEmbeddingBatchGroupOptions,
|
||||||
|
EMBEDDING_BATCH_ENDPOINT,
|
||||||
|
extractBatchErrorMessage,
|
||||||
|
formatUnavailableBatchError,
|
||||||
|
normalizeBatchBaseUrl,
|
||||||
|
postJsonWithRetry,
|
||||||
runEmbeddingBatchGroups,
|
runEmbeddingBatchGroups,
|
||||||
type EmbeddingBatchExecutionParams,
|
type EmbeddingBatchExecutionParams,
|
||||||
} from "./batch-runner.js";
|
type EmbeddingBatchStatus,
|
||||||
import { uploadBatchJsonlFile } from "./batch-upload.js";
|
type ProviderBatchOutputLine,
|
||||||
import { buildBatchHeaders, normalizeBatchBaseUrl } from "./batch-utils.js";
|
uploadBatchJsonlFile,
|
||||||
|
withRemoteHttpResponse,
|
||||||
|
} from "./batch-embedding-common.js";
|
||||||
import type { VoyageEmbeddingClient } from "./embeddings-voyage.js";
|
import type { VoyageEmbeddingClient } from "./embeddings-voyage.js";
|
||||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Voyage Batch API Input Line format.
|
* Voyage Batch API Input Line format.
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import {
|
|||||||
import { listKnownSecretEnvVarNames } from "./provider-env-vars.js";
|
import { listKnownSecretEnvVarNames } from "./provider-env-vars.js";
|
||||||
import { resolveSecretRefValue } from "./resolve.js";
|
import { resolveSecretRefValue } from "./resolve.js";
|
||||||
import { prepareSecretsRuntimeSnapshot } from "./runtime.js";
|
import { prepareSecretsRuntimeSnapshot } from "./runtime.js";
|
||||||
import { isNonEmptyString, isRecord, writeTextFileAtomic } from "./shared.js";
|
import { isNonEmptyString, isRecord, parseEnvValue, writeTextFileAtomic } from "./shared.js";
|
||||||
|
|
||||||
type FileSnapshot = {
|
type FileSnapshot = {
|
||||||
existed: boolean;
|
existed: boolean;
|
||||||
@@ -118,17 +118,6 @@ function resolveTargetPathSegments(target: SecretsPlanTarget): string[] {
|
|||||||
return resolved;
|
return resolved;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseEnvValue(raw: string): string {
|
|
||||||
const trimmed = raw.trim();
|
|
||||||
if (
|
|
||||||
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
|
||||||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
|
||||||
) {
|
|
||||||
return trimmed.slice(1, -1);
|
|
||||||
}
|
|
||||||
return trimmed;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrubEnvRaw(
|
function scrubEnvRaw(
|
||||||
raw: string,
|
raw: string,
|
||||||
migratedValues: Set<string>,
|
migratedValues: Set<string>,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import {
|
|||||||
resolveSecretRefValues,
|
resolveSecretRefValues,
|
||||||
type SecretRefResolveCache,
|
type SecretRefResolveCache,
|
||||||
} from "./resolve.js";
|
} from "./resolve.js";
|
||||||
import { isNonEmptyString, isRecord } from "./shared.js";
|
import { isNonEmptyString, isRecord, parseEnvValue } from "./shared.js";
|
||||||
|
|
||||||
export type SecretsAuditCode =
|
export type SecretsAuditCode =
|
||||||
| "PLAINTEXT_FOUND"
|
| "PLAINTEXT_FOUND"
|
||||||
@@ -116,17 +116,6 @@ function parseDotPath(pathname: string): string[] {
|
|||||||
return pathname.split(".").filter(Boolean);
|
return pathname.split(".").filter(Boolean);
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseEnvValue(raw: string): string {
|
|
||||||
const trimmed = raw.trim();
|
|
||||||
if (
|
|
||||||
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
|
||||||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
|
||||||
) {
|
|
||||||
return trimmed.slice(1, -1);
|
|
||||||
}
|
|
||||||
return trimmed;
|
|
||||||
}
|
|
||||||
|
|
||||||
function collectEnvPlaintext(params: { envPath: string; collector: AuditCollector }): void {
|
function collectEnvPlaintext(params: { envPath: string; collector: AuditCollector }): void {
|
||||||
if (!fs.existsSync(params.envPath)) {
|
if (!fs.existsSync(params.envPath)) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -9,6 +9,17 @@ export function isNonEmptyString(value: unknown): value is string {
|
|||||||
return typeof value === "string" && value.trim().length > 0;
|
return typeof value === "string" && value.trim().length > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function parseEnvValue(raw: string): string {
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
if (
|
||||||
|
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
||||||
|
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
||||||
|
) {
|
||||||
|
return trimmed.slice(1, -1);
|
||||||
|
}
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
export function normalizePositiveInt(value: unknown, fallback: number): number {
|
export function normalizePositiveInt(value: unknown, fallback: number): number {
|
||||||
if (typeof value === "number" && Number.isFinite(value)) {
|
if (typeof value === "number" && Number.isFinite(value)) {
|
||||||
return Math.max(1, Math.floor(value));
|
return Math.max(1, Math.floor(value));
|
||||||
|
|||||||
Reference in New Issue
Block a user