mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-08 11:51:23 +00:00
refactor(infra): share jsonl transcript reader
This commit is contained in:
@@ -212,39 +212,52 @@ const applyCostTotal = (totals: CostUsageTotals, costTotal: number | undefined)
|
|||||||
totals.totalCost += costTotal;
|
totals.totalCost += costTotal;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
async function* readJsonlRecords(filePath: string): AsyncGenerator<Record<string, unknown>> {
|
||||||
|
const fileStream = fs.createReadStream(filePath, { encoding: "utf-8" });
|
||||||
|
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
||||||
|
try {
|
||||||
|
for await (const line of rl) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(trimmed) as unknown;
|
||||||
|
if (!parsed || typeof parsed !== "object") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
yield parsed as Record<string, unknown>;
|
||||||
|
} catch {
|
||||||
|
// Ignore malformed lines
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
rl.close();
|
||||||
|
fileStream.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function scanTranscriptFile(params: {
|
async function scanTranscriptFile(params: {
|
||||||
filePath: string;
|
filePath: string;
|
||||||
config?: OpenClawConfig;
|
config?: OpenClawConfig;
|
||||||
onEntry: (entry: ParsedTranscriptEntry) => void;
|
onEntry: (entry: ParsedTranscriptEntry) => void;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
const fileStream = fs.createReadStream(params.filePath, { encoding: "utf-8" });
|
for await (const parsed of readJsonlRecords(params.filePath)) {
|
||||||
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
const entry = parseTranscriptEntry(parsed);
|
||||||
|
if (!entry) {
|
||||||
for await (const line of rl) {
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (!trimmed) {
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
|
||||||
const entry = parseTranscriptEntry(parsed);
|
|
||||||
if (!entry) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (entry.usage && entry.costTotal === undefined) {
|
if (entry.usage && entry.costTotal === undefined) {
|
||||||
const cost = resolveModelCostConfig({
|
const cost = resolveModelCostConfig({
|
||||||
provider: entry.provider,
|
provider: entry.provider,
|
||||||
model: entry.model,
|
model: entry.model,
|
||||||
config: params.config,
|
config: params.config,
|
||||||
});
|
});
|
||||||
entry.costTotal = estimateUsageCost({ usage: entry.usage, cost });
|
entry.costTotal = estimateUsageCost({ usage: entry.usage, cost });
|
||||||
}
|
|
||||||
|
|
||||||
params.onEntry(entry);
|
|
||||||
} catch {
|
|
||||||
// Ignore malformed lines
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
params.onEntry(entry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -400,16 +413,8 @@ export async function discoverAllSessions(params?: {
|
|||||||
// Try to read first user message for label extraction
|
// Try to read first user message for label extraction
|
||||||
let firstUserMessage: string | undefined;
|
let firstUserMessage: string | undefined;
|
||||||
try {
|
try {
|
||||||
const fileStream = fs.createReadStream(filePath, { encoding: "utf-8" });
|
for await (const parsed of readJsonlRecords(filePath)) {
|
||||||
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
||||||
|
|
||||||
for await (const line of rl) {
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (!trimmed) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
|
||||||
const message = parsed.message as Record<string, unknown> | undefined;
|
const message = parsed.message as Record<string, unknown> | undefined;
|
||||||
if (message?.role === "user") {
|
if (message?.role === "user") {
|
||||||
const content = message.content;
|
const content = message.content;
|
||||||
@@ -436,8 +441,6 @@ export async function discoverAllSessions(params?: {
|
|||||||
// Skip malformed lines
|
// Skip malformed lines
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rl.close();
|
|
||||||
fileStream.destroy();
|
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore read errors
|
// Ignore read errors
|
||||||
}
|
}
|
||||||
@@ -831,16 +834,8 @@ export async function loadSessionLogs(params: {
|
|||||||
const logs: SessionLogEntry[] = [];
|
const logs: SessionLogEntry[] = [];
|
||||||
const limit = params.limit ?? 50;
|
const limit = params.limit ?? 50;
|
||||||
|
|
||||||
const fileStream = fs.createReadStream(sessionFile, { encoding: "utf-8" });
|
for await (const parsed of readJsonlRecords(sessionFile)) {
|
||||||
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
||||||
|
|
||||||
for await (const line of rl) {
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (!trimmed) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
|
||||||
const message = parsed.message as Record<string, unknown> | undefined;
|
const message = parsed.message as Record<string, unknown> | undefined;
|
||||||
if (!message) {
|
if (!message) {
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
Reference in New Issue
Block a user