github/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-08 13:11:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve web_fetch readability extraction

Browse Source
This commit is contained in:
Peter Steinberger
2026-01-16 23:17:55 +00:00
parent 9aad6dfe1b
commit 37fa4f7eef
9 changed files with 242 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/agents/tools/web-tools.readability.test.ts Normal file
View File

@@ -0,0 +1,49 @@
import { describe, expect, it } from "vitest";
import { extractReadableContent } from "./web-tools.js";
const SAMPLE_HTML = `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>Example Article</title>
</head>
<body>
<nav>
<ul>
<li><a href="/home">Home</a></li>
<li><a href="/about">About</a></li>
</ul>
</nav>
<main>
<article>
<h1>Example Article</h1>
<p>Main content starts here with enough words to satisfy readability.</p>
<p>Second paragraph for a bit more signal.</p>
</article>
</main>
<footer>Footer text</footer>
</body>
</html>`;
describe("web fetch readability", () => {
it("extracts readable text", async () => {
const result = await extractReadableContent({
html: SAMPLE_HTML,
url: "https://example.com/article",
extractMode: "text",
});
expect(result?.text).toContain("Main content starts here");
expect(result?.title).toBe("Example Article");
});
it("extracts readable markdown", async () => {
const result = await extractReadableContent({
html: SAMPLE_HTML,
url: "https://example.com/article",
extractMode: "markdown",
});
expect(result?.text).toContain("Main content starts here");
expect(result?.title).toBe("Example Article");
});
});