Fix gateway restart false timeouts on Debian/systemd (#34874)

* daemon(systemd): target sudo caller user scope

* test(systemd): cover sudo user scope commands

* infra(ports): fall back to ss when lsof missing

* test(ports): verify ss fallback listener detection

* cli(gateway): use probe fallback for restart health

* test(gateway): cover restart-health probe fallback
This commit is contained in:
Vincent Koc
2026-03-04 10:52:33 -08:00
committed by GitHub
parent 4cc293d084
commit 2b98cb6d8b
6 changed files with 311 additions and 49 deletions

View File

@@ -57,6 +57,30 @@ function parseLsofFieldOutput(output: string): PortListener[] {
return listeners;
}
async function enrichUnixListenerProcessInfo(listeners: PortListener[]): Promise<void> {
await Promise.all(
listeners.map(async (listener) => {
if (!listener.pid) {
return;
}
const [commandLine, user, parentPid] = await Promise.all([
resolveUnixCommandLine(listener.pid),
resolveUnixUser(listener.pid),
resolveUnixParentPid(listener.pid),
]);
if (commandLine) {
listener.commandLine = commandLine;
}
if (user) {
listener.user = user;
}
if (parentPid !== undefined) {
listener.ppid = parentPid;
}
}),
);
}
async function resolveUnixCommandLine(pid: number): Promise<string | undefined> {
const res = await runCommandSafe(["ps", "-p", String(pid), "-o", "command="]);
if (res.code !== 0) {
@@ -85,35 +109,45 @@ async function resolveUnixParentPid(pid: number): Promise<number | undefined> {
return Number.isFinite(parentPid) && parentPid > 0 ? parentPid : undefined;
}
async function readUnixListeners(
function parseSsListeners(output: string, port: number): PortListener[] {
const lines = output.split(/\r?\n/).map((line) => line.trim());
const listeners: PortListener[] = [];
for (const line of lines) {
if (!line || !line.includes("LISTEN")) {
continue;
}
const parts = line.split(/\s+/);
const localAddress = parts.find((part) => part.includes(`:${port}`));
if (!localAddress) {
continue;
}
const listener: PortListener = {
address: localAddress,
};
const pidMatch = line.match(/pid=(\d+)/);
if (pidMatch) {
const pid = Number.parseInt(pidMatch[1], 10);
if (Number.isFinite(pid)) {
listener.pid = pid;
}
}
const commandMatch = line.match(/users:\(\("([^"]+)"/);
if (commandMatch?.[1]) {
listener.command = commandMatch[1];
}
listeners.push(listener);
}
return listeners;
}
async function readUnixListenersFromSs(
port: number,
): Promise<{ listeners: PortListener[]; detail?: string; errors: string[] }> {
const errors: string[] = [];
const lsof = await resolveLsofCommand();
const res = await runCommandSafe([lsof, "-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-FpFcn"]);
const res = await runCommandSafe(["ss", "-H", "-ltnp", `sport = :${port}`]);
if (res.code === 0) {
const listeners = parseLsofFieldOutput(res.stdout);
await Promise.all(
listeners.map(async (listener) => {
if (!listener.pid) {
return;
}
const [commandLine, user, parentPid] = await Promise.all([
resolveUnixCommandLine(listener.pid),
resolveUnixUser(listener.pid),
resolveUnixParentPid(listener.pid),
]);
if (commandLine) {
listener.commandLine = commandLine;
}
if (user) {
listener.user = user;
}
if (parentPid !== undefined) {
listener.ppid = parentPid;
}
}),
);
const listeners = parseSsListeners(res.stdout, port);
await enrichUnixListenerProcessInfo(listeners);
return { listeners, detail: res.stdout.trim() || undefined, errors };
}
const stderr = res.stderr.trim();
@@ -130,6 +164,41 @@ async function readUnixListeners(
return { listeners: [], detail: undefined, errors };
}
async function readUnixListeners(
port: number,
): Promise<{ listeners: PortListener[]; detail?: string; errors: string[] }> {
const lsof = await resolveLsofCommand();
const res = await runCommandSafe([lsof, "-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-FpFcn"]);
if (res.code === 0) {
const listeners = parseLsofFieldOutput(res.stdout);
await enrichUnixListenerProcessInfo(listeners);
return { listeners, detail: res.stdout.trim() || undefined, errors: [] };
}
const lsofErrors: string[] = [];
const stderr = res.stderr.trim();
if (res.code === 1 && !res.error && !stderr) {
return { listeners: [], detail: undefined, errors: [] };
}
if (res.error) {
lsofErrors.push(res.error);
}
const detail = [stderr, res.stdout.trim()].filter(Boolean).join("\n");
if (detail) {
lsofErrors.push(detail);
}
const ssFallback = await readUnixListenersFromSs(port);
if (ssFallback.listeners.length > 0) {
return ssFallback;
}
return {
listeners: [],
detail: undefined,
errors: [...lsofErrors, ...ssFallback.errors],
};
}
function parseNetstatListeners(output: string, port: number): PortListener[] {
const listeners: PortListener[] = [];
const portToken = `:${port}`;

View File

@@ -111,4 +111,62 @@ describeUnix("inspectPortUsage", () => {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
it("falls back to ss when lsof is unavailable", async () => {
const server = net.createServer();
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const port = (server.address() as net.AddressInfo).port;
runCommandWithTimeoutMock.mockImplementation(async (argv: string[]) => {
const command = argv[0];
if (typeof command !== "string") {
return { stdout: "", stderr: "", code: 1 };
}
if (command.includes("lsof")) {
throw Object.assign(new Error("spawn lsof ENOENT"), { code: "ENOENT" });
}
if (command === "ss") {
return {
stdout: `LISTEN 0 511 127.0.0.1:${port} 0.0.0.0:* users:(("node",pid=${process.pid},fd=23))`,
stderr: "",
code: 0,
};
}
if (command === "ps") {
if (argv.includes("command=")) {
return {
stdout: "node /tmp/openclaw/dist/index.js gateway --port 18789\n",
stderr: "",
code: 0,
};
}
if (argv.includes("user=")) {
return {
stdout: "debian\n",
stderr: "",
code: 0,
};
}
if (argv.includes("ppid=")) {
return {
stdout: "1\n",
stderr: "",
code: 0,
};
}
}
return { stdout: "", stderr: "", code: 1 };
});
try {
const result = await inspectPortUsage(port);
expect(result.status).toBe("busy");
expect(result.listeners.length).toBeGreaterThan(0);
expect(result.listeners[0]?.pid).toBe(process.pid);
expect(result.listeners[0]?.commandLine).toContain("openclaw");
expect(result.errors).toBeUndefined();
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
});