This commit is contained in:
George Pickett
2026-03-04 09:29:39 -08:00
parent 0e3fe7d367
commit c12840ff25
21 changed files with 2118 additions and 587 deletions
+2
View File
@@ -16,6 +16,8 @@
"build": "next build",
"start": "node server/index.js",
"lint": "eslint .",
"probe:agent-latency": "node scripts/probe-agent-history-latency.mjs",
"probe:fleet-latency": "node scripts/probe-fleet-latency.mjs",
"cleanup:ux-artifacts": "node scripts/cleanup-ux-artifacts.mjs",
"studio:setup": "node scripts/studio-setup.js",
"smoke:dev-server": "node scripts/smoke-dev-server.mjs",
+560
View File
@@ -0,0 +1,560 @@
#!/usr/bin/env node
import { performance } from "node:perf_hooks";
import process from "node:process";
const DEFAULT_BASE_URL = "http://127.0.0.1:3000";
const DEFAULT_SAMPLES = 15;
const DEFAULT_WARMUP = 3;
const DEFAULT_TIMEOUT_MS = 15_000;
const DEFAULT_SLO_P95_MS = 750;
const DEFAULT_AGENT_ID = "main";
const asTrimmed = (value) => {
if (typeof value !== "string") return "";
return value.trim();
};
const parseNumericArg = (raw, label) => {
const normalized = asTrimmed(raw);
const parsed = Number(normalized);
if (!Number.isFinite(parsed) || parsed <= 0) {
throw new Error(`Invalid ${label}: ${raw}`);
}
return Math.floor(parsed);
};
export const parseProbeArgs = (argv) => {
const config = {
baseUrl: DEFAULT_BASE_URL,
agentId: null,
sessionKey: null,
samples: DEFAULT_SAMPLES,
warmup: DEFAULT_WARMUP,
timeoutMs: DEFAULT_TIMEOUT_MS,
sloP95Ms: DEFAULT_SLO_P95_MS,
json: false,
allowDisconnected: false,
};
for (let index = 0; index < argv.length; index += 1) {
const token = asTrimmed(argv[index]);
if (!token) continue;
if (token === "--json") {
config.json = true;
continue;
}
if (token === "--allow-disconnected") {
config.allowDisconnected = true;
continue;
}
const value = asTrimmed(argv[index + 1]);
if (!value) {
throw new Error(`Missing value for ${token}`);
}
if (token === "--base-url") {
config.baseUrl = value;
index += 1;
continue;
}
if (token === "--agent-id") {
config.agentId = value;
index += 1;
continue;
}
if (token === "--session-key") {
config.sessionKey = value;
index += 1;
continue;
}
if (token === "--samples") {
config.samples = parseNumericArg(value, "--samples");
index += 1;
continue;
}
if (token === "--warmup") {
config.warmup = parseNumericArg(value, "--warmup");
index += 1;
continue;
}
if (token === "--timeout-ms") {
config.timeoutMs = parseNumericArg(value, "--timeout-ms");
index += 1;
continue;
}
if (token === "--slo-p95-ms") {
config.sloP95Ms = parseNumericArg(value, "--slo-p95-ms");
index += 1;
continue;
}
throw new Error(`Unknown argument: ${token}`);
}
return {
...config,
baseUrl: config.baseUrl.replace(/\/+$/, "") || DEFAULT_BASE_URL,
agentId: config.agentId ? config.agentId.trim() : null,
sessionKey: config.sessionKey ? config.sessionKey.trim() : null,
};
};
export const parseAgentIdFromSessionKey = (sessionKey) => {
const normalized = asTrimmed(sessionKey);
if (!normalized) return null;
const match = normalized.match(/^agent:([^:]+):/i);
const agentId = asTrimmed(match?.[1] ?? "");
return agentId || null;
};
export const resolveTargetFromFleet = (params) => {
const explicitAgentId = asTrimmed(params.explicitAgentId ?? "");
const explicitSessionKey = asTrimmed(params.explicitSessionKey ?? "");
const fromSessionKey = parseAgentIdFromSessionKey(explicitSessionKey) ?? "";
const suggestedAgentId = asTrimmed(params.fleetResult?.suggestedSelectedAgentId ?? "");
const firstSeedAgentId = asTrimmed(params.fleetResult?.seeds?.[0]?.agentId ?? "");
const agentId =
explicitAgentId ||
fromSessionKey ||
suggestedAgentId ||
firstSeedAgentId ||
DEFAULT_AGENT_ID;
const sessionKey = explicitSessionKey || `agent:${agentId}:main`;
return { agentId, sessionKey };
};
export const buildProbePaths = ({ agentId, sessionKey }) => {
const normalizedAgentId = encodeURIComponent(asTrimmed(agentId));
const normalizedSessionKey = encodeURIComponent(asTrimmed(sessionKey));
return [
{
name: "summary",
method: "GET",
path: "/api/runtime/summary",
sloBlocking: false,
},
{
name: "semantic-history",
method: "GET",
path: `/api/runtime/agents/${normalizedAgentId}/history?limit=50&view=semantic&turnLimit=50&scanLimit=800`,
sloBlocking: true,
},
{
name: "chat-history",
method: "GET",
path: `/api/runtime/chat-history?sessionKey=${normalizedSessionKey}&limit=50`,
sloBlocking: true,
},
];
};
export const percentile = (durations, p) => {
if (!Array.isArray(durations) || durations.length === 0) return null;
const sorted = [...durations].sort((a, b) => a - b);
const rank = Math.ceil((p / 100) * sorted.length);
const index = Math.max(0, Math.min(sorted.length - 1, rank - 1));
return sorted[index];
};
export const summarizeDurations = (durations, attempts) => {
if (!Array.isArray(durations) || durations.length === 0) {
return {
attempts,
count: 0,
minMs: null,
maxMs: null,
meanMs: null,
p50Ms: null,
p90Ms: null,
p95Ms: null,
};
}
const count = durations.length;
const total = durations.reduce((acc, value) => acc + value, 0);
return {
attempts,
count,
minMs: Math.min(...durations),
maxMs: Math.max(...durations),
meanMs: total / count,
p50Ms: percentile(durations, 50),
p90Ms: percentile(durations, 90),
p95Ms: percentile(durations, 95),
};
};
export const classifyBottleneckHint = (params) => {
const hasErrors = params.endpoints.some((entry) => (entry.errors?.count ?? 0) > 0);
if (hasErrors) {
return "errors present -> fix endpoint failures before latency diagnosis";
}
const semantic = params.endpoints.find((entry) => entry.name === "semantic-history") ?? null;
const chat = params.endpoints.find((entry) => entry.name === "chat-history") ?? null;
if (!semantic || !chat) {
return "incomplete probe results.";
}
const semanticP95 = semantic.stats.p95Ms;
const chatP95 = chat.stats.p95Ms;
const semanticSlow = typeof semanticP95 === "number" && semanticP95 > params.sloP95Ms;
const chatSlow = typeof chatP95 === "number" && chatP95 > params.sloP95Ms;
if (semanticSlow && !chatSlow) {
return "semantic slow, chat-history fast -> projection/index issue likely";
}
if (chatSlow && !semanticSlow) {
return "chat-history slow, semantic fast -> gateway/transcript read likely";
}
if (semanticSlow && chatSlow) {
return "both slow -> upstream/gateway saturation or host contention";
}
return "semantic and chat-history are within SLO";
};
export const assessProbe = (params) => {
const errorsPresent = params.endpoints.some((entry) => entry.errors.count > 0);
const blockingSloBreach = params.endpoints.some((entry) => {
if (!entry.sloBlocking) return false;
const p95 = entry.stats.p95Ms;
return typeof p95 === "number" && p95 > params.sloP95Ms;
});
return {
pass: !errorsPresent && !blockingSloBreach,
bottleneckHint: classifyBottleneckHint(params),
};
};
export const assessRuntimePreflight = ({ response, allowDisconnected }) => {
if (!response.ok) {
return {
pass: false,
connected: false,
status: null,
message: `runtime preflight failed: unable to read /api/runtime/summary (${response.error || response.status})`,
};
}
const payload = response.body;
const summary = payload && typeof payload === "object" ? payload.summary : null;
const runtimeStatus =
summary && typeof summary === "object"
? asTrimmed(summary.status ?? "")
: "";
const normalizedStatus = runtimeStatus || "unknown";
const connected = normalizedStatus === "connected";
if (connected) {
return {
pass: true,
connected: true,
status: normalizedStatus,
message: null,
};
}
if (allowDisconnected) {
return {
pass: true,
connected: false,
status: normalizedStatus,
message: `runtime preflight warning: summary.status="${normalizedStatus}"; continuing because --allow-disconnected is set`,
};
}
return {
pass: false,
connected: false,
status: normalizedStatus,
message:
`runtime preflight failed: summary.status="${normalizedStatus}". ` +
"Latency samples are invalid for SLO enforcement while disconnected. " +
"Reconnect runtime or rerun with --allow-disconnected.",
};
};
const toFixedMs = (value) => {
if (typeof value !== "number" || !Number.isFinite(value)) return "-";
return `${value.toFixed(1)}ms`;
};
const readErrorSnippet = async (response) => {
try {
const text = await response.text();
const normalized = text.replace(/\s+/g, " ").trim();
return normalized.slice(0, 240);
} catch {
return "";
}
};
const timedRequest = async ({ baseUrl, endpoint, timeoutMs }) => {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
const startedAt = performance.now();
try {
const response = await fetch(`${baseUrl}${endpoint.path}`, {
method: endpoint.method,
signal: controller.signal,
headers: endpoint.method === "POST" ? { "Content-Type": "application/json" } : undefined,
body: endpoint.method === "POST" ? "{}" : undefined,
});
const durationMs = performance.now() - startedAt;
if (response.status !== 200) {
return {
ok: false,
status: response.status,
durationMs,
error: await readErrorSnippet(response),
};
}
return {
ok: true,
status: response.status,
durationMs,
body: await response.json(),
};
} catch (error) {
const durationMs = performance.now() - startedAt;
const message =
error instanceof Error ? error.message : "request_failed";
return { ok: false, status: 0, durationMs, error: message };
} finally {
clearTimeout(timer);
}
};
const loadFleetResult = async ({ baseUrl, timeoutMs }) => {
const endpoint = {
name: "fleet",
method: "POST",
path: "/api/runtime/fleet",
};
const response = await timedRequest({ baseUrl, endpoint, timeoutMs });
if (!response.ok) {
return { result: null, warning: `fleet probe failed: ${response.error || response.status}` };
}
const payload = response.body;
const result = payload && typeof payload === "object" ? payload.result ?? null : null;
return { result, warning: null };
};
const runEndpointProbe = async ({ baseUrl, endpoint, warmup, samples, timeoutMs }) => {
for (let index = 0; index < warmup; index += 1) {
await timedRequest({ baseUrl, endpoint, timeoutMs });
}
const durations = [];
let errorCount = 0;
let lastError = null;
for (let index = 0; index < samples; index += 1) {
const response = await timedRequest({ baseUrl, endpoint, timeoutMs });
if (response.ok) {
durations.push(response.durationMs);
continue;
}
errorCount += 1;
lastError = {
status: response.status,
message: response.error || "request_failed",
};
}
return {
name: endpoint.name,
path: endpoint.path,
sloBlocking: endpoint.sloBlocking,
status: errorCount > 0 ? "fail" : "ok",
stats: summarizeDurations(durations, samples),
errors: {
count: errorCount,
last: lastError,
},
};
};
const printHumanOutput = ({
target,
config,
endpoints,
assessment,
fleetWarning,
preflightMessage,
}) => {
if (preflightMessage) {
process.stdout.write(`${preflightMessage}\n`);
}
if (fleetWarning) {
process.stdout.write(`warning: ${fleetWarning}\n`);
}
process.stdout.write(
`target=${target.agentId} session=${target.sessionKey} baseUrl=${target.baseUrl}\n`
);
process.stdout.write(
`samples=${config.samples} warmup=${config.warmup} timeoutMs=${config.timeoutMs} sloP95Ms=${config.sloP95Ms}\n`
);
process.stdout.write(
"endpoint attempts ok errors p50 p95 mean min max\n"
);
for (const endpoint of endpoints) {
const row = [
endpoint.name.padEnd(18, " "),
String(endpoint.stats.attempts).padStart(8, " "),
String(endpoint.stats.count).padStart(3, " "),
String(endpoint.errors.count).padStart(6, " "),
toFixedMs(endpoint.stats.p50Ms).padStart(8, " "),
toFixedMs(endpoint.stats.p95Ms).padStart(8, " "),
toFixedMs(endpoint.stats.meanMs).padStart(8, " "),
toFixedMs(endpoint.stats.minMs).padStart(8, " "),
toFixedMs(endpoint.stats.maxMs).padStart(8, " "),
].join(" ");
process.stdout.write(`${row}\n`);
if (endpoint.errors.last) {
process.stdout.write(
` last_error: status=${endpoint.errors.last.status} message=${endpoint.errors.last.message}\n`
);
}
}
process.stdout.write(`diagnosis: ${assessment.bottleneckHint}\n`);
process.stdout.write(`result: ${assessment.pass ? "PASS" : "FAIL"}\n`);
};
export const runProbe = async (rawArgs) => {
const args = parseProbeArgs(rawArgs);
const preflightResponse = await timedRequest({
baseUrl: args.baseUrl,
endpoint: {
name: "summary-preflight",
method: "GET",
path: "/api/runtime/summary",
},
timeoutMs: args.timeoutMs,
});
const preflight = assessRuntimePreflight({
response: preflightResponse,
allowDisconnected: args.allowDisconnected,
});
if (!preflight.pass) {
if (args.json) {
process.stdout.write(
`${JSON.stringify(
{
target: {
baseUrl: args.baseUrl,
agentId: args.agentId,
sessionKey: args.sessionKey,
},
config: {
samples: args.samples,
warmup: args.warmup,
timeoutMs: args.timeoutMs,
sloP95Ms: args.sloP95Ms,
allowDisconnected: args.allowDisconnected,
},
endpoints: [],
assessment: {
pass: false,
bottleneckHint: "preflight failed",
},
preflight,
},
null,
2
)}\n`
);
} else {
process.stdout.write(`${preflight.message}\n`);
process.stdout.write("result: FAIL\n");
}
process.exitCode = 1;
return;
}
const fleetResolutionNeeded = !args.agentId && !parseAgentIdFromSessionKey(args.sessionKey);
const fleetLoaded = fleetResolutionNeeded
? await loadFleetResult({ baseUrl: args.baseUrl, timeoutMs: args.timeoutMs })
: { result: null, warning: null };
const target = resolveTargetFromFleet({
explicitAgentId: args.agentId,
explicitSessionKey: args.sessionKey,
fleetResult: fleetLoaded.result,
});
const endpoints = buildProbePaths(target);
const endpointResults = [];
for (const endpoint of endpoints) {
endpointResults.push(
await runEndpointProbe({
baseUrl: args.baseUrl,
endpoint,
warmup: args.warmup,
samples: args.samples,
timeoutMs: args.timeoutMs,
})
);
}
const assessment = assessProbe({
endpoints: endpointResults,
sloP95Ms: args.sloP95Ms,
});
const payload = {
target: {
baseUrl: args.baseUrl,
agentId: target.agentId,
sessionKey: target.sessionKey,
},
config: {
samples: args.samples,
warmup: args.warmup,
timeoutMs: args.timeoutMs,
sloP95Ms: args.sloP95Ms,
allowDisconnected: args.allowDisconnected,
},
endpoints: endpointResults.map((entry) => ({
name: entry.name,
path: entry.path,
status: entry.status,
stats: entry.stats,
errors: entry.errors,
})),
assessment,
preflight,
};
if (args.json) {
process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
} else {
printHumanOutput({
target: payload.target,
config: payload.config,
endpoints: endpointResults,
assessment: payload.assessment,
fleetWarning: fleetLoaded.warning,
preflightMessage: preflight.message,
});
}
if (!assessment.pass) {
process.exitCode = 1;
}
};
const isDirectRun = (() => {
const scriptArg = process.argv[1];
if (!scriptArg) return false;
try {
return new URL(`file://${scriptArg}`).pathname === new URL(import.meta.url).pathname;
} catch {
return false;
}
})();
if (isDirectRun) {
runProbe(process.argv.slice(2)).catch((error) => {
process.stderr.write(`${error instanceof Error ? error.stack || error.message : String(error)}\n`);
process.exitCode = 1;
});
}
+491
View File
@@ -0,0 +1,491 @@
#!/usr/bin/env node
import { performance } from "node:perf_hooks";
import process from "node:process";
const DEFAULT_BASE_URL = "http://127.0.0.1:3000";
const DEFAULT_SAMPLES = 15;
const DEFAULT_WARMUP = 3;
const DEFAULT_TIMEOUT_MS = 15_000;
const DEFAULT_SLO_P95_MS = 900;
const asTrimmed = (value) => {
if (typeof value !== "string") return "";
return value.trim();
};
const parseNumericArg = (raw, label) => {
const normalized = asTrimmed(raw);
const parsed = Number(normalized);
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new Error(`Invalid ${label}: ${raw}`);
}
return parsed;
};
export const parseProbeArgs = (argv) => {
const config = {
baseUrl: DEFAULT_BASE_URL,
samples: DEFAULT_SAMPLES,
warmup: DEFAULT_WARMUP,
timeoutMs: DEFAULT_TIMEOUT_MS,
sloP95Ms: DEFAULT_SLO_P95_MS,
json: false,
allowDisconnected: false,
};
for (let index = 0; index < argv.length; index += 1) {
const token = asTrimmed(argv[index]);
if (!token) continue;
if (token === "--json") {
config.json = true;
continue;
}
if (token === "--allow-disconnected") {
config.allowDisconnected = true;
continue;
}
const value = asTrimmed(argv[index + 1]);
if (!value || value.startsWith("--")) {
throw new Error(`Missing value for ${token}`);
}
if (token === "--base-url") {
config.baseUrl = value;
index += 1;
continue;
}
if (token === "--samples") {
config.samples = parseNumericArg(value, "--samples");
index += 1;
continue;
}
if (token === "--warmup") {
config.warmup = parseNumericArg(value, "--warmup");
index += 1;
continue;
}
if (token === "--timeout-ms") {
config.timeoutMs = parseNumericArg(value, "--timeout-ms");
index += 1;
continue;
}
if (token === "--slo-p95-ms") {
config.sloP95Ms = parseNumericArg(value, "--slo-p95-ms");
index += 1;
continue;
}
throw new Error(`Unknown argument: ${token}`);
}
return {
...config,
baseUrl: config.baseUrl.replace(/\/+$/, "") || DEFAULT_BASE_URL,
};
};
export const percentile = (durations, p) => {
if (!Array.isArray(durations) || durations.length === 0) return null;
const sorted = [...durations].sort((a, b) => a - b);
const rank = Math.ceil((p / 100) * sorted.length);
const index = Math.max(0, Math.min(sorted.length - 1, rank - 1));
return sorted[index];
};
export const summarizeDurations = (durations, attempts) => {
if (!Array.isArray(durations) || durations.length === 0) {
return {
attempts,
count: 0,
minMs: null,
maxMs: null,
meanMs: null,
p50Ms: null,
p90Ms: null,
p95Ms: null,
};
}
const count = durations.length;
const total = durations.reduce((acc, value) => acc + value, 0);
return {
attempts,
count,
minMs: Math.min(...durations),
maxMs: Math.max(...durations),
meanMs: total / count,
p50Ms: percentile(durations, 50),
p90Ms: percentile(durations, 90),
p95Ms: percentile(durations, 95),
};
};
export const assessRuntimePreflight = ({ response, allowDisconnected }) => {
if (!response.ok) {
return {
pass: false,
connected: false,
status: null,
message: `runtime preflight failed: unable to read /api/runtime/summary (${response.error || response.status})`,
};
}
const payload = response.body;
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
return {
pass: false,
connected: false,
status: null,
message: "runtime preflight failed: invalid /api/runtime/summary payload",
};
}
const summary = payload.summary;
if (!summary || typeof summary !== "object" || Array.isArray(summary)) {
return {
pass: false,
connected: false,
status: null,
message: "runtime preflight failed: missing summary in /api/runtime/summary payload",
};
}
const runtimeStatus = asTrimmed(summary.status ?? "");
if (!runtimeStatus) {
return {
pass: false,
connected: false,
status: null,
message: "runtime preflight failed: summary.status missing in /api/runtime/summary payload",
};
}
const connected = runtimeStatus === "connected";
if (connected) {
return {
pass: true,
connected: true,
status: runtimeStatus,
message: null,
};
}
if (allowDisconnected) {
return {
pass: true,
connected: false,
status: runtimeStatus,
message: `runtime preflight warning: summary.status=\"${runtimeStatus}\"; continuing because --allow-disconnected is set`,
};
}
return {
pass: false,
connected: false,
status: runtimeStatus,
message:
`runtime preflight failed: summary.status=\"${runtimeStatus}\". ` +
"Fleet latency samples are invalid for SLO enforcement while disconnected. " +
"Reconnect runtime or rerun with --allow-disconnected.",
};
};
export const assessFleetPayload = (body) => {
if (!body || typeof body !== "object" || Array.isArray(body)) {
return {
ok: false,
message: "invalid fleet response payload",
};
}
if (body.degraded === true) {
const code = typeof body.code === "string" ? body.code.trim() : "";
const reason = typeof body.reason === "string" ? body.reason.trim() : "";
const error = typeof body.error === "string" ? body.error.trim() : "";
const detail = [code, reason, error].filter(Boolean).join(" ").trim();
return {
ok: false,
message: detail ? `degraded fleet response: ${detail}` : "degraded fleet response",
};
}
return {
ok: true,
message: null,
};
};
export const classifyBottleneckHint = (params) => {
const hasErrors = params.endpoints.some((entry) => (entry.errors?.count ?? 0) > 0);
if (hasErrors) {
return "errors present -> fix endpoint failures before latency diagnosis";
}
const fleet = params.endpoints.find((entry) => entry.name === "fleet") ?? null;
if (!fleet) {
return "incomplete probe results.";
}
const fleetP95 = fleet.stats.p95Ms;
const fleetSlow = typeof fleetP95 === "number" && fleetP95 > params.sloP95Ms;
if (fleetSlow) {
return "fleet slow -> bootstrap hydration path likely bottleneck";
}
return "fleet latency is within SLO";
};
export const assessProbe = (params) => {
const errorsPresent = params.endpoints.some((entry) => entry.errors.count > 0);
const blockingSloBreach = params.endpoints.some((entry) => {
if (!entry.sloBlocking) return false;
const p95 = entry.stats.p95Ms;
return typeof p95 === "number" && p95 > params.sloP95Ms;
});
return {
pass: !errorsPresent && !blockingSloBreach,
bottleneckHint: classifyBottleneckHint(params),
};
};
const toFixedMs = (value) => {
if (typeof value !== "number" || !Number.isFinite(value)) return "-";
return `${value.toFixed(1)}ms`;
};
const readErrorSnippet = async (response) => {
try {
const text = await response.text();
const normalized = text.replace(/\s+/g, " ").trim();
return normalized.slice(0, 240);
} catch {
return "";
}
};
const timedRequest = async ({ baseUrl, endpoint, timeoutMs }) => {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
const startedAt = performance.now();
try {
const response = await fetch(`${baseUrl}${endpoint.path}`, {
method: endpoint.method,
signal: controller.signal,
headers: endpoint.method === "POST" ? { "Content-Type": "application/json" } : undefined,
body: endpoint.method === "POST" ? "{}" : undefined,
});
const durationMs = performance.now() - startedAt;
if (response.status !== 200) {
return {
ok: false,
status: response.status,
durationMs,
error: await readErrorSnippet(response),
};
}
return {
ok: true,
status: response.status,
durationMs,
body: await response.json(),
};
} catch (error) {
const durationMs = performance.now() - startedAt;
const message = error instanceof Error ? error.message : "request_failed";
return { ok: false, status: 0, durationMs, error: message };
} finally {
clearTimeout(timer);
}
};
const runEndpointProbe = async ({ baseUrl, endpoint, warmup, samples, timeoutMs }) => {
for (let index = 0; index < warmup; index += 1) {
await timedRequest({ baseUrl, endpoint, timeoutMs });
}
const durations = [];
let errorCount = 0;
let lastError = null;
for (let index = 0; index < samples; index += 1) {
const response = await timedRequest({ baseUrl, endpoint, timeoutMs });
if (!response.ok) {
errorCount += 1;
lastError = {
status: response.status,
message: response.error || "request_failed",
};
continue;
}
const payloadAssessment = assessFleetPayload(response.body);
if (!payloadAssessment.ok) {
errorCount += 1;
lastError = {
status: response.status,
message: payloadAssessment.message ?? "invalid_fleet_payload",
};
continue;
}
durations.push(response.durationMs);
}
return {
name: endpoint.name,
path: endpoint.path,
sloBlocking: endpoint.sloBlocking,
status: errorCount > 0 ? "fail" : "ok",
stats: summarizeDurations(durations, samples),
errors: {
count: errorCount,
last: lastError,
},
};
};
const printHumanOutput = ({ baseUrl, config, endpoint, assessment, preflightMessage }) => {
if (preflightMessage) {
process.stdout.write(`${preflightMessage}\n`);
}
process.stdout.write(`target=fleet baseUrl=${baseUrl}\n`);
process.stdout.write(
`samples=${config.samples} warmup=${config.warmup} timeoutMs=${config.timeoutMs} sloP95Ms=${config.sloP95Ms}\n`
);
process.stdout.write(
"endpoint attempts ok errors p50 p95 mean min max\n"
);
const row = [
endpoint.name.padEnd(8, " "),
String(endpoint.stats.attempts).padStart(8, " "),
String(endpoint.stats.count).padStart(3, " "),
String(endpoint.errors.count).padStart(6, " "),
toFixedMs(endpoint.stats.p50Ms).padStart(8, " "),
toFixedMs(endpoint.stats.p95Ms).padStart(8, " "),
toFixedMs(endpoint.stats.meanMs).padStart(8, " "),
toFixedMs(endpoint.stats.minMs).padStart(8, " "),
toFixedMs(endpoint.stats.maxMs).padStart(8, " "),
].join(" ");
process.stdout.write(`${row}\n`);
if (endpoint.errors.last) {
process.stdout.write(
` last_error: status=${endpoint.errors.last.status} message=${endpoint.errors.last.message}\n`
);
}
process.stdout.write(`diagnosis: ${assessment.bottleneckHint}\n`);
process.stdout.write(`result: ${assessment.pass ? "PASS" : "FAIL"}\n`);
};
export const runProbe = async (rawArgs) => {
const args = parseProbeArgs(rawArgs);
const preflightResponse = await timedRequest({
baseUrl: args.baseUrl,
endpoint: {
name: "summary-preflight",
method: "GET",
path: "/api/runtime/summary",
},
timeoutMs: args.timeoutMs,
});
const preflight = assessRuntimePreflight({
response: preflightResponse,
allowDisconnected: args.allowDisconnected,
});
if (!preflight.pass) {
if (args.json) {
process.stdout.write(
`${JSON.stringify(
{
target: { baseUrl: args.baseUrl },
config: {
samples: args.samples,
warmup: args.warmup,
timeoutMs: args.timeoutMs,
sloP95Ms: args.sloP95Ms,
allowDisconnected: args.allowDisconnected,
},
endpoints: [],
assessment: {
pass: false,
bottleneckHint: "preflight failed",
},
preflight,
},
null,
2
)}\n`
);
} else {
process.stdout.write(`${preflight.message}\n`);
process.stdout.write("result: FAIL\n");
}
process.exitCode = 1;
return;
}
const endpoint = {
name: "fleet",
method: "POST",
path: "/api/runtime/fleet",
sloBlocking: true,
};
const endpointResult = await runEndpointProbe({
baseUrl: args.baseUrl,
endpoint,
warmup: args.warmup,
samples: args.samples,
timeoutMs: args.timeoutMs,
});
const assessment = assessProbe({
endpoints: [endpointResult],
sloP95Ms: args.sloP95Ms,
});
const payload = {
target: {
baseUrl: args.baseUrl,
},
config: {
samples: args.samples,
warmup: args.warmup,
timeoutMs: args.timeoutMs,
sloP95Ms: args.sloP95Ms,
allowDisconnected: args.allowDisconnected,
},
endpoints: [
{
name: endpointResult.name,
path: endpointResult.path,
status: endpointResult.status,
stats: endpointResult.stats,
errors: endpointResult.errors,
},
],
assessment,
preflight,
};
if (args.json) {
process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
} else {
printHumanOutput({
baseUrl: payload.target.baseUrl,
config: payload.config,
endpoint: endpointResult,
assessment: payload.assessment,
preflightMessage: preflight.message,
});
}
if (!assessment.pass) {
process.exitCode = 1;
}
};
const isDirectRun = (() => {
const scriptArg = process.argv[1];
if (!scriptArg) return false;
try {
return new URL(`file://${scriptArg}`).pathname === new URL(import.meta.url).pathname;
} catch {
return false;
}
})();
if (isDirectRun) {
runProbe(process.argv.slice(2)).catch((error) => {
process.stderr.write(`${error instanceof Error ? error.stack || error.message : String(error)}\n`);
process.exitCode = 1;
});
}
@@ -1,6 +1,6 @@
import { NextResponse } from "next/server";
import { deriveRuntimeFreshness, probeOpenClawLocalState } from "@/lib/controlplane/degraded-read";
import { deriveRuntimeFreshness } from "@/lib/controlplane/degraded-read";
import { serializeRuntimeInitFailure } from "@/lib/controlplane/runtime-init-errors";
import { bootstrapDomainRuntime } from "@/lib/controlplane/runtime-route-bootstrap";
import {
@@ -126,7 +126,6 @@ export async function GET(
url.searchParams.get("beforeOutboxId"),
snapshot.outboxHead + 1
);
const probe = snapshot.status === "connected" ? null : await probeOpenClawLocalState();
const loadWindowWithBackfill = (
targetLimit: number
@@ -221,7 +220,6 @@ export async function GET(
semanticTurnsIncluded,
activeRun,
windowTruncated,
freshness: deriveRuntimeFreshness(snapshot, probe),
...(probe ? { probe } : {}),
freshness: deriveRuntimeFreshness(snapshot, null),
});
}
+3 -2
View File
@@ -1,6 +1,7 @@
import { NextResponse } from "next/server";
import { executeRuntimeGatewayRead } from "@/lib/controlplane/runtime-read-route";
import { clampGatewayChatHistoryLimit } from "@/lib/gateway/chatHistoryLimits";
export const runtime = "nodejs";
@@ -11,10 +12,10 @@ export async function GET(request: Request) {
return NextResponse.json({ error: "sessionKey is required." }, { status: 400 });
}
const limitRaw = (url.searchParams.get("limit") ?? "0").trim();
const limit = Number(limitRaw);
const limit = clampGatewayChatHistoryLimit(Number(limitRaw));
return await executeRuntimeGatewayRead("chat.history", {
sessionKey,
...(Number.isFinite(limit) && limit > 0 ? { limit: Math.floor(limit) } : {}),
...(typeof limit === "number" ? { limit } : {}),
});
}
+2 -4
View File
@@ -1,6 +1,6 @@
import { NextResponse } from "next/server";
import { deriveRuntimeFreshness, probeOpenClawLocalState } from "@/lib/controlplane/degraded-read";
import { deriveRuntimeFreshness } from "@/lib/controlplane/degraded-read";
import { serializeRuntimeInitFailure } from "@/lib/controlplane/runtime-init-errors";
import { bootstrapDomainRuntime } from "@/lib/controlplane/runtime-route-bootstrap";
@@ -24,12 +24,10 @@ export async function GET() {
const startError = bootstrap.kind === "start-failed" ? bootstrap.message : null;
const snapshot = controlPlane.snapshot();
const probe = snapshot.status === "connected" ? null : await probeOpenClawLocalState();
return NextResponse.json({
enabled: true,
...(startError ? { error: startError } : {}),
summary: snapshot,
freshness: deriveRuntimeFreshness(snapshot, probe),
...(probe ? { probe } : {}),
freshness: deriveRuntimeFreshness(snapshot, null),
});
}
@@ -1,4 +1,4 @@
import { buildAgentMainSessionKey, isSameSessionKey } from "@/lib/gateway/session-keys";
import { buildAgentMainSessionKey } from "@/lib/gateway/session-keys";
import { type GatewayModelPolicySnapshot } from "@/lib/gateway/models";
import { type StudioSettings } from "@/lib/studio/settings";
import {
@@ -128,29 +128,33 @@ export async function hydrateAgentFleetFromGateway(params: {
const mainKey = agentsResult.mainKey?.trim() || "main";
const mainSessionKeyByAgent = new Map<string, SessionsListEntry | null>();
await Promise.all(
agentsResult.agents.map(async (agent) => {
try {
if (agentsResult.agents.length > 0) {
try {
const sessions = await callGateway<SessionsListResult>(params.client, "sessions.list", {
includeGlobal: false,
includeUnknown: false,
search: `:${mainKey}`,
});
const entries = Array.isArray(sessions.sessions) ? sessions.sessions : [];
const bySessionKey = new Map<string, SessionsListEntry>();
for (const entry of entries) {
const key = typeof entry.key === "string" ? entry.key.trim() : "";
if (!key || bySessionKey.has(key)) continue;
bySessionKey.set(key, entry);
}
for (const agent of agentsResult.agents) {
const expectedMainKey = buildAgentMainSessionKey(agent.id, mainKey);
const sessions = await callGateway<SessionsListResult>(params.client, "sessions.list", {
agentId: agent.id,
includeGlobal: false,
includeUnknown: false,
search: expectedMainKey,
limit: 4,
});
const entries = Array.isArray(sessions.sessions) ? sessions.sessions : [];
const mainEntry =
entries.find((entry) => isSameSessionKey(entry.key ?? "", expectedMainKey)) ?? null;
mainSessionKeyByAgent.set(agent.id, mainEntry);
} catch (err) {
if (!params.isDisconnectLikeError(err)) {
logError("Failed to list sessions while resolving agent session.", err);
}
mainSessionKeyByAgent.set(agent.id, bySessionKey.get(expectedMainKey) ?? null);
}
} catch (err) {
if (!params.isDisconnectLikeError(err)) {
logError("Failed to list sessions while resolving fleet sessions.", err);
}
for (const agent of agentsResult.agents) {
mainSessionKeyByAgent.set(agent.id, null);
}
})
);
}
}
let statusSummary: SummaryStatusSnapshot | null = null;
let previewResult: SummaryPreviewSnapshot | null = null;
@@ -1,4 +1,5 @@
import type { AgentState } from "@/features/agents/state/store";
import { GATEWAY_CHAT_HISTORY_MAX_LIMIT } from "@/lib/gateway/chatHistoryLimits";
type HistoryRequestIntent =
| {
@@ -32,11 +33,12 @@ const resolveHistoryFetchLimit = (params: {
defaultLimit: number;
maxLimit: number;
}): number => {
const effectiveMax = Math.max(1, Math.min(params.maxLimit, GATEWAY_CHAT_HISTORY_MAX_LIMIT));
const requested = params.requestedLimit;
if (typeof requested !== "number" || !Number.isFinite(requested) || requested <= 0) {
return params.defaultLimit;
return Math.min(effectiveMax, Math.max(1, Math.floor(params.defaultLimit)));
}
return Math.min(params.maxLimit, Math.floor(requested));
return Math.min(effectiveMax, Math.floor(requested));
};
export const resolveHistoryRequestIntent = (params: {
@@ -1,13 +1,14 @@
import type { AgentState } from "@/features/agents/state/store";
import { GATEWAY_CHAT_HISTORY_MAX_LIMIT } from "@/lib/gateway/chatHistoryLimits";
type RuntimeSyncStatus = "disconnected" | "connecting" | "connected";
export const RUNTIME_SYNC_RECONCILE_INTERVAL_MS = 3000;
export const RUNTIME_SYNC_FOCUSED_HISTORY_INTERVAL_MS = 4500;
export const RUNTIME_SYNC_DEFAULT_HISTORY_LIMIT = 200;
export const RUNTIME_SYNC_MAX_HISTORY_LIMIT = 5000;
export const RUNTIME_SYNC_DEFAULT_HISTORY_LIMIT = 50;
export const RUNTIME_SYNC_MAX_HISTORY_LIMIT = GATEWAY_CHAT_HISTORY_MAX_LIMIT;
const RUNTIME_SYNC_MIN_LOAD_MORE_HISTORY_LIMIT = 400;
const RUNTIME_SYNC_MIN_LOAD_MORE_HISTORY_LIMIT = 100;
type RuntimeSyncHistoryBootstrapAgent = Pick<
AgentState,
@@ -20,17 +20,16 @@ import {
shouldRuntimeSyncContinueFocusedHistoryPolling,
} from "@/features/agents/operations/runtimeSyncControlWorkflow";
import {
buildDomainHistoryRunStatePatch,
type DomainHistoryActiveRun,
buildSummarySnapshotPatches,
type SummaryPreviewSnapshot,
type SummaryStatusSnapshot,
} from "@/features/agents/state/runtimeEventBridge";
import type { AgentState } from "@/features/agents/state/store";
import { TRANSCRIPT_V2_ENABLED, logTranscriptDebugMetric } from "@/features/agents/state/transcript";
import type { ControlPlaneOutboxEntry } from "@/lib/controlplane/contracts";
import { randomUUID } from "@/lib/uuid";
import { loadDomainChatHistory } from "@/lib/controlplane/domain-runtime-client";
import { GATEWAY_CHAT_HISTORY_MAX_LIMIT } from "@/lib/gateway/chatHistoryLimits";
import { fetchJson } from "@/lib/http";
import { randomUUID } from "@/lib/uuid";
type RuntimeSyncDispatchAction = {
type: "updateAgent";
@@ -65,53 +64,6 @@ type RuntimeSyncController = {
clearHistoryInFlight: (sessionKey: string) => void;
};
type DomainAgentHistoryResponse = {
entries?: unknown[];
hasMore?: unknown;
semanticTurnsIncluded?: unknown;
windowTruncated?: unknown;
activeRun?: unknown;
};
const DOMAIN_SEMANTIC_TURN_LIMIT = 50;
const DOMAIN_SEMANTIC_SCAN_LIMIT = 800;
const DOMAIN_CHAT_HISTORY_MAX_LIMIT = 1000;
type DomainChatHistoryEnvelope = {
ok?: unknown;
payload?: {
sessionKey?: unknown;
messages?: unknown;
} | null;
error?: unknown;
};
type DomainChatHistoryPayload = {
sessionKey: string;
messages: Record<string, unknown>[];
};
const asRecord = (value: unknown): Record<string, unknown> | null =>
value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: null;
const resolveDomainHistoryActiveRun = (value: unknown): DomainHistoryActiveRun | null => {
const record = asRecord(value);
if (!record) return null;
const statusRaw = record.status;
const status =
statusRaw === "running" || statusRaw === "idle" || statusRaw === "error"
? statusRaw
: null;
if (!status) return null;
const runIdRaw = record.runId;
const runId =
typeof runIdRaw === "string" ? (runIdRaw.trim() || null) : runIdRaw === null ? null : null;
const complete = record.complete === true;
return { runId, status, complete };
};
export function useRuntimeSyncController(
params: UseRuntimeSyncControllerParams
): RuntimeSyncController {
@@ -133,42 +85,6 @@ export function useRuntimeSyncController(
const defaultHistoryLimit = params.defaultHistoryLimit ?? RUNTIME_SYNC_DEFAULT_HISTORY_LIMIT;
const maxHistoryLimit = params.maxHistoryLimit ?? RUNTIME_SYNC_MAX_HISTORY_LIMIT;
const loadDomainChatHistory = useCallback(
async (params: { sessionKey: string; limit?: number }): Promise<DomainChatHistoryPayload> => {
const query = new URLSearchParams({ sessionKey: params.sessionKey.trim() });
if (typeof params.limit === "number" && Number.isFinite(params.limit) && params.limit > 0) {
const bounded = Math.min(Math.floor(params.limit), DOMAIN_CHAT_HISTORY_MAX_LIMIT);
query.set("limit", String(bounded));
}
const response = await fetchJson<DomainChatHistoryEnvelope>(
`/api/runtime/chat-history?${query.toString()}`,
{ cache: "no-store" }
);
if (response?.ok !== true) {
const message =
typeof response?.error === "string" ? response.error.trim() : "Domain chat history read failed.";
throw new Error(message || "Domain chat history read failed.");
}
const payload =
response.payload && typeof response.payload === "object"
? response.payload
: {};
const messages = Array.isArray(payload.messages)
? payload.messages.filter(
(entry): entry is Record<string, unknown> => Boolean(entry && typeof entry === "object")
)
: [];
return {
sessionKey:
typeof payload.sessionKey === "string" && payload.sessionKey.trim()
? payload.sessionKey.trim()
: params.sessionKey.trim(),
messages,
};
},
[]
);
useEffect(() => {
agentsRef.current = agents;
}, [agents]);
@@ -226,153 +142,54 @@ export function useRuntimeSyncController(
}
}, [client, dispatch, isDisconnectLikeError, useDomainApiReads]);
const loadAgentHistoryViaDomainApi = useCallback(
async (agentId: string, limit: number) => {
const normalizedAgentId = agentId.trim();
const encodedAgentId = encodeURIComponent(normalizedAgentId);
if (!encodedAgentId) return;
const boundedLimit = Math.min(Math.max(1, Math.floor(limit)), DOMAIN_CHAT_HISTORY_MAX_LIMIT);
const fetchPage = async (params: {
turnLimit?: number;
scanLimit?: number;
}): Promise<{
entries: ControlPlaneOutboxEntry[];
semanticTurnsIncluded: number | null;
windowTruncated: boolean;
activeRun: DomainHistoryActiveRun | null;
}> => {
const query = new URLSearchParams();
query.set("limit", String(boundedLimit));
query.set("view", "semantic");
query.set("turnLimit", String(params.turnLimit ?? DOMAIN_SEMANTIC_TURN_LIMIT));
query.set("scanLimit", String(params.scanLimit ?? DOMAIN_SEMANTIC_SCAN_LIMIT));
const result = await fetchJson<DomainAgentHistoryResponse>(
`/api/runtime/agents/${encodedAgentId}/history?${query.toString()}`,
{ cache: "no-store" }
);
const entries = Array.isArray(result.entries)
? (result.entries as ControlPlaneOutboxEntry[])
: [];
const semanticTurnsIncluded =
typeof result.semanticTurnsIncluded === "number" &&
Number.isFinite(result.semanticTurnsIncluded) &&
result.semanticTurnsIncluded >= 0
? Math.floor(result.semanticTurnsIncluded)
: null;
const windowTruncated =
result.windowTruncated === true ? true : result.hasMore === true;
const activeRun = resolveDomainHistoryActiveRun(result.activeRun);
return {
entries,
semanticTurnsIncluded,
windowTruncated,
activeRun,
};
};
const loadedAt = Date.now();
const firstPage = await fetchPage({
turnLimit: DOMAIN_SEMANTIC_TURN_LIMIT,
scanLimit: DOMAIN_SEMANTIC_SCAN_LIMIT,
});
logTranscriptDebugMetric("domain_history_semantic_window", {
agentId: normalizedAgentId,
turns: firstPage.semanticTurnsIncluded,
entries: firstPage.entries.length,
truncated: firstPage.windowTruncated,
});
const latestAgent =
agentsRef.current.find((entry) => entry.agentId === normalizedAgentId) ?? null;
if (
latestAgent?.sessionCreated &&
typeof latestAgent.sessionKey === "string" &&
latestAgent.sessionKey.trim()
) {
const commands = await runHistorySyncOperation({
client: {
call: async <T = unknown>(method: string, request: unknown) => {
if (method !== "chat.history") {
throw new Error(`Unsupported domain history method: ${method}`);
}
const body =
request && typeof request === "object"
? (request as { sessionKey?: unknown; limit?: unknown })
: {};
const sessionKey =
typeof body.sessionKey === "string" ? body.sessionKey.trim() : latestAgent.sessionKey.trim();
const requestedLimit =
typeof body.limit === "number" && Number.isFinite(body.limit) && body.limit > 0
? Math.floor(body.limit)
: undefined;
return (await loadDomainChatHistory({
sessionKey,
limit: requestedLimit,
})) as T;
},
},
agentId: normalizedAgentId,
requestedLimit: boundedLimit,
getAgent: (targetAgentId) =>
agentsRef.current.find((entry) => entry.agentId === targetAgentId) ?? null,
inFlightSessionKeys: historyInFlightRef.current,
requestId: randomUUID(),
loadedAt,
defaultLimit: defaultHistoryLimit,
maxLimit: maxHistoryLimit,
transcriptV2Enabled: TRANSCRIPT_V2_ENABLED,
allowTranscriptRevisionSkew: true,
});
executeHistorySyncCommands({
commands,
dispatch,
logMetric: (metric, meta) => logTranscriptDebugMetric(metric, meta),
isDisconnectLikeError,
logError: (message, error) => console.error(message, error),
});
}
const domainRunStatePatch =
firstPage.activeRun
? buildDomainHistoryRunStatePatch({
activeRun: firstPage.activeRun,
currentStatus: latestAgent?.status ?? "idle",
currentRunId: latestAgent?.runId ?? null,
})
: null;
dispatch({
type: "updateAgent",
agentId,
patch: {
historyLoadedAt: loadedAt,
historyFetchLimit: boundedLimit,
historyFetchedCount:
typeof firstPage.semanticTurnsIncluded === "number"
? firstPage.semanticTurnsIncluded
: firstPage.entries.length,
historyMaybeTruncated: firstPage.windowTruncated,
...(domainRunStatePatch ?? {}),
},
});
},
[
defaultHistoryLimit,
dispatch,
isDisconnectLikeError,
loadDomainChatHistory,
maxHistoryLimit,
]
);
const loadAgentHistory = useCallback(
async (agentId: string, options?: { limit?: number }) => {
if (useDomainApiReads) {
const agent = agentsRef.current.find((entry) => entry.agentId === agentId) ?? null;
const rawLimit =
typeof options?.limit === "number" && Number.isFinite(options.limit)
? Math.floor(options.limit)
: agent?.historyFetchLimit ?? defaultHistoryLimit;
const limit = Math.min(Math.max(1, rawLimit), maxHistoryLimit);
try {
await loadAgentHistoryViaDomainApi(agentId, limit);
const commands = await runHistorySyncOperation({
client: {
call: async <T = unknown>(method: string, request: unknown) => {
if (method !== "chat.history") {
throw new Error(`Unsupported domain history method: ${method}`);
}
const body =
request && typeof request === "object"
? (request as { sessionKey?: unknown; limit?: unknown })
: {};
const sessionKey =
typeof body.sessionKey === "string" ? body.sessionKey.trim() : "";
if (!sessionKey) {
throw new Error("Unsupported domain history request: missing sessionKey");
}
const requestedLimit =
typeof body.limit === "number" && Number.isFinite(body.limit) && body.limit > 0
? Math.floor(body.limit)
: undefined;
return (await loadDomainChatHistory({
sessionKey,
limit: requestedLimit,
})) as T;
},
},
agentId,
requestedLimit: options?.limit,
getAgent: (targetAgentId) =>
agentsRef.current.find((entry) => entry.agentId === targetAgentId) ?? null,
inFlightSessionKeys: historyInFlightRef.current,
requestId: randomUUID(),
loadedAt: Date.now(),
defaultLimit: defaultHistoryLimit,
maxLimit: Math.min(maxHistoryLimit, GATEWAY_CHAT_HISTORY_MAX_LIMIT),
transcriptV2Enabled: TRANSCRIPT_V2_ENABLED,
allowTranscriptRevisionSkew: true,
});
executeHistorySyncCommands({
commands,
dispatch,
logMetric: (metric, meta) => logTranscriptDebugMetric(metric, meta),
isDisconnectLikeError,
logError: (message, error) => console.error(message, error),
});
} catch (error) {
if (!isDisconnectLikeError(error)) {
console.error("Failed to load domain runtime history.", error);
@@ -406,7 +223,6 @@ export function useRuntimeSyncController(
defaultHistoryLimit,
dispatch,
isDisconnectLikeError,
loadAgentHistoryViaDomainApi,
maxHistoryLimit,
useDomainApiReads,
]
@@ -414,16 +230,6 @@ export function useRuntimeSyncController(
const loadMoreAgentHistory = useCallback(
(agentId: string) => {
if (useDomainApiReads) {
const agent = agentsRef.current.find((entry) => entry.agentId === agentId) ?? null;
const nextLimit = resolveRuntimeSyncLoadMoreHistoryLimit({
currentLimit: agent?.historyFetchLimit ?? null,
defaultLimit: defaultHistoryLimit,
maxLimit: maxHistoryLimit,
});
void loadAgentHistory(agentId, { limit: nextLimit });
return;
}
const agent = agentsRef.current.find((entry) => entry.agentId === agentId) ?? null;
const nextLimit = resolveRuntimeSyncLoadMoreHistoryLimit({
currentLimit: agent?.historyFetchLimit ?? null,
@@ -432,7 +238,7 @@ export function useRuntimeSyncController(
});
void loadAgentHistory(agentId, { limit: nextLimit });
},
[defaultHistoryLimit, loadAgentHistory, maxHistoryLimit, useDomainApiReads]
[defaultHistoryLimit, loadAgentHistory, maxHistoryLimit]
);
const reconcileRunningAgents = useCallback(async () => {
@@ -140,12 +140,6 @@ type HistorySyncPatchInput = {
runId: string | null;
};
export type DomainHistoryActiveRun = {
runId: string | null;
status: "running" | "idle" | "error";
complete: boolean;
};
type GatewayEventKind =
| "summary-refresh"
| "runtime-chat"
@@ -360,39 +354,6 @@ export const resolveHistoryRunStatePatch = (params: {
};
};
export const buildDomainHistoryRunStatePatch = (params: {
activeRun: DomainHistoryActiveRun | null;
currentStatus: AgentState["status"];
currentRunId: string | null;
}): Partial<AgentState> | null => {
const activeRun = params.activeRun;
if (!activeRun) return null;
if (activeRun.status === "running") {
const nextRunId = activeRun.runId?.trim() || params.currentRunId?.trim() || null;
if (params.currentStatus === "running" && nextRunId === (params.currentRunId?.trim() || null)) {
return null;
}
return {
status: "running",
runId: nextRunId,
sessionCreated: true,
};
}
if (params.currentStatus === activeRun.status && !params.currentRunId) {
return null;
}
return {
status: activeRun.status,
runId: null,
runStartedAt: null,
streamText: null,
thinkingTrace: null,
};
};
export const mergeHistoryWithPending = (
historyLines: string[],
currentLines: string[]
@@ -7,6 +7,7 @@ import type {
CronRunResult,
} from "@/lib/cron/types";
import type { SkillStatusReport } from "@/lib/skills/types";
import { clampGatewayChatHistoryLimit } from "@/lib/gateway/chatHistoryLimits";
type Envelope<T> = {
ok?: boolean;
@@ -208,8 +209,9 @@ export const loadDomainChatHistory = async (params: {
limit?: number;
}): Promise<ChatHistoryResult> => {
const query = new URLSearchParams({ sessionKey: params.sessionKey.trim() });
if (typeof params.limit === "number" && Number.isFinite(params.limit) && params.limit > 0) {
query.set("limit", String(Math.floor(params.limit)));
const boundedLimit = clampGatewayChatHistoryLimit(params.limit);
if (typeof boundedLimit === "number") {
query.set("limit", String(boundedLimit));
}
const result = await fetchJson<Envelope<ChatHistoryResult>>(
`/api/runtime/chat-history?${query.toString()}`,
+10
View File
@@ -0,0 +1,10 @@
export const GATEWAY_CHAT_HISTORY_MAX_LIMIT = 1000;
export const clampGatewayChatHistoryLimit = (
value: number | undefined
): number | undefined => {
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
return undefined;
}
return Math.min(GATEWAY_CHAT_HISTORY_MAX_LIMIT, Math.floor(value));
};
+190 -4
View File
@@ -12,7 +12,7 @@ describe("hydrateAgentFleetFromGateway", () => {
gateway: null,
focused: {},
avatars: {
[gatewayUrl]: {
"ws://localhost:18789": {
"agent-1": "persisted-seed",
},
},
@@ -61,16 +61,38 @@ describe("hydrateAgentFleetFromGateway", () => {
};
}
if (method === "sessions.list") {
const { agentId, search } = params as Record<string, unknown>;
const query = params as Record<string, unknown>;
expect(query.includeGlobal).toBe(false);
expect(query.includeUnknown).toBe(false);
expect(query.search).toBe(":main");
expect("limit" in query).toBe(false);
expect("includeDerivedTitles" in query).toBe(false);
expect("includeLastMessage" in query).toBe(false);
return {
sessions: [
{
key: search,
key: "agent:agent-2:main",
updatedAt: 1,
displayName: "Main",
thinkingLevel: "medium",
modelProvider: "openai",
model: agentId === "agent-2" ? "gpt-5" : "gpt-4.1",
model: "gpt-5",
},
{
key: "agent:agent-1:main",
updatedAt: 1,
displayName: "Main",
thinkingLevel: "medium",
modelProvider: "openai",
model: "gpt-4.1",
},
{
key: "agent:agent-3:work",
updatedAt: 1,
displayName: "Noise",
thinkingLevel: "low",
modelProvider: "openai",
model: "gpt-4.1",
},
],
};
@@ -117,6 +139,8 @@ describe("hydrateAgentFleetFromGateway", () => {
expect(call).toHaveBeenCalledWith("agents.list", {});
expect(call).toHaveBeenCalledWith("exec.approvals.get", {});
expect(call).toHaveBeenCalledTimes(6);
expect(call.mock.calls.filter(([method]) => method === "sessions.list")).toHaveLength(1);
expect(result.seeds).toHaveLength(2);
expect(result.seeds[0]).toEqual(
expect.objectContaining({
@@ -145,4 +169,166 @@ describe("hydrateAgentFleetFromGateway", () => {
expect(result.suggestedSelectedAgentId).toBe("agent-2");
expect(result.summaryPatches.length).toBeGreaterThan(0);
});
it("hydrates many agents with one sessions.list call", async () => {
const agentCount = 25;
const agents = Array.from({ length: agentCount }, (_, index) => ({
id: `agent-${index + 1}`,
name: `Agent ${index + 1}`,
identity: { avatarUrl: `https://example.com/${index + 1}.png` },
}));
const sessions = agents.map((agent) => ({
key: `agent:${agent.id}:main`,
updatedAt: 1,
displayName: `${agent.name} Main`,
thinkingLevel: "medium",
modelProvider: "openai",
model: "gpt-5",
}));
const call = vi.fn(async (method: string, params: unknown) => {
if (method === "agents.list") {
return {
defaultId: "agent-1",
mainKey: "main",
agents,
};
}
if (method === "sessions.list") {
const query = params as Record<string, unknown>;
expect(query).toEqual({
includeGlobal: false,
includeUnknown: false,
search: ":main",
});
return { sessions };
}
if (method === "exec.approvals.get") {
return { file: { agents: {} } };
}
if (method === "status") {
return {
sessions: {
recent: [],
byAgent: [],
},
};
}
if (method === "sessions.preview") {
return {
ts: 1,
previews: sessions.map((entry) => ({
key: entry.key,
status: "ok",
items: [{ role: "assistant", text: "ok", timestamp: "2026-03-01T00:00:00Z" }],
})),
};
}
if (method === "config.get") {
return {
hash: "hash-many",
config: { agents: { defaults: { model: "openai/gpt-5" }, list: [] } },
};
}
throw new Error(`Unhandled method: ${method}`);
});
const result = await hydrateAgentFleetFromGateway({
client: { call },
gatewayUrl: "ws://127.0.0.1:18789",
cachedConfigSnapshot: null,
loadStudioSettings: async () => ({ version: 1, gateway: null, focused: {}, avatars: {} }),
isDisconnectLikeError: () => false,
});
expect(call.mock.calls.filter(([method]) => method === "sessions.list")).toHaveLength(1);
expect(result.seeds).toHaveLength(agentCount);
expect(result.sessionCreatedAgentIds).toHaveLength(agentCount);
expect(result.sessionSettingsSyncedAgentIds).toHaveLength(agentCount);
});
it("returns safely when batched sessions.list fails", async () => {
const call = vi.fn(async (method: string) => {
if (method === "agents.list") {
return {
defaultId: "agent-1",
mainKey: "main",
agents: [
{ id: "agent-1", name: "One" },
{ id: "agent-2", name: "Two" },
],
};
}
if (method === "sessions.list") {
throw new Error("sessions list failed");
}
if (method === "exec.approvals.get") {
return { file: { agents: {} } };
}
if (method === "config.get") {
return {
hash: "hash-failure",
config: { agents: { defaults: { model: "openai/gpt-5" }, list: [] } },
};
}
throw new Error(`Unhandled method: ${method}`);
});
const logError = vi.fn();
const result = await hydrateAgentFleetFromGateway({
client: { call },
gatewayUrl: "ws://127.0.0.1:18789",
cachedConfigSnapshot: null,
loadStudioSettings: async () => ({ version: 1, gateway: null, focused: {}, avatars: {} }),
isDisconnectLikeError: () => false,
logError,
});
expect(call.mock.calls.filter(([method]) => method === "sessions.list")).toHaveLength(1);
expect(logError).toHaveBeenCalledWith(
"Failed to list sessions while resolving fleet sessions.",
expect.any(Error)
);
expect(result.sessionCreatedAgentIds).toEqual([]);
expect(result.sessionSettingsSyncedAgentIds).toEqual([]);
expect(result.summaryPatches).toEqual([]);
expect(result.suggestedSelectedAgentId).toBeNull();
expect(result.seeds).toHaveLength(2);
});
it("skips sessions.list when no agents are returned", async () => {
const call = vi.fn(async (method: string) => {
if (method === "agents.list") {
return {
defaultId: "main",
mainKey: "main",
agents: [],
};
}
if (method === "exec.approvals.get") {
return { file: { agents: {} } };
}
if (method === "config.get") {
return {
hash: "hash-empty",
config: { agents: { defaults: { model: "openai/gpt-5" }, list: [] } },
};
}
throw new Error(`Unhandled method: ${method}`);
});
const result = await hydrateAgentFleetFromGateway({
client: { call },
gatewayUrl: "ws://127.0.0.1:18789",
cachedConfigSnapshot: null,
loadStudioSettings: async () => ({ version: 1, gateway: null, focused: {}, avatars: {} }),
isDisconnectLikeError: () => false,
});
expect(call.mock.calls.filter(([method]) => method === "sessions.list")).toHaveLength(0);
expect(result.seeds).toEqual([]);
expect(result.sessionCreatedAgentIds).toEqual([]);
expect(result.sessionSettingsSyncedAgentIds).toEqual([]);
expect(result.summaryPatches).toEqual([]);
expect(result.suggestedSelectedAgentId).toBeNull();
});
});
+48
View File
@@ -0,0 +1,48 @@
// @vitest-environment node
import { beforeEach, describe, expect, it, vi } from "vitest";
const mockedExecuteRuntimeGatewayRead = vi.fn();
vi.mock("@/lib/controlplane/runtime-read-route", () => ({
executeRuntimeGatewayRead: (...args: unknown[]) =>
mockedExecuteRuntimeGatewayRead(...args),
}));
describe("/api/runtime/chat-history route", () => {
beforeEach(() => {
vi.resetModules();
mockedExecuteRuntimeGatewayRead.mockReset();
mockedExecuteRuntimeGatewayRead.mockResolvedValue(
new Response(JSON.stringify({ ok: true, payload: { messages: [] } }), {
status: 200,
headers: { "Content-Type": "application/json" },
})
);
});
it("returns 400 when sessionKey is missing", async () => {
const route = await import("@/app/api/runtime/chat-history/route");
const response = await route.GET(
new Request("http://localhost/api/runtime/chat-history")
);
expect(response.status).toBe(400);
expect(mockedExecuteRuntimeGatewayRead).not.toHaveBeenCalled();
});
it("forwards gateway reads with a capped limit", async () => {
const route = await import("@/app/api/runtime/chat-history/route");
const response = await route.GET(
new Request(
"http://localhost/api/runtime/chat-history?sessionKey=agent%3Amain%3Amain&limit=5000"
)
);
expect(response.status).toBe(200);
expect(mockedExecuteRuntimeGatewayRead).toHaveBeenCalledWith("chat.history", {
sessionKey: "agent:main:main",
limit: 1000,
});
});
});
+1 -1
View File
@@ -95,7 +95,7 @@ describe("historyLifecycleWorkflow", () => {
).toEqual({
kind: "fetch",
sessionKey: "agent:agent-1:main",
limit: 5000,
limit: 1000,
requestRevision: 14,
requestEpoch: 0,
requestId: "req-42",
+39
View File
@@ -73,6 +73,45 @@ describe("historySyncOperation", () => {
expect(commands).toEqual([{ kind: "noop", reason: "missing-agent" }]);
});
it("caps chat.history request limits at gateway maximum", async () => {
const agent = createAgent({
transcriptRevision: 1,
outputLines: ["> local question"],
});
const gatewayCalls: Array<{ method: string; params: unknown }> = [];
await runHistorySyncOperation({
client: {
call: async <T>(method: string, params: unknown) => {
gatewayCalls.push({ method, params });
return {
sessionKey: agent.sessionKey,
messages: [],
} as T;
},
},
agentId: "agent-1",
requestedLimit: 9_000,
getAgent: () => agent,
inFlightSessionKeys: new Set<string>(),
requestId: "req-cap-1",
loadedAt: 1_999,
defaultLimit: 200,
maxLimit: 5_000,
transcriptV2Enabled: true,
});
expect(gatewayCalls).toEqual([
{
method: "chat.history",
params: {
sessionKey: "agent:agent-1:main",
limit: 1000,
},
},
]);
});
it("applies history updates even when latest agent is running with active run", async () => {
const agent = createAgent({
status: "running",
+371
View File
@@ -0,0 +1,371 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import {
assessRuntimePreflight,
assessProbe,
buildProbePaths,
classifyBottleneckHint,
parseProbeArgs,
percentile,
resolveTargetFromFleet,
summarizeDurations,
} from "../../scripts/probe-agent-history-latency.mjs";
describe("probe-agent-history-latency", () => {
it("parses cli defaults", () => {
expect(parseProbeArgs([])).toEqual({
baseUrl: "http://127.0.0.1:3000",
agentId: null,
sessionKey: null,
samples: 15,
warmup: 3,
timeoutMs: 15_000,
sloP95Ms: 750,
json: false,
allowDisconnected: false,
});
});
it("parses cli overrides", () => {
expect(
parseProbeArgs([
"--base-url",
"http://localhost:3100/",
"--agent-id",
"alpha",
"--session-key",
"agent:alpha:main",
"--samples",
"20",
"--warmup",
"4",
"--timeout-ms",
"9000",
"--slo-p95-ms",
"600",
"--allow-disconnected",
"--json",
])
).toEqual({
baseUrl: "http://localhost:3100",
agentId: "alpha",
sessionKey: "agent:alpha:main",
samples: 20,
warmup: 4,
timeoutMs: 9000,
sloP95Ms: 600,
json: true,
allowDisconnected: true,
});
});
it("resolves target in priority order", () => {
expect(
resolveTargetFromFleet({
explicitAgentId: "explicit-agent",
explicitSessionKey: null,
fleetResult: {
suggestedSelectedAgentId: "suggested-agent",
seeds: [{ agentId: "seed-agent" }],
},
})
).toEqual({
agentId: "explicit-agent",
sessionKey: "agent:explicit-agent:main",
});
expect(
resolveTargetFromFleet({
explicitAgentId: null,
explicitSessionKey: null,
fleetResult: {
suggestedSelectedAgentId: "suggested-agent",
seeds: [{ agentId: "seed-agent" }],
},
})
).toEqual({
agentId: "suggested-agent",
sessionKey: "agent:suggested-agent:main",
});
expect(
resolveTargetFromFleet({
explicitAgentId: null,
explicitSessionKey: null,
fleetResult: {
suggestedSelectedAgentId: "",
seeds: [{ agentId: "seed-agent" }],
},
})
).toEqual({
agentId: "seed-agent",
sessionKey: "agent:seed-agent:main",
});
expect(
resolveTargetFromFleet({
explicitAgentId: null,
explicitSessionKey: null,
fleetResult: null,
})
).toEqual({
agentId: "main",
sessionKey: "agent:main:main",
});
});
it("builds tiered endpoint paths for one agent", () => {
expect(
buildProbePaths({
agentId: "main",
sessionKey: "agent:main:main",
}).map((entry) => entry.path)
).toEqual([
"/api/runtime/summary",
"/api/runtime/agents/main/history?limit=50&view=semantic&turnLimit=50&scanLimit=800",
"/api/runtime/chat-history?sessionKey=agent%3Amain%3Amain&limit=50",
]);
});
it("computes percentile and stats summaries", () => {
const durations = [100, 200, 300, 400, 500];
expect(percentile(durations, 50)).toBe(300);
expect(percentile(durations, 90)).toBe(500);
expect(percentile(durations, 95)).toBe(500);
expect(summarizeDurations(durations, 5)).toEqual({
attempts: 5,
count: 5,
minMs: 100,
maxMs: 500,
meanMs: 300,
p50Ms: 300,
p90Ms: 500,
p95Ms: 500,
});
});
it("classifies bottleneck hints from endpoint stats", () => {
expect(
classifyBottleneckHint({
endpoints: [
{
name: "summary",
stats: { p95Ms: null },
errors: { count: 1 },
},
{
name: "semantic-history",
stats: { p95Ms: null },
errors: { count: 0 },
},
{
name: "chat-history",
stats: { p95Ms: null },
errors: { count: 0 },
},
],
sloP95Ms: 750,
})
).toBe("errors present -> fix endpoint failures before latency diagnosis");
const baseEndpoints = [
{
name: "summary",
stats: { p95Ms: 1500 },
errors: { count: 0 },
},
{
name: "semantic-history",
stats: { p95Ms: 900 },
errors: { count: 0 },
},
{
name: "chat-history",
stats: { p95Ms: 300 },
errors: { count: 0 },
},
];
expect(
classifyBottleneckHint({
endpoints: baseEndpoints,
sloP95Ms: 750,
})
).toBe("semantic slow, chat-history fast -> projection/index issue likely");
expect(
classifyBottleneckHint({
endpoints: [
baseEndpoints[0],
{ name: "semantic-history", stats: { p95Ms: 300 }, errors: { count: 0 } },
{ name: "chat-history", stats: { p95Ms: 900 }, errors: { count: 0 } },
],
sloP95Ms: 750,
})
).toBe("chat-history slow, semantic fast -> gateway/transcript read likely");
expect(
classifyBottleneckHint({
endpoints: [
baseEndpoints[0],
{ name: "semantic-history", stats: { p95Ms: 900 }, errors: { count: 0 } },
{ name: "chat-history", stats: { p95Ms: 950 }, errors: { count: 0 } },
],
sloP95Ms: 750,
})
).toBe("both slow -> upstream/gateway saturation or host contention");
});
it("assesses pass/fail with endpoint errors and blocking slo breaches only", () => {
expect(
assessProbe({
sloP95Ms: 750,
endpoints: [
{
name: "summary",
sloBlocking: false,
stats: { p95Ms: 2_000 },
errors: { count: 0 },
},
{
name: "semantic-history",
sloBlocking: true,
stats: { p95Ms: 700 },
errors: { count: 0 },
},
{
name: "chat-history",
sloBlocking: true,
stats: { p95Ms: 700 },
errors: { count: 0 },
},
],
}).pass
).toBe(true);
expect(
assessProbe({
sloP95Ms: 750,
endpoints: [
{
name: "summary",
sloBlocking: false,
stats: { p95Ms: 300 },
errors: { count: 0 },
},
{
name: "semantic-history",
sloBlocking: true,
stats: { p95Ms: 900 },
errors: { count: 0 },
},
{
name: "chat-history",
sloBlocking: true,
stats: { p95Ms: 300 },
errors: { count: 0 },
},
],
}).pass
).toBe(false);
expect(
assessProbe({
sloP95Ms: 750,
endpoints: [
{
name: "summary",
sloBlocking: false,
stats: { p95Ms: 300 },
errors: { count: 1 },
},
{
name: "semantic-history",
sloBlocking: true,
stats: { p95Ms: 300 },
errors: { count: 0 },
},
{
name: "chat-history",
sloBlocking: true,
stats: { p95Ms: 300 },
errors: { count: 0 },
},
],
}).pass
).toBe(false);
});
it("enforces runtime connectivity preflight unless explicitly allowed", () => {
expect(
assessRuntimePreflight({
response: {
ok: true,
body: {
summary: { status: "connected" },
},
},
allowDisconnected: false,
})
).toEqual({
pass: true,
connected: true,
status: "connected",
message: null,
});
expect(
assessRuntimePreflight({
response: {
ok: true,
body: {
summary: { status: "stopped" },
},
},
allowDisconnected: false,
})
).toEqual({
pass: false,
connected: false,
status: "stopped",
message:
'runtime preflight failed: summary.status="stopped". Latency samples are invalid for SLO enforcement while disconnected. Reconnect runtime or rerun with --allow-disconnected.',
});
expect(
assessRuntimePreflight({
response: {
ok: true,
body: {
summary: { status: "stopped" },
},
},
allowDisconnected: true,
})
).toEqual({
pass: true,
connected: false,
status: "stopped",
message:
'runtime preflight warning: summary.status="stopped"; continuing because --allow-disconnected is set',
});
expect(
assessRuntimePreflight({
response: {
ok: false,
status: 503,
error: "service unavailable",
},
allowDisconnected: false,
})
).toEqual({
pass: false,
connected: false,
status: null,
message:
"runtime preflight failed: unable to read /api/runtime/summary (service unavailable)",
});
});
});
+214
View File
@@ -0,0 +1,214 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import {
assessFleetPayload,
assessRuntimePreflight,
assessProbe,
classifyBottleneckHint,
parseProbeArgs,
percentile,
summarizeDurations,
} from "../../scripts/probe-fleet-latency.mjs";
describe("probe-fleet-latency", () => {
it("parses cli defaults", () => {
expect(parseProbeArgs([])).toEqual({
baseUrl: "http://127.0.0.1:3000",
samples: 15,
warmup: 3,
timeoutMs: 15_000,
sloP95Ms: 900,
json: false,
allowDisconnected: false,
});
});
it("parses cli overrides", () => {
expect(
parseProbeArgs([
"--base-url",
"http://localhost:3100/",
"--samples",
"20",
"--warmup",
"4",
"--timeout-ms",
"9000",
"--slo-p95-ms",
"700",
"--allow-disconnected",
"--json",
])
).toEqual({
baseUrl: "http://localhost:3100",
samples: 20,
warmup: 4,
timeoutMs: 9000,
sloP95Ms: 700,
json: true,
allowDisconnected: true,
});
});
it("fails fast on malformed cli values", () => {
expect(() => parseProbeArgs(["--base-url", "--json"])).toThrow(
"Missing value for --base-url"
);
expect(() => parseProbeArgs(["--samples", "1.5"])).toThrow("Invalid --samples: 1.5");
});
it("computes percentile and stats summaries", () => {
const durations = [100, 200, 300, 400, 500];
expect(percentile(durations, 50)).toBe(300);
expect(percentile(durations, 90)).toBe(500);
expect(percentile(durations, 95)).toBe(500);
expect(summarizeDurations(durations, 5)).toEqual({
attempts: 5,
count: 5,
minMs: 100,
maxMs: 500,
meanMs: 300,
p50Ms: 300,
p90Ms: 500,
p95Ms: 500,
});
});
it("assesses runtime preflight with connection requirement", () => {
expect(
assessRuntimePreflight({
response: {
ok: true,
body: {
summary: { status: "connected" },
},
},
allowDisconnected: false,
})
).toEqual({
pass: true,
connected: true,
status: "connected",
message: null,
});
const disconnected = assessRuntimePreflight({
response: {
ok: true,
body: {
summary: { status: "stopped" },
},
},
allowDisconnected: false,
});
expect(disconnected.pass).toBe(false);
expect(disconnected.connected).toBe(false);
expect(disconnected.status).toBe("stopped");
const allowedDisconnected = assessRuntimePreflight({
response: {
ok: true,
body: {
summary: { status: "stopped" },
},
},
allowDisconnected: true,
});
expect(allowedDisconnected.pass).toBe(true);
expect(allowedDisconnected.connected).toBe(false);
expect(allowedDisconnected.status).toBe("stopped");
});
it("fails preflight on malformed summary payload", () => {
const invalidPayload = assessRuntimePreflight({
response: { ok: true, body: null },
allowDisconnected: true,
});
expect(invalidPayload.pass).toBe(false);
expect(invalidPayload.message).toContain("invalid /api/runtime/summary payload");
const missingSummary = assessRuntimePreflight({
response: { ok: true, body: {} },
allowDisconnected: true,
});
expect(missingSummary.pass).toBe(false);
expect(missingSummary.message).toContain("missing summary");
const missingStatus = assessRuntimePreflight({
response: { ok: true, body: { summary: {} } },
allowDisconnected: true,
});
expect(missingStatus.pass).toBe(false);
expect(missingStatus.message).toContain("summary.status missing");
});
it("classifies degraded fleet payloads as failures", () => {
expect(assessFleetPayload({ enabled: true, degraded: false })).toEqual({
ok: true,
message: null,
});
expect(
assessFleetPayload({
enabled: true,
degraded: true,
code: "GATEWAY_UNAVAILABLE",
reason: "gateway_unavailable",
})
).toEqual({
ok: false,
message: "degraded fleet response: GATEWAY_UNAVAILABLE gateway_unavailable",
});
expect(assessFleetPayload(null)).toEqual({
ok: false,
message: "invalid fleet response payload",
});
});
it("assesses pass/fail and diagnosis for fleet latency", () => {
const endpoint = {
name: "fleet",
sloBlocking: true,
stats: { p95Ms: 750 },
errors: { count: 0 },
};
expect(
classifyBottleneckHint({
endpoints: [endpoint],
sloP95Ms: 900,
})
).toBe("fleet latency is within SLO");
expect(
classifyBottleneckHint({
endpoints: [{ ...endpoint, stats: { p95Ms: 1200 } }],
sloP95Ms: 900,
})
).toBe("fleet slow -> bootstrap hydration path likely bottleneck");
expect(
assessProbe({
sloP95Ms: 900,
endpoints: [endpoint],
}).pass
).toBe(true);
expect(
assessProbe({
sloP95Ms: 900,
endpoints: [{ ...endpoint, stats: { p95Ms: 1200 } }],
}).pass
).toBe(false);
expect(
assessProbe({
sloP95Ms: 900,
endpoints: [{ ...endpoint, errors: { count: 1 } }],
}).pass
).toBe(false);
});
});
+15 -6
View File
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
RUNTIME_SYNC_FOCUSED_HISTORY_INTERVAL_MS,
RUNTIME_SYNC_MAX_HISTORY_LIMIT,
RUNTIME_SYNC_RECONCILE_INTERVAL_MS,
resolveRuntimeSyncBootstrapHistoryAgentIds,
resolveRuntimeSyncFocusedHistoryPollingIntent,
@@ -118,7 +119,7 @@ describe("runtimeSyncControlWorkflow", () => {
resolveRuntimeSyncLoadMoreHistoryLimit({
currentLimit: 200,
defaultLimit: 200,
maxLimit: 5000,
maxLimit: RUNTIME_SYNC_MAX_HISTORY_LIMIT,
})
).toBe(400);
@@ -126,17 +127,25 @@ describe("runtimeSyncControlWorkflow", () => {
resolveRuntimeSyncLoadMoreHistoryLimit({
currentLimit: 3000,
defaultLimit: 200,
maxLimit: 5000,
maxLimit: RUNTIME_SYNC_MAX_HISTORY_LIMIT,
})
).toBe(5000);
).toBe(1000);
expect(
resolveRuntimeSyncLoadMoreHistoryLimit({
currentLimit: 20,
defaultLimit: 50,
maxLimit: RUNTIME_SYNC_MAX_HISTORY_LIMIT,
})
).toBe(100);
expect(
resolveRuntimeSyncLoadMoreHistoryLimit({
currentLimit: null,
defaultLimit: 200,
maxLimit: 5000,
defaultLimit: 50,
maxLimit: RUNTIME_SYNC_MAX_HISTORY_LIMIT,
})
).toBe(400);
).toBe(100);
});
it("always plans summary refresh plus reconcile for gap recovery", () => {
+85 -257
View File
@@ -2,6 +2,7 @@ import { createElement, useEffect } from "react";
import { act, render } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { RUNTIME_SYNC_MAX_HISTORY_LIMIT } from "@/features/agents/operations/runtimeSyncControlWorkflow";
import { useRuntimeSyncController } from "@/features/agents/operations/useRuntimeSyncController";
import type { AgentState } from "@/features/agents/state/store";
@@ -278,7 +279,7 @@ describe("useRuntimeSyncController", () => {
expect(bootstrappedAgentIds).not.toContain("agent-3");
});
it("in domain mode bootstraps missing history only for the focused agent", async () => {
it("in domain mode bootstraps focused missing history through chat-history only", async () => {
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
const url = String(input);
if (url.includes("/api/runtime/summary")) {
@@ -291,19 +292,13 @@ describe("useRuntimeSyncController", () => {
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
if (url.includes("/api/runtime/agents/")) {
if (url.includes("/api/runtime/chat-history")) {
return new Response(
JSON.stringify({
enabled: true,
entries: [],
hasMore: false,
nextBeforeOutboxId: null,
semanticTurnsIncluded: 0,
windowTruncated: false,
activeRun: {
runId: null,
status: "idle",
complete: true,
ok: true,
payload: {
sessionKey: "agent:agent-2:main",
messages: [],
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
@@ -316,6 +311,14 @@ describe("useRuntimeSyncController", () => {
});
vi.stubGlobal("fetch", fetchMock);
mockedRunHistorySyncOperation.mockImplementation(async (params) => {
await params.client.call("chat.history", {
sessionKey: "agent:agent-2:main",
limit: 50,
});
return [];
});
renderController({
status: "connected",
useDomainApiReads: true,
@@ -332,11 +335,19 @@ describe("useRuntimeSyncController", () => {
await Promise.resolve();
});
const historyCalls = fetchMock.mock.calls
expect(mockedRunHistorySyncOperation).toHaveBeenCalledWith(
expect.objectContaining({
agentId: "agent-2",
})
);
const chatHistoryCalls = fetchMock.mock.calls
.map((call) => String(call[0]))
.filter((url) => url.includes("/api/runtime/chat-history"));
expect(chatHistoryCalls.some((url) => url.includes("sessionKey=agent%3Aagent-2%3Amain"))).toBe(true);
const semanticCalls = fetchMock.mock.calls
.map((call) => String(call[0]))
.filter((url) => url.includes("/api/runtime/agents/"));
expect(historyCalls.some((url) => url.includes("/api/runtime/agents/agent-2/history"))).toBe(true);
expect(historyCalls.some((url) => url.includes("/api/runtime/agents/agent-1/history"))).toBe(false);
expect(semanticCalls.length).toBe(0);
vi.unstubAllGlobals();
});
@@ -438,7 +449,30 @@ describe("useRuntimeSyncController", () => {
expect(inFlightSeen).toEqual([false, true, false]);
});
it("uses domain runtime APIs and expands semantic transcript limit for load-more", async () => {
it("uses shared gateway chat-history max in non-domain mode by default", async () => {
const ctx = renderController({
status: "disconnected",
useDomainApiReads: false,
maxHistoryLimit: undefined,
agents: [createAgent({ historyFetchLimit: 5_000 })],
focusedAgentId: null,
focusedAgentRunning: false,
});
await act(async () => {
await ctx.getValue().loadAgentHistory("agent-1", { limit: 5_000 });
});
expect(mockedRunHistorySyncOperation).toHaveBeenCalledWith(
expect.objectContaining({
agentId: "agent-1",
requestedLimit: 5_000,
maxLimit: RUNTIME_SYNC_MAX_HISTORY_LIMIT,
})
);
});
it("uses domain runtime APIs and uses chat-history load-more limit floor", async () => {
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
const url = String(input);
if (url.includes("/api/runtime/summary")) {
@@ -451,19 +485,13 @@ describe("useRuntimeSyncController", () => {
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
if (url.includes("/api/runtime/agents/agent-1/history")) {
if (url.includes("/api/runtime/chat-history")) {
return new Response(
JSON.stringify({
enabled: true,
entries: [],
hasMore: true,
nextBeforeOutboxId: null,
semanticTurnsIncluded: 2,
windowTruncated: true,
activeRun: {
runId: null,
status: "idle",
complete: true,
ok: true,
payload: {
sessionKey: "agent:agent-1:main",
messages: [],
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
@@ -484,80 +512,54 @@ describe("useRuntimeSyncController", () => {
focusedAgentRunning: false,
});
mockedRunHistorySyncOperation.mockImplementation(async (params) => {
await params.client.call("chat.history", {
sessionKey: "agent:agent-1:main",
limit: 100,
});
return [];
});
await act(async () => {
await ctx.getValue().loadAgentHistory("agent-1", { limit: 2 });
});
expect(ctx.dispatch).toHaveBeenCalledWith({
type: "updateAgent",
agentId: "agent-1",
patch: expect.objectContaining({
historyFetchLimit: 2,
historyFetchedCount: 2,
historyMaybeTruncated: true,
}),
});
await act(async () => {
ctx.getValue().loadMoreAgentHistory("agent-1");
await Promise.resolve();
await Promise.resolve();
});
expect(mockedRunHistorySyncOperation).toHaveBeenCalledWith(
expect.objectContaining({
agentId: "agent-1",
requestedLimit: 100,
})
);
expect(fetchMock).toHaveBeenCalledWith(
expect.stringContaining(
"/api/runtime/agents/agent-1/history?limit=400&view=semantic&turnLimit=50&scanLimit=800"
"/api/runtime/chat-history?sessionKey=agent%3Aagent-1%3Amain&limit=100"
),
expect.anything()
);
expect(
fetchMock.mock.calls.map((call) => String(call[0])).some((url) => url.includes("/api/runtime/agents/"))
).toBe(false);
expect(ctx.call).not.toHaveBeenCalledWith("status", {});
vi.unstubAllGlobals();
ctx.unmount();
});
it("hydrates semantic domain history through history sync so user turns can be restored", async () => {
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
const url = String(input);
if (url.includes("/api/runtime/agents/agent-1/history")) {
return new Response(
JSON.stringify({
enabled: true,
entries: [
{
id: 6,
event: {
type: "gateway.event",
event: "chat",
seq: 6,
payload: {
runId: "run-1",
sessionKey: "agent:agent-1:main",
state: "final",
message: { role: "assistant", content: "assistant only outbox turn" },
},
asOf: "2026-03-01T00:00:06.000Z",
},
createdAt: "2026-03-01T00:00:06.000Z",
},
],
hasMore: false,
nextBeforeOutboxId: null,
semanticTurnsIncluded: 1,
windowTruncated: false,
activeRun: {
runId: null,
status: "idle",
complete: true,
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
return new Response(JSON.stringify({ enabled: true }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
});
vi.stubGlobal("fetch", fetchMock);
it("executes domain history sync commands through shared history operation", async () => {
vi.stubGlobal(
"fetch",
vi.fn(async () =>
new Response(JSON.stringify({ enabled: true }), {
status: 200,
headers: { "Content-Type": "application/json" },
})
)
);
mockedRunHistorySyncOperation.mockResolvedValue([
{
@@ -621,41 +623,6 @@ describe("useRuntimeSyncController", () => {
it("clamps domain transcript hydration requests to gateway chat.history max limit", async () => {
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
const url = String(input);
if (url.includes("/api/runtime/agents/agent-1/history")) {
return new Response(
JSON.stringify({
enabled: true,
entries: [
{
id: 6,
event: {
type: "gateway.event",
event: "chat",
seq: 6,
payload: {
runId: "run-1",
sessionKey: "agent:agent-1:main",
state: "final",
message: { role: "assistant", content: "assistant only outbox turn" },
},
asOf: "2026-03-01T00:00:06.000Z",
},
createdAt: "2026-03-01T00:00:06.000Z",
},
],
hasMore: false,
nextBeforeOutboxId: null,
semanticTurnsIncluded: 1,
windowTruncated: false,
activeRun: {
runId: null,
status: "idle",
complete: true,
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
if (url.includes("/api/runtime/chat-history")) {
return new Response(
JSON.stringify({
@@ -698,153 +665,14 @@ describe("useRuntimeSyncController", () => {
expect(mockedRunHistorySyncOperation).toHaveBeenCalledWith(
expect.objectContaining({
agentId: "agent-1",
requestedLimit: 1_000,
requestedLimit: 5_000,
maxLimit: 1_000,
})
);
expect(fetchMock).toHaveBeenCalledWith(
expect.stringContaining(
"/api/runtime/agents/agent-1/history?limit=1000&view=semantic&turnLimit=50&scanLimit=800"
),
expect.anything()
);
expect(fetchMock).toHaveBeenCalledWith(
expect.stringContaining("/api/runtime/chat-history?sessionKey=agent%3Aagent-1%3Amain&limit=1000"),
expect.anything()
);
expect(ctx.dispatch).toHaveBeenCalledWith({
type: "updateAgent",
agentId: "agent-1",
patch: expect.objectContaining({
historyFetchLimit: 1_000,
}),
});
vi.unstubAllGlobals();
ctx.unmount();
});
it("does not request raw outbox backfill pages during semantic domain history loads", async () => {
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
const url = String(input);
if (url.includes("/api/runtime/agents/agent-1/history")) {
return new Response(
JSON.stringify({
enabled: true,
entries: [],
hasMore: true,
nextBeforeOutboxId: 10,
semanticTurnsIncluded: 0,
windowTruncated: true,
activeRun: {
runId: "run-1",
status: "running",
complete: false,
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
return new Response(JSON.stringify({ enabled: true }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
});
vi.stubGlobal("fetch", fetchMock);
const ctx = renderController({
status: "disconnected",
useDomainApiReads: true,
agents: [createAgent({ historyFetchLimit: 2 })],
focusedAgentId: null,
focusedAgentRunning: false,
});
await act(async () => {
await ctx.getValue().loadAgentHistory("agent-1", { limit: 2 });
});
const historyCalls = fetchMock.mock.calls
.map((call) => String(call[0]))
.filter((url) => url.includes("/api/runtime/agents/agent-1/history"));
expect(historyCalls.some((url) => url.includes("view=semantic"))).toBe(true);
expect(historyCalls.some((url) => url.includes("view=raw"))).toBe(false);
vi.unstubAllGlobals();
ctx.unmount();
});
it("does not replay outbox history entries through ingest callback", async () => {
let historyCallCount = 0;
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
const url = String(input);
if (url.includes("/api/runtime/agents/agent-1/history")) {
historyCallCount += 1;
if (historyCallCount === 1) {
return new Response(
JSON.stringify({
enabled: true,
entries: [
{
id: 5,
event: {
type: "gateway.event",
event: "runtime.delta",
seq: 5,
payload: { sessionKey: "agent:agent-1:main", delta: "old" },
asOf: "2026-03-01T00:00:05.000Z",
},
createdAt: "2026-03-01T00:00:05.000Z",
},
],
hasMore: false,
nextBeforeOutboxId: null,
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
return new Response(
JSON.stringify({
enabled: true,
entries: [
{
id: 5,
event: {
type: "gateway.event",
event: "runtime.delta",
seq: 5,
payload: { sessionKey: "agent:agent-1:main", delta: "new" },
asOf: "2026-03-02T00:00:05.000Z",
},
createdAt: "2026-03-02T00:00:05.000Z",
},
],
hasMore: false,
nextBeforeOutboxId: null,
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
return new Response(JSON.stringify({ enabled: true }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
});
vi.stubGlobal("fetch", fetchMock);
const ctx = renderController({
status: "disconnected",
useDomainApiReads: true,
agents: [createAgent({ historyFetchLimit: 2 })],
focusedAgentId: null,
focusedAgentRunning: false,
});
await act(async () => {
await ctx.getValue().loadAgentHistory("agent-1", { limit: 2 });
});
await act(async () => {
await ctx.getValue().loadAgentHistory("agent-1", { limit: 2 });
});
vi.unstubAllGlobals();
ctx.unmount();