diff --git a/apps/web/src/App.test.tsx b/apps/web/src/App.test.tsx index c319cc6..6124538 100644 --- a/apps/web/src/App.test.tsx +++ b/apps/web/src/App.test.tsx @@ -82,4 +82,42 @@ describe("React app dashboard state view", () => { id: "auto-refresh", }); }); + + test("prioritizes status, telemetry, modules, then services for hydration", () => { + const tiles = dashboardHydrationTiles(dimensionLabDashboardFixture); + const kinds = tiles.map((tile) => tile.kind); + const firstServiceIndex = kinds.indexOf("service"); + + expect(kinds.slice(0, 5)).toEqual([ + "status", + "status", + "status", + "status", + "status", + ]); + expect(kinds.lastIndexOf("telemetry")).toBeLessThan(kinds.indexOf("module")); + expect(kinds.lastIndexOf("module")).toBeLessThan(firstServiceIndex); + expect( + tiles.filter((tile) => tile.kind === "telemetry").map((tile) => tile.id), + ).toEqual( + dimensionLabDashboardFixture.telemetry + .filter((card) => card.datasource?.type === "external") + .map((card) => card.id), + ); + expect( + tiles.filter((tile) => tile.kind === "service").map((tile) => ({ + groupId: tile.groupId, + id: tile.id, + })), + ).toEqual( + dimensionLabDashboardFixture.serviceGroups.flatMap((group) => + group.services + .filter((service) => service.datasource?.type === "external") + .map((service) => ({ + groupId: group.id, + id: service.id, + })) + ), + ); + }); }); diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index c886391..c2f890e 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -9,7 +9,9 @@ import type { import type { DashboardRuntimeState } from "$lib/server/dashboard"; import { attachDashboardRefreshLifecycle, + createDashboardTileBackoff, createDashboardRequestAborter, + runDashboardHydrationQueue, shouldPauseDashboardRefresh, } from "$lib/client/dashboard-refresh"; import { dashboardDocumentToUiDashboard } from "$lib/ui-adapter/model-renderer"; @@ -42,6 +44,10 @@ type DashboardTileResponse = message: string; }; +type DashboardTileHydrationResult = "aborted" | "failed" | "ready"; + +const dashboardTileHydrationConcurrency = 6; + export function AppStateView({ dashboard, onThemeChange, @@ -157,6 +163,7 @@ export default function App() { let refreshTimer: number | undefined; let hydrationRun = 0; const requestAborter = createDashboardRequestAborter(); + const tileBackoff = createDashboardTileBackoff(); function refreshPaused() { return shouldPauseDashboardRefresh({ @@ -195,7 +202,9 @@ export default function App() { nextDashboard.state === "ready" && nextDashboard.liveDatasourceHydration?.enabled !== false ) { - const tiles = dashboardHydrationTiles(nextDashboard.document); + const tiles = dashboardHydrationTiles(nextDashboard.document).filter( + (tile) => tileBackoff.canAttempt(dashboardTileKey(tile)), + ); const tileSignal = requestAborter.beginTileRun(); setHydratingItemIds(new Set(tiles.map(dashboardTileKey))); hydrateDashboardTiles(tiles, currentRun, tileSignal); @@ -228,8 +237,20 @@ export default function App() { run: number, signal: AbortSignal, ) { - tiles.forEach((tile) => { - void hydrateDashboardTile(tile, run, signal); + void runDashboardHydrationQueue({ + concurrency: dashboardTileHydrationConcurrency, + hydrate: async (tile) => { + const key = dashboardTileKey(tile); + const result = await hydrateDashboardTile(tile, run, signal); + + if (result === "ready") { + tileBackoff.recordSuccess(key); + } else if (result === "failed") { + tileBackoff.recordFailure(key); + } + }, + items: tiles, + signal, }); } @@ -237,7 +258,7 @@ export default function App() { tile: DashboardTileReference, run: number, signal: AbortSignal, - ) { + ): Promise { const key = dashboardTileKey(tile); try { @@ -251,7 +272,9 @@ export default function App() { !response.ok || tileResponse.state !== "ready" ) { - return; + return signal.aborted || cancelled || run !== hydrationRun + ? "aborted" + : "failed"; } const readyTileResponse = tileResponse; @@ -263,10 +286,13 @@ export default function App() { } : current, ); + return "ready"; } catch (error) { if (!isAbortError(error) && !cancelled) { console.error("Dashboard tile hydration failed", error); + return "failed"; } + return "aborted"; } finally { if (!cancelled && !signal.aborted && run === hydrationRun) { setHydratingItemIds((current) => { @@ -415,7 +441,7 @@ export function dashboardHydrationTiles( })), ); - return [...telemetry, ...services, ...modules, ...status]; + return [...status, ...telemetry, ...modules, ...services]; } function dashboardTileKey(tile: DashboardTileReference): string { diff --git a/apps/web/src/lib/client/dashboard-refresh.test.ts b/apps/web/src/lib/client/dashboard-refresh.test.ts index 75305d5..1cdb872 100644 --- a/apps/web/src/lib/client/dashboard-refresh.test.ts +++ b/apps/web/src/lib/client/dashboard-refresh.test.ts @@ -1,7 +1,9 @@ import { describe, expect, test } from "vitest"; import { attachDashboardRefreshLifecycle, + createDashboardTileBackoff, createDashboardRequestAborter, + runDashboardHydrationQueue, shouldPauseDashboardRefresh, } from "./dashboard-refresh"; @@ -73,4 +75,69 @@ describe("dashboard refresh lifecycle", () => { expect(calls).toEqual(["pause", "pause", "pause", "load", "load"]); }); + + test("limits tile hydration concurrency", async () => { + let active = 0; + let maxActive = 0; + const started: number[] = []; + const releases = new Map void>(); + + const queue = runDashboardHydrationQueue({ + concurrency: 2, + items: [1, 2, 3, 4], + signal: new AbortController().signal, + hydrate: async (item) => { + active += 1; + maxActive = Math.max(maxActive, active); + started.push(item); + await new Promise((resolve) => releases.set(item, resolve)); + active -= 1; + }, + }); + + await waitFor(() => started.length === 2); + expect(started).toEqual([1, 2]); + expect(maxActive).toBe(2); + + releases.get(1)?.(); + await waitFor(() => started.length === 3); + expect(started).toEqual([1, 2, 3]); + expect(maxActive).toBe(2); + + releases.get(2)?.(); + releases.get(3)?.(); + await waitFor(() => started.length === 4); + releases.get(4)?.(); + await queue; + expect(maxActive).toBe(2); + }); + + test("backs off failed tile keys and resets after success", () => { + const backoff = createDashboardTileBackoff(); + + backoff.recordFailure("telemetry:infra-ram", 1_000); + expect(backoff.canAttempt("telemetry:infra-ram", 15_999)).toBe(false); + expect(backoff.canAttempt("telemetry:infra-ram", 16_000)).toBe(true); + + backoff.recordFailure("telemetry:infra-ram", 16_000); + expect(backoff.canAttempt("telemetry:infra-ram", 45_999)).toBe(false); + expect(backoff.canAttempt("telemetry:infra-ram", 46_000)).toBe(true); + + backoff.recordFailure("telemetry:infra-ram", 46_000); + backoff.recordFailure("telemetry:infra-ram", 106_000); + expect(backoff.canAttempt("telemetry:infra-ram", 225_999)).toBe(false); + expect(backoff.canAttempt("telemetry:infra-ram", 226_000)).toBe(true); + + backoff.recordSuccess("telemetry:infra-ram"); + expect(backoff.canAttempt("telemetry:infra-ram", 107_000)).toBe(true); + }); }); + +async function waitFor(predicate: () => boolean) { + for (let attempt = 0; attempt < 20; attempt += 1) { + if (predicate()) return; + await Promise.resolve(); + } + + throw new Error("condition was not met"); +} diff --git a/apps/web/src/lib/client/dashboard-refresh.ts b/apps/web/src/lib/client/dashboard-refresh.ts index 1124b27..998de5f 100644 --- a/apps/web/src/lib/client/dashboard-refresh.ts +++ b/apps/web/src/lib/client/dashboard-refresh.ts @@ -88,3 +88,60 @@ export function attachDashboardRefreshLifecycle( options.windowTarget.removeEventListener("pagehide", handlePageHide); }; } + +export interface DashboardHydrationQueueOptions { + concurrency: number; + hydrate: (item: TItem) => Promise | void; + items: TItem[]; + signal: AbortSignal; +} + +export async function runDashboardHydrationQueue( + options: DashboardHydrationQueueOptions, +): Promise { + const concurrency = Math.max(1, Math.floor(options.concurrency)); + let nextIndex = 0; + + async function worker() { + while (!options.signal.aborted) { + const item = options.items[nextIndex]; + nextIndex += 1; + if (item === undefined) return; + + await options.hydrate(item); + } + } + + const workerCount = Math.min(concurrency, options.items.length); + await Promise.all( + Array.from({ length: workerCount }, () => worker()), + ); +} + +const dashboardTileBackoffDelaysMs = [15_000, 30_000, 60_000, 120_000]; + +export function createDashboardTileBackoff() { + const failures = new Map(); + + return { + canAttempt(key: string, now = Date.now()): boolean { + const failure = failures.get(key); + return !failure || now >= failure.nextAttemptAt; + }, + recordFailure(key: string, now = Date.now()): void { + const previousAttempts = failures.get(key)?.attempts || 0; + const attempts = previousAttempts + 1; + const delay = + dashboardTileBackoffDelaysMs[ + Math.min(attempts - 1, dashboardTileBackoffDelaysMs.length - 1) + ]; + failures.set(key, { + attempts, + nextAttemptAt: now + delay, + }); + }, + recordSuccess(key: string): void { + failures.delete(key); + }, + }; +}