apps/web/src/App.test.tsx

@@ -82,4 +82,42 @@ describe("React app dashboard state view", () => {
       id: "auto-refresh",
     });
   });
+
+  test("prioritizes status, telemetry, modules, then services for hydration", () => {
+    const tiles = dashboardHydrationTiles(dimensionLabDashboardFixture);
+    const kinds = tiles.map((tile) => tile.kind);
+    const firstServiceIndex = kinds.indexOf("service");
+
+    expect(kinds.slice(0, 5)).toEqual([
+      "status",
+      "status",
+      "status",
+      "status",
+      "status",
+    ]);
+    expect(kinds.lastIndexOf("telemetry")).toBeLessThan(kinds.indexOf("module"));
+    expect(kinds.lastIndexOf("module")).toBeLessThan(firstServiceIndex);
+    expect(
+      tiles.filter((tile) => tile.kind === "telemetry").map((tile) => tile.id),
+    ).toEqual(
+      dimensionLabDashboardFixture.telemetry
+        .filter((card) => card.datasource?.type === "external")
+        .map((card) => card.id),
+    );
+    expect(
+      tiles.filter((tile) => tile.kind === "service").map((tile) => ({
+        groupId: tile.groupId,
+        id: tile.id,
+      })),
+    ).toEqual(
+      dimensionLabDashboardFixture.serviceGroups.flatMap((group) =>
+        group.services
+          .filter((service) => service.datasource?.type === "external")
+          .map((service) => ({
+            groupId: group.id,
+            id: service.id,
+          }))
+      ),
+    );
+  });
 });

apps/web/src/App.tsx

@@ -9,7 +9,9 @@ import type {
 import type { DashboardRuntimeState } from "$lib/server/dashboard";
 import {
   attachDashboardRefreshLifecycle,
+  createDashboardTileBackoff,
   createDashboardRequestAborter,
+  runDashboardHydrationQueue,
   shouldPauseDashboardRefresh,
 } from "$lib/client/dashboard-refresh";
 import { dashboardDocumentToUiDashboard } from "$lib/ui-adapter/model-renderer";
@@ -42,6 +44,10 @@ type DashboardTileResponse =
       message: string;
     };
 
+type DashboardTileHydrationResult = "aborted" | "failed" | "ready";
+
+const dashboardTileHydrationConcurrency = 6;
+
 export function AppStateView({
   dashboard,
   onThemeChange,
@@ -157,6 +163,7 @@ export default function App() {
     let refreshTimer: number | undefined;
     let hydrationRun = 0;
     const requestAborter = createDashboardRequestAborter();
+    const tileBackoff = createDashboardTileBackoff();
 
     function refreshPaused() {
       return shouldPauseDashboardRefresh({
@@ -195,7 +202,9 @@ export default function App() {
           nextDashboard.state === "ready" &&
           nextDashboard.liveDatasourceHydration?.enabled !== false
         ) {
-          const tiles = dashboardHydrationTiles(nextDashboard.document);
+          const tiles = dashboardHydrationTiles(nextDashboard.document).filter(
+            (tile) => tileBackoff.canAttempt(dashboardTileKey(tile)),
+          );
           const tileSignal = requestAborter.beginTileRun();
           setHydratingItemIds(new Set(tiles.map(dashboardTileKey)));
           hydrateDashboardTiles(tiles, currentRun, tileSignal);
@@ -228,8 +237,20 @@ export default function App() {
       run: number,
       signal: AbortSignal,
     ) {
-      tiles.forEach((tile) => {
-        void hydrateDashboardTile(tile, run, signal);
+      void runDashboardHydrationQueue({
+        concurrency: dashboardTileHydrationConcurrency,
+        hydrate: async (tile) => {
+          const key = dashboardTileKey(tile);
+          const result = await hydrateDashboardTile(tile, run, signal);
+
+          if (result === "ready") {
+            tileBackoff.recordSuccess(key);
+          } else if (result === "failed") {
+            tileBackoff.recordFailure(key);
+          }
+        },
+        items: tiles,
+        signal,
       });
     }
 
@@ -237,7 +258,7 @@ export default function App() {
       tile: DashboardTileReference,
       run: number,
       signal: AbortSignal,
-    ) {
+    ): Promise<DashboardTileHydrationResult> {
       const key = dashboardTileKey(tile);
 
       try {
@@ -251,7 +272,9 @@ export default function App() {
           !response.ok ||
           tileResponse.state !== "ready"
         ) {
-          return;
+          return signal.aborted || cancelled || run !== hydrationRun
+            ? "aborted"
+            : "failed";
         }
 
         const readyTileResponse = tileResponse;
@@ -263,10 +286,13 @@ export default function App() {
               }
             : current,
         );
+        return "ready";
       } catch (error) {
         if (!isAbortError(error) && !cancelled) {
           console.error("Dashboard tile hydration failed", error);
+          return "failed";
         }
+        return "aborted";
       } finally {
         if (!cancelled && !signal.aborted && run === hydrationRun) {
           setHydratingItemIds((current) => {
@@ -415,7 +441,7 @@ export function dashboardHydrationTiles(
       })),
   );
 
-  return [...telemetry, ...services, ...modules, ...status];
+  return [...status, ...telemetry, ...modules, ...services];
 }
 
 function dashboardTileKey(tile: DashboardTileReference): string {

apps/web/src/lib/client/dashboard-refresh.test.ts

@@ -1,7 +1,9 @@
 import { describe, expect, test } from "vitest";
 import {
   attachDashboardRefreshLifecycle,
+  createDashboardTileBackoff,
   createDashboardRequestAborter,
+  runDashboardHydrationQueue,
   shouldPauseDashboardRefresh,
 } from "./dashboard-refresh";
 
@@ -73,4 +75,69 @@ describe("dashboard refresh lifecycle", () => {
 
     expect(calls).toEqual(["pause", "pause", "pause", "load", "load"]);
   });
+
+  test("limits tile hydration concurrency", async () => {
+    let active = 0;
+    let maxActive = 0;
+    const started: number[] = [];
+    const releases = new Map<number, () => void>();
+
+    const queue = runDashboardHydrationQueue({
+      concurrency: 2,
+      items: [1, 2, 3, 4],
+      signal: new AbortController().signal,
+      hydrate: async (item) => {
+        active += 1;
+        maxActive = Math.max(maxActive, active);
+        started.push(item);
+        await new Promise<void>((resolve) => releases.set(item, resolve));
+        active -= 1;
+      },
+    });
+
+    await waitFor(() => started.length === 2);
+    expect(started).toEqual([1, 2]);
+    expect(maxActive).toBe(2);
+
+    releases.get(1)?.();
+    await waitFor(() => started.length === 3);
+    expect(started).toEqual([1, 2, 3]);
+    expect(maxActive).toBe(2);
+
+    releases.get(2)?.();
+    releases.get(3)?.();
+    await waitFor(() => started.length === 4);
+    releases.get(4)?.();
+    await queue;
+    expect(maxActive).toBe(2);
+  });
+
+  test("backs off failed tile keys and resets after success", () => {
+    const backoff = createDashboardTileBackoff();
+
+    backoff.recordFailure("telemetry:infra-ram", 1_000);
+    expect(backoff.canAttempt("telemetry:infra-ram", 15_999)).toBe(false);
+    expect(backoff.canAttempt("telemetry:infra-ram", 16_000)).toBe(true);
+
+    backoff.recordFailure("telemetry:infra-ram", 16_000);
+    expect(backoff.canAttempt("telemetry:infra-ram", 45_999)).toBe(false);
+    expect(backoff.canAttempt("telemetry:infra-ram", 46_000)).toBe(true);
+
+    backoff.recordFailure("telemetry:infra-ram", 46_000);
+    backoff.recordFailure("telemetry:infra-ram", 106_000);
+    expect(backoff.canAttempt("telemetry:infra-ram", 225_999)).toBe(false);
+    expect(backoff.canAttempt("telemetry:infra-ram", 226_000)).toBe(true);
+
+    backoff.recordSuccess("telemetry:infra-ram");
+    expect(backoff.canAttempt("telemetry:infra-ram", 107_000)).toBe(true);
+  });
 });
+
+async function waitFor(predicate: () => boolean) {
+  for (let attempt = 0; attempt < 20; attempt += 1) {
+    if (predicate()) return;
+    await Promise.resolve();
+  }
+
+  throw new Error("condition was not met");
+}

apps/web/src/lib/client/dashboard-refresh.ts

@@ -88,3 +88,60 @@ export function attachDashboardRefreshLifecycle(
     options.windowTarget.removeEventListener("pagehide", handlePageHide);
   };
 }
+
+export interface DashboardHydrationQueueOptions<TItem> {
+  concurrency: number;
+  hydrate: (item: TItem) => Promise<void> | void;
+  items: TItem[];
+  signal: AbortSignal;
+}
+
+export async function runDashboardHydrationQueue<TItem>(
+  options: DashboardHydrationQueueOptions<TItem>,
+): Promise<void> {
+  const concurrency = Math.max(1, Math.floor(options.concurrency));
+  let nextIndex = 0;
+
+  async function worker() {
+    while (!options.signal.aborted) {
+      const item = options.items[nextIndex];
+      nextIndex += 1;
+      if (item === undefined) return;
+
+      await options.hydrate(item);
+    }
+  }
+
+  const workerCount = Math.min(concurrency, options.items.length);
+  await Promise.all(
+    Array.from({ length: workerCount }, () => worker()),
+  );
+}
+
+const dashboardTileBackoffDelaysMs = [15_000, 30_000, 60_000, 120_000];
+
+export function createDashboardTileBackoff() {
+  const failures = new Map<string, { attempts: number; nextAttemptAt: number }>();
+
+  return {
+    canAttempt(key: string, now = Date.now()): boolean {
+      const failure = failures.get(key);
+      return !failure || now >= failure.nextAttemptAt;
+    },
+    recordFailure(key: string, now = Date.now()): void {
+      const previousAttempts = failures.get(key)?.attempts || 0;
+      const attempts = previousAttempts + 1;
+      const delay =
+        dashboardTileBackoffDelaysMs[
+          Math.min(attempts - 1, dashboardTileBackoffDelaysMs.length - 1)
+        ];
+      failures.set(key, {
+        attempts,
+        nextAttemptAt: now + delay,
+      });
+    },
+    recordSuccess(key: string): void {
+      failures.delete(key);
+    },
+  };
+}