perf(web): queue dashboard tile hydration

apps/web/src/App.tsx

@@ -9,7 +9,9 @@ import type {
 import type { DashboardRuntimeState } from "$lib/server/dashboard";
 import {
   attachDashboardRefreshLifecycle,
+  createDashboardTileBackoff,
   createDashboardRequestAborter,
+  runDashboardHydrationQueue,
   shouldPauseDashboardRefresh,
 } from "$lib/client/dashboard-refresh";
 import { dashboardDocumentToUiDashboard } from "$lib/ui-adapter/model-renderer";
@@ -42,6 +44,10 @@ type DashboardTileResponse =
       message: string;
     };
 
+type DashboardTileHydrationResult = "aborted" | "failed" | "ready";
+
+const dashboardTileHydrationConcurrency = 6;
+
 export function AppStateView({
   dashboard,
   onThemeChange,
@@ -157,6 +163,7 @@ export default function App() {
     let refreshTimer: number | undefined;
     let hydrationRun = 0;
     const requestAborter = createDashboardRequestAborter();
+    const tileBackoff = createDashboardTileBackoff();
 
     function refreshPaused() {
       return shouldPauseDashboardRefresh({
@@ -195,7 +202,9 @@ export default function App() {
           nextDashboard.state === "ready" &&
           nextDashboard.liveDatasourceHydration?.enabled !== false
         ) {
-          const tiles = dashboardHydrationTiles(nextDashboard.document);
+          const tiles = dashboardHydrationTiles(nextDashboard.document).filter(
+            (tile) => tileBackoff.canAttempt(dashboardTileKey(tile)),
+          );
           const tileSignal = requestAborter.beginTileRun();
           setHydratingItemIds(new Set(tiles.map(dashboardTileKey)));
           hydrateDashboardTiles(tiles, currentRun, tileSignal);
@@ -228,8 +237,20 @@ export default function App() {
       run: number,
       signal: AbortSignal,
     ) {
-      tiles.forEach((tile) => {
-        void hydrateDashboardTile(tile, run, signal);
+      void runDashboardHydrationQueue({
+        concurrency: dashboardTileHydrationConcurrency,
+        hydrate: async (tile) => {
+          const key = dashboardTileKey(tile);
+          const result = await hydrateDashboardTile(tile, run, signal);
+
+          if (result === "ready") {
+            tileBackoff.recordSuccess(key);
+          } else if (result === "failed") {
+            tileBackoff.recordFailure(key);
+          }
+        },
+        items: tiles,
+        signal,
       });
     }
 
@@ -237,7 +258,7 @@ export default function App() {
       tile: DashboardTileReference,
       run: number,
       signal: AbortSignal,
-    ) {
+    ): Promise<DashboardTileHydrationResult> {
       const key = dashboardTileKey(tile);
 
       try {
@@ -251,7 +272,9 @@ export default function App() {
           !response.ok ||
           tileResponse.state !== "ready"
         ) {
-          return;
+          return signal.aborted || cancelled || run !== hydrationRun
+            ? "aborted"
+            : "failed";
         }
 
         const readyTileResponse = tileResponse;
@@ -263,10 +286,13 @@ export default function App() {
               }
             : current,
         );
+        return "ready";
       } catch (error) {
         if (!isAbortError(error) && !cancelled) {
           console.error("Dashboard tile hydration failed", error);
+          return "failed";
         }
+        return "aborted";
       } finally {
         if (!cancelled && !signal.aborted && run === hydrationRun) {
           setHydratingItemIds((current) => {
@@ -415,7 +441,7 @@ export function dashboardHydrationTiles(
       })),
   );
 
-  return [...telemetry, ...services, ...modules, ...status];
+  return [...status, ...telemetry, ...modules, ...services];
 }
 
 function dashboardTileKey(tile: DashboardTileReference): string {