test: add MVP QA gate #21

Merged
vince merged 2 commits from codex/issue-10-mvp-qa-gate into main 2026-06-18 19:55:13 +02:00
11 changed files with 479 additions and 44 deletions
Showing only changes of commit bdb02daa96 - Show all commits

View file

@ -2,6 +2,7 @@ import type { StorybookConfig } from "@storybook/sveltekit";
const config: StorybookConfig = {
stories: ["../src/**/*.stories.@(js|ts|svelte)"],
staticDirs: ["../static"],
addons: [
"@storybook/addon-svelte-csf",
"@storybook/addon-a11y",

View file

@ -1,5 +1,17 @@
import "../src/app.css";
import type { Preview } from "@storybook/sveltekit";
import { setupWorker } from "msw/browser";
import { externalApiHandlers } from "../src/lib/testing/external-api-mocks";
if (typeof window !== "undefined") {
const worker = setupWorker(...externalApiHandlers);
void worker.start({
onUnhandledRequest: "bypass",
serviceWorker: {
url: "/mockServiceWorker.js",
},
});
}
const preview: Preview = {
parameters: {

View file

@ -22,7 +22,7 @@ This MVP uses Drizzle with Bun SQLite for local file-backed persistence.
- `bun run check`: run Svelte and TypeScript checks.
- `bun run test`: run Vitest.
- `bun run test:unit`: run Vitest explicitly as the unit test stage.
- `bun run test:e2e`: run Playwright browser smoke and QA checks.
- `bun run test:e2e`: build and run Playwright browser smoke and QA checks.
- `bun run test:qa`: run the MVP release gate.
- `bun run build`: build the production app.
- `bun run preview`: preview the production build.
@ -90,9 +90,13 @@ bun run test:qa
The gate runs Svelte/TypeScript checks, Vitest coverage for model,
persistence, renderer, datasource mocks, and presentation boundaries, the
production build, the static Storybook build, and Playwright desktop/mobile
smoke checks. Playwright also performs basic screenshot sanity checks,
keyboard-focus checks, reduced-motion checks, landmark checks, and axe
accessibility checks against the real model-driven route.
smoke checks against the built adapter output. Playwright also performs
baseline screenshot checks, keyboard navigation checks, reduced-motion checks,
landmark checks, and axe accessibility checks against the real model-driven
route.
Playwright uses an isolated SQLite database per run unless
`PLAYWRIGHT_DATABASE_URL` is set explicitly.
Presentation code is checked for Dimension Lab content leakage. Environment
specific labels, links, icon names, fallback values, and datasource references

View file

@ -44,5 +44,10 @@
"typescript": "^6.0.3",
"vite": "^8.0.16",
"vitest": "^4.1.9"
},
"msw": {
"workerDirectory": [
"static"
]
}
}

View file

@ -2,6 +2,9 @@ import { defineConfig, devices } from "@playwright/test";
const port = Number(process.env.PLAYWRIGHT_PORT || 4173);
const baseURL = `http://127.0.0.1:${port}`;
const databaseUrl =
process.env.PLAYWRIGHT_DATABASE_URL ||
`file:./data/playwright-${process.pid}-${Date.now()}.sqlite`;
export default defineConfig({
testDir: "tests/e2e",
@ -15,9 +18,9 @@ export default defineConfig({
screenshot: "only-on-failure",
},
webServer: {
command: `DATABASE_URL=file:./data/playwright.sqlite bun run dev -- --port ${port}`,
command: `bun run build && DATABASE_URL=${databaseUrl} HOST=127.0.0.1 PORT=${port} bun build/index.js`,
url: baseURL,
reuseExistingServer: !process.env.CI,
reuseExistingServer: false,
timeout: 120_000,
},
projects: [

View file

@ -37,7 +37,7 @@ function readPresentationSource(path: string): string {
if (stats.isFile()) {
if (path.endsWith(".test.ts")) return "";
if (path.includes(`${join("src", "lib", "ui", "stories")}${"/"}`)) return "";
if (!/\.(svelte|ts|css)$/.test(path)) return "";
if (!/\.(svelte|svelte\.js|ts|js|mjs|css|json)$/.test(path)) return "";
return readFileSync(path, "utf8");
}

View file

@ -1,7 +1,22 @@
import { describe, expect, test } from "vitest";
import { setupServer } from "msw/node";
import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest";
import { externalApiHandlers } from "./external-api-mocks";
const server = setupServer(...externalApiHandlers);
describe("external API mocks", () => {
beforeAll(() => {
server.listen({ onUnhandledRequest: "error" });
});
afterEach(() => {
server.resetHandlers();
});
afterAll(() => {
server.close();
});
test("defines deterministic handlers for deferred datasource adapters", () => {
expect(externalApiHandlers).toHaveLength(3);
expect(externalApiHandlers.map((handler) => handler.info.header)).toEqual([
@ -10,4 +25,31 @@ describe("external API mocks", () => {
"GET https://api.open-meteo.com/v1/forecast",
]);
});
test("intercepts deferred datasource requests without live services", async () => {
const prometheus = await fetch(
"https://prometheus.dimensionlab.net/api/v1/query",
).then((response) => response.json());
const status = await fetch(
"https://uptime.dimensionlab.net/api/status-page/dimensionlab",
).then((response) => response.json());
const weather = await fetch(
"https://api.open-meteo.com/v1/forecast?latitude=52.37&longitude=4.9",
).then((response) => response.json());
expect(prometheus).toMatchObject({
status: "success",
data: { result: [{ value: [1771430400, "1"] }] },
});
expect(status).toMatchObject({
status: "ok",
incidents: [],
});
expect(weather).toMatchObject({
current: {
temperature_2m: 21.4,
weather_code: 0,
},
});
});
});

349
static/mockServiceWorker.js Normal file
View file

@ -0,0 +1,349 @@
/* eslint-disable */
/* tslint:disable */
/**
* Mock Service Worker.
* @see https://github.com/mswjs/msw
* - Please do NOT modify this file.
*/
const PACKAGE_VERSION = '2.14.6'
const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82'
const IS_MOCKED_RESPONSE = Symbol('isMockedResponse')
const activeClientIds = new Set()
addEventListener('install', function () {
self.skipWaiting()
})
addEventListener('activate', function (event) {
event.waitUntil(self.clients.claim())
})
addEventListener('message', async function (event) {
const clientId = Reflect.get(event.source || {}, 'id')
if (!clientId || !self.clients) {
return
}
const client = await self.clients.get(clientId)
if (!client) {
return
}
const allClients = await self.clients.matchAll({
type: 'window',
})
switch (event.data) {
case 'KEEPALIVE_REQUEST': {
sendToClient(client, {
type: 'KEEPALIVE_RESPONSE',
})
break
}
case 'INTEGRITY_CHECK_REQUEST': {
sendToClient(client, {
type: 'INTEGRITY_CHECK_RESPONSE',
payload: {
packageVersion: PACKAGE_VERSION,
checksum: INTEGRITY_CHECKSUM,
},
})
break
}
case 'MOCK_ACTIVATE': {
activeClientIds.add(clientId)
sendToClient(client, {
type: 'MOCKING_ENABLED',
payload: {
client: {
id: client.id,
frameType: client.frameType,
},
},
})
break
}
case 'CLIENT_CLOSED': {
activeClientIds.delete(clientId)
const remainingClients = allClients.filter((client) => {
return client.id !== clientId
})
// Unregister itself when there are no more clients
if (remainingClients.length === 0) {
self.registration.unregister()
}
break
}
}
})
addEventListener('fetch', function (event) {
const requestInterceptedAt = Date.now()
// Bypass navigation requests.
if (event.request.mode === 'navigate') {
return
}
// Opening the DevTools triggers the "only-if-cached" request
// that cannot be handled by the worker. Bypass such requests.
if (
event.request.cache === 'only-if-cached' &&
event.request.mode !== 'same-origin'
) {
return
}
// Bypass all requests when there are no active clients.
// Prevents the self-unregistered worked from handling requests
// after it's been terminated (still remains active until the next reload).
if (activeClientIds.size === 0) {
return
}
const requestId = crypto.randomUUID()
event.respondWith(handleRequest(event, requestId, requestInterceptedAt))
})
/**
* @param {FetchEvent} event
* @param {string} requestId
* @param {number} requestInterceptedAt
*/
async function handleRequest(event, requestId, requestInterceptedAt) {
const client = await resolveMainClient(event)
const requestCloneForEvents = event.request.clone()
const response = await getResponse(
event,
client,
requestId,
requestInterceptedAt,
)
// Send back the response clone for the "response:*" life-cycle events.
// Ensure MSW is active and ready to handle the message, otherwise
// this message will pend indefinitely.
if (client && activeClientIds.has(client.id)) {
const serializedRequest = await serializeRequest(requestCloneForEvents)
// Clone the response so both the client and the library could consume it.
const responseClone = response.clone()
sendToClient(
client,
{
type: 'RESPONSE',
payload: {
isMockedResponse: IS_MOCKED_RESPONSE in response,
request: {
id: requestId,
...serializedRequest,
},
response: {
type: responseClone.type,
status: responseClone.status,
statusText: responseClone.statusText,
headers: Object.fromEntries(responseClone.headers.entries()),
body: responseClone.body,
},
},
},
responseClone.body ? [serializedRequest.body, responseClone.body] : [],
)
}
return response
}
/**
* Resolve the main client for the given event.
* Client that issues a request doesn't necessarily equal the client
* that registered the worker. It's with the latter the worker should
* communicate with during the response resolving phase.
* @param {FetchEvent} event
* @returns {Promise<Client | undefined>}
*/
async function resolveMainClient(event) {
const client = await self.clients.get(event.clientId)
if (activeClientIds.has(event.clientId)) {
return client
}
if (client?.frameType === 'top-level') {
return client
}
const allClients = await self.clients.matchAll({
type: 'window',
})
return allClients
.filter((client) => {
// Get only those clients that are currently visible.
return client.visibilityState === 'visible'
})
.find((client) => {
// Find the client ID that's recorded in the
// set of clients that have registered the worker.
return activeClientIds.has(client.id)
})
}
/**
* @param {FetchEvent} event
* @param {Client | undefined} client
* @param {string} requestId
* @param {number} requestInterceptedAt
* @returns {Promise<Response>}
*/
async function getResponse(event, client, requestId, requestInterceptedAt) {
// Clone the request because it might've been already used
// (i.e. its body has been read and sent to the client).
const requestClone = event.request.clone()
function passthrough() {
// Cast the request headers to a new Headers instance
// so the headers can be manipulated with.
const headers = new Headers(requestClone.headers)
// Remove the "accept" header value that marked this request as passthrough.
// This prevents request alteration and also keeps it compliant with the
// user-defined CORS policies.
const acceptHeader = headers.get('accept')
if (acceptHeader) {
const values = acceptHeader.split(',').map((value) => value.trim())
const filteredValues = values.filter(
(value) => value !== 'msw/passthrough',
)
if (filteredValues.length > 0) {
headers.set('accept', filteredValues.join(', '))
} else {
headers.delete('accept')
}
}
return fetch(requestClone, { headers })
}
// Bypass mocking when the client is not active.
if (!client) {
return passthrough()
}
// Bypass initial page load requests (i.e. static assets).
// The absence of the immediate/parent client in the map of the active clients
// means that MSW hasn't dispatched the "MOCK_ACTIVATE" event yet
// and is not ready to handle requests.
if (!activeClientIds.has(client.id)) {
return passthrough()
}
// Notify the client that a request has been intercepted.
const serializedRequest = await serializeRequest(event.request)
const clientMessage = await sendToClient(
client,
{
type: 'REQUEST',
payload: {
id: requestId,
interceptedAt: requestInterceptedAt,
...serializedRequest,
},
},
[serializedRequest.body],
)
switch (clientMessage.type) {
case 'MOCK_RESPONSE': {
return respondWithMock(clientMessage.data)
}
case 'PASSTHROUGH': {
return passthrough()
}
}
return passthrough()
}
/**
* @param {Client} client
* @param {any} message
* @param {Array<Transferable>} transferrables
* @returns {Promise<any>}
*/
function sendToClient(client, message, transferrables = []) {
return new Promise((resolve, reject) => {
const channel = new MessageChannel()
channel.port1.onmessage = (event) => {
if (event.data && event.data.error) {
return reject(event.data.error)
}
resolve(event.data)
}
client.postMessage(message, [
channel.port2,
...transferrables.filter(Boolean),
])
})
}
/**
* @param {Response} response
* @returns {Response}
*/
function respondWithMock(response) {
// Setting response status code to 0 is a no-op.
// However, when responding with a "Response.error()", the produced Response
// instance will have status code set to 0. Since it's not possible to create
// a Response instance with status code 0, handle that use-case separately.
if (response.status === 0) {
return Response.error()
}
const mockedResponse = new Response(response.body, response)
Reflect.defineProperty(mockedResponse, IS_MOCKED_RESPONSE, {
value: true,
enumerable: true,
})
return mockedResponse
}
/**
* @param {Request} request
*/
async function serializeRequest(request) {
return {
url: request.url,
mode: request.mode,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
cache: request.cache,
credentials: request.credentials,
destination: request.destination,
integrity: request.integrity,
redirect: request.redirect,
referrer: request.referrer,
referrerPolicy: request.referrerPolicy,
body: await request.arrayBuffer(),
keepalive: request.keepalive,
}
}

View file

@ -1,6 +1,26 @@
import AxeBuilder from "@axe-core/playwright";
import { expect, test } from "@playwright/test";
const linkedServiceIds = [
"vaultwarden",
"forgejo",
"wiki",
"aws-start",
"grafana",
"uptime-kuma",
"prometheus",
"backrest",
"n8n",
"open-webui",
"comfyui",
"models",
"adminer",
"assistant",
"suna",
"cockpit-infra",
"forgejo-ssh-relay",
];
test.describe("dashboard page QA gate", () => {
test("renders the model-driven dashboard on desktop", async ({
page,
@ -23,8 +43,9 @@ test.describe("dashboard page QA gate", () => {
const bodyBox = await page.locator("body").boundingBox();
expect(bodyBox?.width).toBeGreaterThan(1000);
const screenshot = await page.screenshot({ fullPage: true });
expect(screenshot.byteLength).toBeGreaterThan(20_000);
await expect(page).toHaveScreenshot("dashboard-desktop.png", {
fullPage: true,
});
});
test("keeps the first screen usable on mobile", async ({ page }, testInfo) => {
@ -37,8 +58,9 @@ test.describe("dashboard page QA gate", () => {
).toBeVisible();
await expect(page.getByLabel("Service groups")).toBeVisible();
const screenshot = await page.screenshot({ fullPage: true });
expect(screenshot.byteLength).toBeGreaterThan(12_000);
await expect(page).toHaveScreenshot("dashboard-mobile.png", {
fullPage: true,
});
});
test("exposes usable landmarks and a visible keyboard focus state", async ({
@ -47,15 +69,17 @@ test.describe("dashboard page QA gate", () => {
await page.goto("/");
await expect(page.getByRole("main")).toHaveCount(1);
await page.keyboard.press("Tab");
for (const serviceId of linkedServiceIds) {
await page.keyboard.press("Tab");
const focused = page.locator(":focus");
await expect(focused).toHaveAttribute("href", /vault\.dimensionlab\.net/);
const focused = page.locator(":focus");
await expect(focused).toHaveAttribute("data-model-id", serviceId);
const focusBoxShadow = await focused.evaluate((element) => {
return window.getComputedStyle(element).boxShadow;
});
expect(focusBoxShadow).not.toBe("none");
const focusBoxShadow = await focused.evaluate((element) => {
return window.getComputedStyle(element).boxShadow;
});
expect(focusBoxShadow).not.toBe("none");
}
});
test("passes automated accessibility checks", async ({ page }) => {
@ -65,34 +89,29 @@ test.describe("dashboard page QA gate", () => {
expect(results.violations).toEqual([]);
});
test("honors reduced-motion preferences", async ({ browser }) => {
const context = await browser.newContext({ reducedMotion: "reduce" });
const page = await context.newPage();
test("honors reduced-motion preferences", async ({ page }) => {
await page.emulateMedia({ reducedMotion: "reduce" });
try {
await page.goto("/");
const durations = await page.evaluate(() => {
const element = document.createElement("div");
element.style.animation = "qa-motion-check 10s infinite";
element.style.transition = "opacity 10s linear";
document.body.append(element);
await page.goto("/");
const durations = await page.evaluate(() => {
const element = document.createElement("div");
element.style.animation = "qa-motion-check 10s infinite";
element.style.transition = "opacity 10s linear";
document.body.append(element);
const styles = window.getComputedStyle(element);
return {
animation: styles.animationDuration,
transition: styles.transitionDuration,
};
});
const styles = window.getComputedStyle(element);
return {
animation: styles.animationDuration,
transition: styles.transitionDuration,
};
});
expect(cssDurationToMilliseconds(durations.animation)).toBeLessThanOrEqual(
0.01,
);
expect(cssDurationToMilliseconds(durations.transition)).toBeLessThanOrEqual(
0.01,
);
} finally {
await context.close();
}
expect(cssDurationToMilliseconds(durations.animation)).toBeLessThanOrEqual(
0.01,
);
expect(cssDurationToMilliseconds(durations.transition)).toBeLessThanOrEqual(
0.01,
);
});
});

Binary file not shown.

After

Width:  |  Height:  |  Size: 379 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 351 KiB