fix(ci): harden website deploy rollback

.forgejo/workflows/dimensionlab-website.yml

@@ -67,6 +67,6 @@ jobs:
           DEPLOY_CONTAINER_CLI: docker
           DEPLOY_EVENT_NAME: ${{ github.event_name }}
           DEPLOY_REF: ${{ github.ref }}
-          DEPLOY_RESTART_STRATEGY: kill-container
+          DEPLOY_RESTART_STRATEGY: quadlet-container
           DEPLOY_SHA: ${{ github.sha }}
         run: scripts/deploy-dimensionlab-website.sh

apps/web/src/lib/workspace-boundary.test.ts

@@ -1,4 +1,6 @@
-import { existsSync, readFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { describe, expect, test } from "vitest";
 
@@ -292,6 +294,59 @@ describe("workspace boundaries", () => {
     expect(deployScript).toContain("--dry-run");
     expect(deployScript).toContain("DEPLOY_RESTART_STRATEGY");
   });
+
+  test("rolls back the latest image when production smoke checks fail", () => {
+    const result = runDeployScriptWithFakes(
+      {
+        curl: failingCurlCommand,
+        git: fakeGitCommand,
+        podman: fakePodmanCommand("dimensionlab-website.service"),
+      },
+      {
+        DEPLOY_CONTAINER_CLI: "podman",
+        DEPLOY_EVENT_NAME: "push",
+        DEPLOY_REF: "refs/heads/main",
+        DEPLOY_RESTART_STRATEGY: "quadlet-container",
+        DEPLOY_SHA: "1234567890abcdef",
+        DEPLOY_SMOKE_TIMEOUT_SECONDS: "0",
+      },
+    );
+
+    expect(result.status).toBe(1);
+    expect(result.stderr).toContain("smoke checks failed");
+    expect(result.stderr).toContain("rolling back to localhost/dimensionlab-website:rollback-");
+    expect(result.log).toMatch(
+      /tag localhost\/dimensionlab-website:latest localhost\/dimensionlab-website:rollback-\d{14}/,
+    );
+    expect(result.log).toMatch(
+      /tag localhost\/dimensionlab-website:rollback-\d{14} localhost\/dimensionlab-website:latest/,
+    );
+    expect(result.log.match(/^stop dimensionlab-website$/gm)).toHaveLength(2);
+  });
+
+  test("refuses stop-based deploys unless the container belongs to the expected unit", () => {
+    const result = runDeployScriptWithFakes(
+      {
+        curl: passingCurlCommand,
+        git: fakeGitCommand,
+        podman: fakePodmanCommand("other.service"),
+      },
+      {
+        DEPLOY_CONTAINER_CLI: "podman",
+        DEPLOY_EVENT_NAME: "push",
+        DEPLOY_REF: "refs/heads/main",
+        DEPLOY_RESTART_STRATEGY: "quadlet-container",
+        DEPLOY_SHA: "1234567890abcdef",
+      },
+    );
+
+    expect(result.status).toBe(1);
+    expect(result.stderr).toContain(
+      "refusing to stop dimensionlab-website; expected PODMAN_SYSTEMD_UNIT=dimensionlab-website.service",
+    );
+    expect(result.log).not.toContain("build ");
+    expect(result.log).not.toContain("stop dimensionlab-website");
+  });
 });
 
 type WorkspacePackageExport =
@@ -308,3 +363,86 @@ function expectPackageExport(
 ): void {
   expect(actual).toMatchObject(expected);
 }
+
+function runDeployScriptWithFakes(
+  commands: Record<string, string>,
+  env: Record<string, string>,
+): { log: string; status: number | null; stderr: string; stdout: string } {
+  const tempDir = mkdtempSync(join(tmpdir(), "dimensionlab-deploy-test-"));
+  const logPath = join(tempDir, "commands.log");
+
+  for (const [name, source] of Object.entries(commands)) {
+    const commandPath = join(tempDir, name);
+    writeFileSync(commandPath, source);
+    chmodSync(commandPath, 0o755);
+  }
+
+  const result = spawnSync("bash", [join(root, "scripts/deploy-dimensionlab-website.sh")], {
+    cwd: root,
+    encoding: "utf8",
+    env: {
+      ...process.env,
+      ...env,
+      DEPLOY_TEST_LOG: logPath,
+      PATH: `${tempDir}:${process.env.PATH ?? ""}`,
+    },
+  });
+
+  return {
+    log: existsSync(logPath) ? readFileSync(logPath, "utf8") : "",
+    status: result.status,
+    stderr: result.stderr,
+    stdout: result.stdout,
+  };
+}
+
+const fakeGitCommand = `#!/usr/bin/env bash
+case "$1" in
+  branch)
+    echo main
+    ;;
+  rev-parse)
+    echo 1234567890abcdef
+    ;;
+  config|submodule)
+    exit 0
+    ;;
+esac
+`;
+
+function fakePodmanCommand(systemdUnit: string): string {
+  return `#!/usr/bin/env bash
+printf '%s\\n' "$*" >> "$DEPLOY_TEST_LOG"
+if [ "$1" = "image" ] && [ "$2" = "inspect" ]; then
+  if [ "$4" = "--format" ]; then
+    echo sha256:new
+  fi
+  exit 0
+fi
+if [ "$1" = "inspect" ]; then
+  case "$*" in
+    *PODMAN_SYSTEMD_UNIT*)
+      echo ${systemdUnit}
+      ;;
+    *State.Running*)
+      echo true
+      ;;
+    *'.Image'*|*'{{.Image}}'*)
+      echo sha256:new
+      ;;
+  esac
+fi
+`;
+}
+
+const failingCurlCommand = `#!/usr/bin/env bash
+printf 'curl %s\\n' "$*" >> "$DEPLOY_TEST_LOG"
+exit 22
+`;
+
+const passingCurlCommand = `#!/usr/bin/env bash
+printf 'curl %s\\n' "$*" >> "$DEPLOY_TEST_LOG"
+if [ "$*" = *'/api/dashboard/tiles'* ]; then
+  printf '{"state":"ready","tiles":[]}'
+fi
+`;

scripts/deploy-dimensionlab-website.sh

@@ -18,6 +18,8 @@ release_tag=""
 latest_tag="${IMAGE_REPO}:latest"
 container_cli=""
 deployment_started=false
+rollback_done=false
+rollback_in_progress=false
 
 usage() {
   cat <<USAGE
@@ -36,6 +38,9 @@ log() {
 
 fail() {
   printf '[deploy:%s] ERROR: %s\n' "$APP_NAME" "$*" >&2
+  if [ "${deployment_started:-false}" = "true" ] && [ "${rollback_in_progress:-false}" != "true" ]; then
+    rollback || true
+  fi
   exit 1
 }
 
@@ -135,6 +140,46 @@ tag_existing_latest_for_rollback() {
   fi
 }
 
+container_systemd_unit() {
+  "$container_cli" inspect "$CONTAINER_NAME" \
+    --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true
+}
+
+require_container_managed_by_service() {
+  local unit
+
+  if "$dry_run"; then
+    log "DRY-RUN: would require $CONTAINER_NAME to be managed by $SERVICE_NAME"
+    return 0
+  fi
+
+  unit="$(container_systemd_unit)"
+  [ "$unit" = "$SERVICE_NAME" ] || fail "refusing to stop $CONTAINER_NAME; expected PODMAN_SYSTEMD_UNIT=$SERVICE_NAME, got '${unit:-unset}'"
+}
+
+validate_restart_strategy() {
+  case "$DEPLOY_RESTART_STRATEGY" in
+    systemctl)
+      if ! "$dry_run" && ! systemctl --user show "$SERVICE_NAME" >/dev/null 2>&1; then
+        fail "systemctl --user cannot access $SERVICE_NAME"
+      fi
+      ;;
+    quadlet-container|kill-container)
+      require_container_managed_by_service
+      ;;
+    auto)
+      if "$dry_run"; then
+        log "DRY-RUN: would validate automatic restart strategy"
+      elif ! command -v systemctl >/dev/null 2>&1 || ! systemctl --user show "$SERVICE_NAME" >/dev/null 2>&1; then
+        require_container_managed_by_service
+      fi
+      ;;
+    *)
+      fail "unknown DEPLOY_RESTART_STRATEGY: $DEPLOY_RESTART_STRATEGY"
+      ;;
+  esac
+}
+
 initialize_submodules() {
   log "initializing submodules"
   run git config --global url."https://git.dimensionlab.net/".insteadOf "ssh://git@git.dimensionlab.net/"
@@ -162,7 +207,8 @@ restart_service() {
     systemctl)
       run systemctl --user restart "$SERVICE_NAME"
       ;;
-    kill-container)
+    quadlet-container|kill-container)
+      require_container_managed_by_service
       run "$container_cli" stop "$CONTAINER_NAME"
       ;;
     auto)
@@ -232,7 +278,7 @@ smoke_tiles() {
       "$TILE_BATCH_URL"
   )"
 
-  [[ "$response" == *'"state":"ready"'* ]] || fail "tile batch smoke did not return ready state"
+  [[ "$response" == *'"state":"ready"'* ]]
 }
 
 wait_for_smoke() {
@@ -256,13 +302,16 @@ wait_for_smoke() {
 }
 
 rollback() {
-  if [ "$deployment_started" != "true" ] || [ -z "$rollback_tag" ]; then
+  if [ "$deployment_started" != "true" ] || [ -z "$rollback_tag" ] || [ "$rollback_done" = "true" ]; then
     return 0
   fi
 
+  rollback_done=true
+  rollback_in_progress=true
   printf '[deploy:%s] rolling back to %s\n' "$APP_NAME" "$rollback_tag" >&2
   "$container_cli" tag "$rollback_tag" "$latest_tag" || true
   restart_service || true
+  rollback_in_progress=false
 }
 
 on_error() {
@@ -275,6 +324,7 @@ trap on_error ERR
 
 require_main_push
 select_container_cli
+validate_restart_strategy
 
 log "using container CLI: $container_cli"
 log "target image: $latest_tag"