fix(ci): verify deployment socket and rollback image

.forgejo/workflows/dimensionlab-website.yml

@@ -62,6 +62,12 @@ jobs:
           git config --global url."https://git.dimensionlab.net/".insteadOf "ssh://git@git.dimensionlab.net/"
           git submodule update --init --recursive
 
+      - name: Verify Podman deployment socket
+        run: |
+          timeout 15s docker version
+          unit="$(timeout 15s docker inspect dimensionlab-website --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}')"
+          test "$unit" = "dimensionlab-website.service"
+
       - name: Deploy production website
         env:
           DEPLOY_CONTAINER_CLI: docker

apps/web/src/lib/workspace-boundary.test.ts

@@ -277,6 +277,8 @@ describe("workspace boundaries", () => {
     expect(workflow).toContain("needs: ci");
     expect(workflow).toContain("github.event_name == 'push'");
     expect(workflow).toContain("github.ref == 'refs/heads/main'");
+    expect(workflow).toContain("docker inspect dimensionlab-website");
+    expect(workflow).toContain("PODMAN_SYSTEMD_UNIT");
     expect(workflow).toContain("scripts/deploy-dimensionlab-website.sh");
   });
 
@@ -335,7 +337,8 @@ describe("workspace boundaries", () => {
       },
       {
         DEPLOY_CONTAINER_CLI: "podman",
-        DEPLOY_CONTAINER_START_TIMEOUT_SECONDS: "0",
+        DEPLOY_CONTAINER_START_TIMEOUT_SECONDS: "1",
+        DEPLOY_TEST_CONTAINER_IMAGE: "wrong",
         DEPLOY_EVENT_NAME: "push",
         DEPLOY_REF: "refs/heads/main",
         DEPLOY_RESTART_STRATEGY: "quadlet-container",
@@ -346,6 +349,7 @@ describe("workspace boundaries", () => {
     expect(result.status).toBe(1);
     expect(result.stderr).toContain("did not restart on localhost/dimensionlab-website:latest");
     expect(result.stderr).toContain("rolling back to localhost/dimensionlab-website:rollback-");
+    expect(result.stderr).toContain("rollback image is running");
     expect(result.log).toMatch(
       /tag localhost\/dimensionlab-website:rollback-\d{14} localhost\/dimensionlab-website:latest/,
     );
@@ -492,13 +496,29 @@ esac
 
 function fakePodmanCommand(systemdUnit: string): string {
   return `#!/usr/bin/env bash
+state_file="$DEPLOY_TEST_LOG.state"
+[ -f "$state_file" ] || printf 'initial' > "$state_file"
 printf '%s\\n' "$*" >> "$DEPLOY_TEST_LOG"
 if [ "$1" = "image" ] && [ "$2" = "inspect" ]; then
   if [ "$4" = "--format" ]; then
+    case "$3" in
+      *:rollback-*)
+        echo sha256:old
+        ;;
+      *)
+        if [ "$(cat "$state_file")" = "rollback" ]; then
+          echo sha256:old
+        else
           echo sha256:new
         fi
+        ;;
+    esac
+  fi
   exit 0
 fi
+if [ "$1" = "tag" ] && [ "$2" != "localhost/dimensionlab-website:latest" ]; then
+  printf 'rollback' > "$state_file"
+fi
 if [ "$1" = "inspect" ]; then
   case "$*" in
     *PODMAN_SYSTEMD_UNIT*)
@@ -508,7 +528,13 @@ if [ "$1" = "inspect" ]; then
       echo true
       ;;
     *'.Image'*|*'{{.Image}}'*)
+      if [ "$(cat "$state_file")" = "rollback" ]; then
+        echo sha256:old
+      elif [ "\${DEPLOY_TEST_CONTAINER_IMAGE:-new}" = "wrong" ]; then
+        echo sha256:wrong
+      else
         echo sha256:new
+      fi
       ;;
   esac
 fi

scripts/deploy-dimensionlab-website.sh

@@ -241,7 +241,6 @@ container_running() {
 
 wait_for_container_restart() {
   local expected_image
-  local deadline
 
   if "$dry_run"; then
     log "DRY-RUN: would wait for $CONTAINER_NAME to run $latest_tag"
@@ -250,18 +249,27 @@ wait_for_container_restart() {
 
   expected_image="$(latest_image_id)"
   [ -n "$expected_image" ] || fail "could not resolve image id for $latest_tag"
+
+  wait_for_container_image "$expected_image" "new image" || fail "$CONTAINER_NAME did not restart on $latest_tag within ${DEPLOY_CONTAINER_START_TIMEOUT_SECONDS}s"
+}
+
+wait_for_container_image() {
+  local expected_image="$1"
+  local label="$2"
+  local deadline
+
   deadline=$((SECONDS + DEPLOY_CONTAINER_START_TIMEOUT_SECONDS))
 
   while [ "$SECONDS" -lt "$deadline" ]; do
     if container_running && [ "$(container_image_id)" = "$expected_image" ]; then
-      log "$CONTAINER_NAME is running the new image"
+      log "$CONTAINER_NAME is running the $label"
       return 0
     fi
 
     sleep 2
   done
 
-  fail "$CONTAINER_NAME did not restart on $latest_tag within ${DEPLOY_CONTAINER_START_TIMEOUT_SECONDS}s"
+  return 1
 }
 
 smoke_get() {
@@ -303,6 +311,8 @@ wait_for_smoke() {
 }
 
 rollback() {
+  local rollback_image
+
   if [ "$deployment_started" != "true" ] || [ -z "$rollback_tag" ] || [ "$rollback_done" = "true" ]; then
     return 0
   fi
@@ -310,8 +320,18 @@ rollback() {
   rollback_done=true
   rollback_in_progress=true
   printf '[deploy:%s] rolling back to %s\n' "$APP_NAME" "$rollback_tag" >&2
+  rollback_image="$("$container_cli" image inspect "$rollback_tag" --format '{{.Id}}' 2>/dev/null || true)"
   "$container_cli" tag "$rollback_tag" "$latest_tag" || true
+  if container_running; then
     restart_service || true
+  else
+    printf '[deploy:%s] waiting for %s to recover with rollback image\n' "$APP_NAME" "$SERVICE_NAME" >&2
+  fi
+  if [ -n "$rollback_image" ] && wait_for_container_image "$rollback_image" "rollback image"; then
+    printf '[deploy:%s] rollback image is running\n' "$APP_NAME" >&2
+  else
+    printf '[deploy:%s] ERROR: rollback image did not become healthy\n' "$APP_NAME" >&2
+  fi
   rollback_in_progress=false
 }