diff --git a/.forgejo/workflows/dimensionlab-website.yml b/.forgejo/workflows/dimensionlab-website.yml index f272de5..8cac0a1 100644 --- a/.forgejo/workflows/dimensionlab-website.yml +++ b/.forgejo/workflows/dimensionlab-website.yml @@ -62,6 +62,12 @@ jobs: git config --global url."https://git.dimensionlab.net/".insteadOf "ssh://git@git.dimensionlab.net/" git submodule update --init --recursive + - name: Verify Podman deployment socket + run: | + timeout 15s docker version + unit="$(timeout 15s docker inspect dimensionlab-website --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}')" + test "$unit" = "dimensionlab-website.service" + - name: Deploy production website env: DEPLOY_CONTAINER_CLI: docker diff --git a/apps/web/src/lib/workspace-boundary.test.ts b/apps/web/src/lib/workspace-boundary.test.ts index 77ee6a6..50dcac6 100644 --- a/apps/web/src/lib/workspace-boundary.test.ts +++ b/apps/web/src/lib/workspace-boundary.test.ts @@ -277,6 +277,8 @@ describe("workspace boundaries", () => { expect(workflow).toContain("needs: ci"); expect(workflow).toContain("github.event_name == 'push'"); expect(workflow).toContain("github.ref == 'refs/heads/main'"); + expect(workflow).toContain("docker inspect dimensionlab-website"); + expect(workflow).toContain("PODMAN_SYSTEMD_UNIT"); expect(workflow).toContain("scripts/deploy-dimensionlab-website.sh"); }); @@ -335,7 +337,8 @@ describe("workspace boundaries", () => { }, { DEPLOY_CONTAINER_CLI: "podman", - DEPLOY_CONTAINER_START_TIMEOUT_SECONDS: "0", + DEPLOY_CONTAINER_START_TIMEOUT_SECONDS: "1", + DEPLOY_TEST_CONTAINER_IMAGE: "wrong", DEPLOY_EVENT_NAME: "push", DEPLOY_REF: "refs/heads/main", DEPLOY_RESTART_STRATEGY: "quadlet-container", @@ -346,6 +349,7 @@ describe("workspace boundaries", () => { expect(result.status).toBe(1); expect(result.stderr).toContain("did not restart on localhost/dimensionlab-website:latest"); expect(result.stderr).toContain("rolling back to localhost/dimensionlab-website:rollback-"); + expect(result.stderr).toContain("rollback image is running"); expect(result.log).toMatch( /tag localhost\/dimensionlab-website:rollback-\d{14} localhost\/dimensionlab-website:latest/, ); @@ -492,13 +496,29 @@ esac function fakePodmanCommand(systemdUnit: string): string { return `#!/usr/bin/env bash +state_file="$DEPLOY_TEST_LOG.state" +[ -f "$state_file" ] || printf 'initial' > "$state_file" printf '%s\\n' "$*" >> "$DEPLOY_TEST_LOG" if [ "$1" = "image" ] && [ "$2" = "inspect" ]; then if [ "$4" = "--format" ]; then - echo sha256:new + case "$3" in + *:rollback-*) + echo sha256:old + ;; + *) + if [ "$(cat "$state_file")" = "rollback" ]; then + echo sha256:old + else + echo sha256:new + fi + ;; + esac fi exit 0 fi +if [ "$1" = "tag" ] && [ "$2" != "localhost/dimensionlab-website:latest" ]; then + printf 'rollback' > "$state_file" +fi if [ "$1" = "inspect" ]; then case "$*" in *PODMAN_SYSTEMD_UNIT*) @@ -508,7 +528,13 @@ if [ "$1" = "inspect" ]; then echo true ;; *'.Image'*|*'{{.Image}}'*) - echo sha256:new + if [ "$(cat "$state_file")" = "rollback" ]; then + echo sha256:old + elif [ "\${DEPLOY_TEST_CONTAINER_IMAGE:-new}" = "wrong" ]; then + echo sha256:wrong + else + echo sha256:new + fi ;; esac fi diff --git a/scripts/deploy-dimensionlab-website.sh b/scripts/deploy-dimensionlab-website.sh index 1101e90..4fac4cd 100755 --- a/scripts/deploy-dimensionlab-website.sh +++ b/scripts/deploy-dimensionlab-website.sh @@ -241,7 +241,6 @@ container_running() { wait_for_container_restart() { local expected_image - local deadline if "$dry_run"; then log "DRY-RUN: would wait for $CONTAINER_NAME to run $latest_tag" @@ -250,18 +249,27 @@ wait_for_container_restart() { expected_image="$(latest_image_id)" [ -n "$expected_image" ] || fail "could not resolve image id for $latest_tag" + + wait_for_container_image "$expected_image" "new image" || fail "$CONTAINER_NAME did not restart on $latest_tag within ${DEPLOY_CONTAINER_START_TIMEOUT_SECONDS}s" +} + +wait_for_container_image() { + local expected_image="$1" + local label="$2" + local deadline + deadline=$((SECONDS + DEPLOY_CONTAINER_START_TIMEOUT_SECONDS)) while [ "$SECONDS" -lt "$deadline" ]; do if container_running && [ "$(container_image_id)" = "$expected_image" ]; then - log "$CONTAINER_NAME is running the new image" + log "$CONTAINER_NAME is running the $label" return 0 fi sleep 2 done - fail "$CONTAINER_NAME did not restart on $latest_tag within ${DEPLOY_CONTAINER_START_TIMEOUT_SECONDS}s" + return 1 } smoke_get() { @@ -303,6 +311,8 @@ wait_for_smoke() { } rollback() { + local rollback_image + if [ "$deployment_started" != "true" ] || [ -z "$rollback_tag" ] || [ "$rollback_done" = "true" ]; then return 0 fi @@ -310,8 +320,18 @@ rollback() { rollback_done=true rollback_in_progress=true printf '[deploy:%s] rolling back to %s\n' "$APP_NAME" "$rollback_tag" >&2 + rollback_image="$("$container_cli" image inspect "$rollback_tag" --format '{{.Id}}' 2>/dev/null || true)" "$container_cli" tag "$rollback_tag" "$latest_tag" || true - restart_service || true + if container_running; then + restart_service || true + else + printf '[deploy:%s] waiting for %s to recover with rollback image\n' "$APP_NAME" "$SERVICE_NAME" >&2 + fi + if [ -n "$rollback_image" ] && wait_for_container_image "$rollback_image" "rollback image"; then + printf '[deploy:%s] rollback image is running\n' "$APP_NAME" >&2 + else + printf '[deploy:%s] ERROR: rollback image did not become healthy\n' "$APP_NAME" >&2 + fi rollback_in_progress=false }