diff --git a/.forgejo/workflows/dimensionlab-website.yml b/.forgejo/workflows/dimensionlab-website.yml index cf9c2fc..f272de5 100644 --- a/.forgejo/workflows/dimensionlab-website.yml +++ b/.forgejo/workflows/dimensionlab-website.yml @@ -67,6 +67,6 @@ jobs: DEPLOY_CONTAINER_CLI: docker DEPLOY_EVENT_NAME: ${{ github.event_name }} DEPLOY_REF: ${{ github.ref }} - DEPLOY_RESTART_STRATEGY: kill-container + DEPLOY_RESTART_STRATEGY: quadlet-container DEPLOY_SHA: ${{ github.sha }} run: scripts/deploy-dimensionlab-website.sh diff --git a/apps/web/src/lib/workspace-boundary.test.ts b/apps/web/src/lib/workspace-boundary.test.ts index fac4673..efb9e5b 100644 --- a/apps/web/src/lib/workspace-boundary.test.ts +++ b/apps/web/src/lib/workspace-boundary.test.ts @@ -1,4 +1,6 @@ -import { existsSync, readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; @@ -292,6 +294,59 @@ describe("workspace boundaries", () => { expect(deployScript).toContain("--dry-run"); expect(deployScript).toContain("DEPLOY_RESTART_STRATEGY"); }); + + test("rolls back the latest image when production smoke checks fail", () => { + const result = runDeployScriptWithFakes( + { + curl: failingCurlCommand, + git: fakeGitCommand, + podman: fakePodmanCommand("dimensionlab-website.service"), + }, + { + DEPLOY_CONTAINER_CLI: "podman", + DEPLOY_EVENT_NAME: "push", + DEPLOY_REF: "refs/heads/main", + DEPLOY_RESTART_STRATEGY: "quadlet-container", + DEPLOY_SHA: "1234567890abcdef", + DEPLOY_SMOKE_TIMEOUT_SECONDS: "0", + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("smoke checks failed"); + expect(result.stderr).toContain("rolling back to localhost/dimensionlab-website:rollback-"); + expect(result.log).toMatch( + /tag localhost\/dimensionlab-website:latest localhost\/dimensionlab-website:rollback-\d{14}/, + ); + expect(result.log).toMatch( + /tag localhost\/dimensionlab-website:rollback-\d{14} localhost\/dimensionlab-website:latest/, + ); + expect(result.log.match(/^stop dimensionlab-website$/gm)).toHaveLength(2); + }); + + test("refuses stop-based deploys unless the container belongs to the expected unit", () => { + const result = runDeployScriptWithFakes( + { + curl: passingCurlCommand, + git: fakeGitCommand, + podman: fakePodmanCommand("other.service"), + }, + { + DEPLOY_CONTAINER_CLI: "podman", + DEPLOY_EVENT_NAME: "push", + DEPLOY_REF: "refs/heads/main", + DEPLOY_RESTART_STRATEGY: "quadlet-container", + DEPLOY_SHA: "1234567890abcdef", + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "refusing to stop dimensionlab-website; expected PODMAN_SYSTEMD_UNIT=dimensionlab-website.service", + ); + expect(result.log).not.toContain("build "); + expect(result.log).not.toContain("stop dimensionlab-website"); + }); }); type WorkspacePackageExport = @@ -308,3 +363,86 @@ function expectPackageExport( ): void { expect(actual).toMatchObject(expected); } + +function runDeployScriptWithFakes( + commands: Record, + env: Record, +): { log: string; status: number | null; stderr: string; stdout: string } { + const tempDir = mkdtempSync(join(tmpdir(), "dimensionlab-deploy-test-")); + const logPath = join(tempDir, "commands.log"); + + for (const [name, source] of Object.entries(commands)) { + const commandPath = join(tempDir, name); + writeFileSync(commandPath, source); + chmodSync(commandPath, 0o755); + } + + const result = spawnSync("bash", [join(root, "scripts/deploy-dimensionlab-website.sh")], { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + ...env, + DEPLOY_TEST_LOG: logPath, + PATH: `${tempDir}:${process.env.PATH ?? ""}`, + }, + }); + + return { + log: existsSync(logPath) ? readFileSync(logPath, "utf8") : "", + status: result.status, + stderr: result.stderr, + stdout: result.stdout, + }; +} + +const fakeGitCommand = `#!/usr/bin/env bash +case "$1" in + branch) + echo main + ;; + rev-parse) + echo 1234567890abcdef + ;; + config|submodule) + exit 0 + ;; +esac +`; + +function fakePodmanCommand(systemdUnit: string): string { + return `#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$DEPLOY_TEST_LOG" +if [ "$1" = "image" ] && [ "$2" = "inspect" ]; then + if [ "$4" = "--format" ]; then + echo sha256:new + fi + exit 0 +fi +if [ "$1" = "inspect" ]; then + case "$*" in + *PODMAN_SYSTEMD_UNIT*) + echo ${systemdUnit} + ;; + *State.Running*) + echo true + ;; + *'.Image'*|*'{{.Image}}'*) + echo sha256:new + ;; + esac +fi +`; +} + +const failingCurlCommand = `#!/usr/bin/env bash +printf 'curl %s\\n' "$*" >> "$DEPLOY_TEST_LOG" +exit 22 +`; + +const passingCurlCommand = `#!/usr/bin/env bash +printf 'curl %s\\n' "$*" >> "$DEPLOY_TEST_LOG" +if [ "$*" = *'/api/dashboard/tiles'* ]; then + printf '{"state":"ready","tiles":[]}' +fi +`; diff --git a/scripts/deploy-dimensionlab-website.sh b/scripts/deploy-dimensionlab-website.sh index 238a72c..05178e0 100755 --- a/scripts/deploy-dimensionlab-website.sh +++ b/scripts/deploy-dimensionlab-website.sh @@ -18,6 +18,8 @@ release_tag="" latest_tag="${IMAGE_REPO}:latest" container_cli="" deployment_started=false +rollback_done=false +rollback_in_progress=false usage() { cat <&2 + if [ "${deployment_started:-false}" = "true" ] && [ "${rollback_in_progress:-false}" != "true" ]; then + rollback || true + fi exit 1 } @@ -135,6 +140,46 @@ tag_existing_latest_for_rollback() { fi } +container_systemd_unit() { + "$container_cli" inspect "$CONTAINER_NAME" \ + --format '{{ index .Config.Labels "PODMAN_SYSTEMD_UNIT" }}' 2>/dev/null || true +} + +require_container_managed_by_service() { + local unit + + if "$dry_run"; then + log "DRY-RUN: would require $CONTAINER_NAME to be managed by $SERVICE_NAME" + return 0 + fi + + unit="$(container_systemd_unit)" + [ "$unit" = "$SERVICE_NAME" ] || fail "refusing to stop $CONTAINER_NAME; expected PODMAN_SYSTEMD_UNIT=$SERVICE_NAME, got '${unit:-unset}'" +} + +validate_restart_strategy() { + case "$DEPLOY_RESTART_STRATEGY" in + systemctl) + if ! "$dry_run" && ! systemctl --user show "$SERVICE_NAME" >/dev/null 2>&1; then + fail "systemctl --user cannot access $SERVICE_NAME" + fi + ;; + quadlet-container|kill-container) + require_container_managed_by_service + ;; + auto) + if "$dry_run"; then + log "DRY-RUN: would validate automatic restart strategy" + elif ! command -v systemctl >/dev/null 2>&1 || ! systemctl --user show "$SERVICE_NAME" >/dev/null 2>&1; then + require_container_managed_by_service + fi + ;; + *) + fail "unknown DEPLOY_RESTART_STRATEGY: $DEPLOY_RESTART_STRATEGY" + ;; + esac +} + initialize_submodules() { log "initializing submodules" run git config --global url."https://git.dimensionlab.net/".insteadOf "ssh://git@git.dimensionlab.net/" @@ -162,7 +207,8 @@ restart_service() { systemctl) run systemctl --user restart "$SERVICE_NAME" ;; - kill-container) + quadlet-container|kill-container) + require_container_managed_by_service run "$container_cli" stop "$CONTAINER_NAME" ;; auto) @@ -232,7 +278,7 @@ smoke_tiles() { "$TILE_BATCH_URL" )" - [[ "$response" == *'"state":"ready"'* ]] || fail "tile batch smoke did not return ready state" + [[ "$response" == *'"state":"ready"'* ]] } wait_for_smoke() { @@ -256,13 +302,16 @@ wait_for_smoke() { } rollback() { - if [ "$deployment_started" != "true" ] || [ -z "$rollback_tag" ]; then + if [ "$deployment_started" != "true" ] || [ -z "$rollback_tag" ] || [ "$rollback_done" = "true" ]; then return 0 fi + rollback_done=true + rollback_in_progress=true printf '[deploy:%s] rolling back to %s\n' "$APP_NAME" "$rollback_tag" >&2 "$container_cli" tag "$rollback_tag" "$latest_tag" || true restart_service || true + rollback_in_progress=false } on_error() { @@ -275,6 +324,7 @@ trap on_error ERR require_main_push select_container_cli +validate_restart_strategy log "using container CLI: $container_cli" log "target image: $latest_tag"