scripts/deploy.sh
Ref: Size: 19.6 KiB History
#!/usr/bin/env bash
#
# Roll the freshly-built HEAD binaries to the live fleet: the local control
# plane (eitri-server) and every remote host (eitri-agent).
#
# This is the manual, restart-based deploy. It bounces the server (a brief
# API/SSE + QUIC-sync blip; agents redial within a few seconds) and each agent.
# Running VMs SURVIVE an agent restart: cloud-hypervisor runs in its own process
# group and the agent re-adopts a still-running VM on startup (lost-detection
# keys on /proc/sys/kernel/random/boot_id, stable across an agent restart). This
# is NOT zero-downtime — the control plane has a gap during the bounce. Genuine
# rolling updates are a separate, later effort.
#
# Config is sourced from $EITRI_DEPLOY_ENV (default ~/eitri-deploy/deploy.env);
# copy scripts/deploy.env.example there and fill it in. No site-specific values
# or secrets live in the repo.
set -euo pipefail
ENV_FILE="${EITRI_DEPLOY_ENV:-$HOME/eitri-deploy/deploy.env}"
if [[ ! -f "$ENV_FILE" ]]; then
echo "deploy: config not found: $ENV_FILE" >&2
echo " cp scripts/deploy.env.example \"$ENV_FILE\" and edit it." >&2
exit 1
fi
# shellcheck disable=SC1090
source "$ENV_FILE"
: "${SERVER_BIN:?set in $ENV_FILE}" "${SERVER_CONFIG:?}" "${SERVER_LOG:?}" "${SERVER_URL:?}"
: "${AGENT_HOSTS:?}" "${AGENT_BIN:?}" "${AGENT_STATE_DIR:?}"
: "${CH_BIN:?}" "${FIRMWARE:?}"
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$REPO_ROOT"
# Coverage output dirs. The fleet is always built with coverage instrumentation,
# so the boot-gate below can flush + merge server+agent coverage. Defaults live
# under the repo's gitignored coverage/ tree (server) and the agent's state dir
# (agent, a path ON the remote host); override either in deploy.env.
SERVER_GOCOVERDIR="${SERVER_GOCOVERDIR:-$REPO_ROOT/coverage/server}"
AGENT_GOCOVERDIR="${AGENT_GOCOVERDIR:-$AGENT_STATE_DIR/coverage}"
# SSH-CA gate boot-check config. The smoke proves guest access through the jump
# gate (a hard gate). SMOKE_GATE defaults to the server's ssh_listen (the
# host:port the gate accepts, whose host is the gate host-cert principal);
# override in deploy.env if that address isn't dialable from here. SMOKE_VM_USER
# falls back to the smoke's own default (ubuntu) when unset. The smoke's user CA
# is load-or-created at SMOKE_USER_CA_FILE and registered with the tenant (the
# operator PAT's, derived via Me()) before the throwaway VM is created.
SMOKE_GATE="${SMOKE_GATE:-$(python3 -c "import json; print(json.load(open('$SERVER_CONFIG')).get('ssh_listen',''))" 2>/dev/null || true)}"
SMOKE_USER_CA_FILE="${SMOKE_USER_CA_FILE:-$HOME/eitri-deploy/smoke_user_ca}"
# Sign-in replaces the admin token: the boot-gate authenticates as the deploy
# machine identity in the bundled eitri-oidc issuer, driving the real code flow
# to mint a short-lived PAT (spec §3) that proves the credential chain. That
# identity does NOT own the fleet — the fleet's hosts belong to the operator's
# tenant, so the machine identity lands in its own empty JIT tenant and the
# VM-lifecycle half of the gate runs on an operator PAT (CI_PAT_FILE) instead. eitri-oidc runs as a sibling of eitri-server on THIS
# (control-plane) box. Everything the issuer owns on this box lives under ONE
# directory — ~/eitri-deploy/oidc/ — so the deploy dir doesn't accumulate loose
# per-service files; only the binary sits in the shared bin/.
OIDC_BIN="$HOME/eitri-deploy/bin/eitri-oidc"
OIDC_DIR="$HOME/eitri-deploy/oidc"
OIDC_CONFIG="$OIDC_DIR/eitri-oidc.json"
OIDC_LOG="$OIDC_DIR/eitri-oidc.log"
CI_USER="${CI_USER:-deploy@eitri.local}"
CI_PASSWORD_FILE="${CI_PASSWORD_FILE:-$OIDC_DIR/deploy-password}"
# The machine identity only proves the credential chain (its JIT tenant owns no
# hosts — those belong to the operator's tenant). The VM-lifecycle half of the
# boot-gate authenticates with an OPERATOR-minted, tenant-scoped console PAT
# the deployer saves here once (see the boot-gate).
CI_PAT_FILE="${CI_PAT_FILE:-$OIDC_DIR/deploy-pat}"
bold() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; }
short() { sha256sum "$1" | cut -c1-12; }
SHA="$(git rev-parse --short HEAD)"
git diff --quiet || SHA="$SHA-dirty"
bold "Deploying HEAD $SHA to fleet"
# Version stamp baked into every binary (internal/version.Version), matching
# the Makefile's build target so a deploy.sh build and a `make build` binary
# report the same string for the same tree. An array (not a plain string) so
# the "-X ...=..." value survives intact as ONE argument to go build.
VERSION="$(git describe --tags --always --dirty 2>/dev/null || echo dev)"
GO_LDFLAGS=(-ldflags "-X github.com/a73x/eitri/internal/version.Version=$VERSION")
# ── 0. Build ──────────────────────────────────────────────────────────────────
# The fleet binaries are ALWAYS built with coverage instrumentation. -coverpkg
# spans the whole module so internal/* (not just main) is measured through the
# real binaries. -covermode=atomic is REQUIRED: runtime/coverage.WriteCountersDir
# (the mid-run SIGUSR1 flush in internal/covsnap) refuses to run under the default
# 'set' mode, and atomic is correct anyway for a live server flushed across
# concurrent goroutines. Coverage is a by-product of the boot-gate below and a
# no-op unless GOCOVERDIR is set at launch. The gate driver (eitri-smoke) builds
# normally.
bold "Building binaries (coverage-instrumented server+agent)"
make web
go build "${GO_LDFLAGS[@]}" -cover -covermode=atomic -coverpkg=github.com/a73x/eitri/... -o bin/eitri-server ./cmd/eitri-server
go build "${GO_LDFLAGS[@]}" -cover -covermode=atomic -coverpkg=github.com/a73x/eitri/... -o bin/eitri-agent ./cmd/eitri-agent
go build "${GO_LDFLAGS[@]}" -o bin/eitri-smoke ./cmd/eitri-smoke
# The bundled OIDC issuer (sibling of the server; built plain — it is not part of
# the coverage-instrumented fleet). Installed and configured in §1.5 below.
go build "${GO_LDFLAGS[@]}" -o bin/eitri-oidc ./cmd/eitri-oidc
# ── 0a. Pre-flight: server.json must carry the oidc block ──────────────────────
# The server is a pure OIDC relying party now: it will not boot without
# oidc.issuer. Deploy scripts never rewrite server.json (site-specific config is
# the operator's; see the header) — so if the block is missing, fail EARLY with
# the exact JSON to paste rather than bouncing the server into a crash loop.
if ! jq -e '.oidc.issuer' "$SERVER_CONFIG" >/dev/null 2>&1; then
cat >&2 <<EOF
deploy: $SERVER_CONFIG is missing the required "oidc" block.
The server authenticates the console via OIDC; admin_token is gone. Add this
top-level block to server.json (issuer points at the bundled eitri-oidc this
script installs on 127.0.0.1:9111):
"oidc": {
"issuer": "http://127.0.0.1:9111",
"client_id": "eitri-console",
"public_url": "$SERVER_URL"
}
public_url must match the redirect this script registers with eitri-oidc
($SERVER_URL/auth/callback), so keep public_url == $SERVER_URL for the bundled
single-box issuer. If you front the console at a routable address instead, set
public_url to it AND edit clients[0].redirect_url in $OIDC_CONFIG to match.
Then re-run make deploy.
EOF
exit 1
fi
# ── 1. Control plane (local eitri-server) ─────────────────────────────────────
bold "Rolling server -> $SERVER_BIN"
# Stop first: a running executable cannot be overwritten (ETXTBSY); install
# does unlink+create so the new binary lands cleanly, then relaunch detached.
pkill -TERM -f "eitri-server --config $SERVER_CONFIG" 2>/dev/null || true
for _ in $(seq 1 20); do pgrep -x eitri-server >/dev/null || break; sleep 0.5; done
if pgrep -x eitri-server >/dev/null; then
echo "deploy: server did not stop; aborting before swap" >&2
exit 1
fi
install -m 0755 bin/eitri-server "$SERVER_BIN"
# Fresh coverage dir per deploy: covdata rejects a dir mixing covermodes, and
# coverage should reflect only this deploy's run.
rm -rf "$SERVER_GOCOVERDIR"; mkdir -p "$SERVER_GOCOVERDIR"
GOCOVERDIR="$SERVER_GOCOVERDIR" setsid "$SERVER_BIN" --config "$SERVER_CONFIG" </dev/null >>"$SERVER_LOG" 2>&1 &
# Liveness gate: /livez is 200 as soon as the HTTP mux serves — an honest
# "process is up" signal (unlike GET /, which returns the SPA even before the
# app is wired). Readiness (DB) is gated separately below, after the agents
# have had a chance to redial.
for _ in $(seq 1 30); do
curl -fsS -o /dev/null "$SERVER_URL/livez" 2>/dev/null && break
sleep 0.5
done
if ! curl -fsS -o /dev/null "$SERVER_URL/livez" 2>/dev/null; then
echo "deploy: server did not come up at $SERVER_URL (see $SERVER_LOG)" >&2
exit 1
fi
echo "server up: $(short "$SERVER_BIN") listening at $SERVER_URL"
# ── 1.5 Bundled OIDC issuer (eitri-oidc) + ci user ────────────────────────────
# Same user-space convention as the server: this fleet runs everything from
# ~/eitri-deploy as the deploy user — no sudo, no system units (the systemd
# unit ships in the release tarball for hosts installed per the quickstart).
# Stop first (ETXTBSY), swap, relaunch detached; the issuer is stateless
# beyond its key+users files, so an unconditional bounce is harmless (sessions
# live in eitri-server, not here).
bold "Rolling eitri-oidc issuer -> $OIDC_BIN"
pkill -TERM -f "eitri-oidc -config $OIDC_CONFIG" 2>/dev/null || true
for _ in $(seq 1 20); do pgrep -x eitri-oidc >/dev/null || break; sleep 0.5; done
if pgrep -x eitri-oidc >/dev/null; then
echo "deploy: eitri-oidc did not stop; aborting before swap" >&2
exit 1
fi
install -m 0755 bin/eitri-oidc "$OIDC_BIN"
mkdir -p "$OIDC_DIR" && chmod 700 "$OIDC_DIR"
if [[ ! -f "$OIDC_CONFIG" ]]; then
# redirect_url must equal <the server's oidc.public_url>/auth/callback — the
# server builds its RedirectURL from public_url, and the issuer exact-matches
# it. Derive it from server.json (public_url is not a secret) so the two can
# never disagree; SERVER_URL is only the fallback.
callback_base="$(jq -r '.oidc.public_url // empty' "$SERVER_CONFIG")"
callback_base="${callback_base:-$SERVER_URL}"
( umask 077; printf '%s\n' "{
\"listen\": \"127.0.0.1:9111\",
\"issuer\": \"http://127.0.0.1:9111\",
\"users_file\": \"$OIDC_DIR/users.json\",
\"signing_key\": \"$OIDC_DIR/signing.key\",
\"clients\": [
{\"id\": \"eitri-console\", \"redirect_url\": \"$callback_base/auth/callback\"}
]
}" > "$OIDC_CONFIG" )
echo "wrote $OIDC_CONFIG (single-box loopback issuer)"
else
echo "$OIDC_CONFIG present — left as-is"
fi
setsid "$OIDC_BIN" -config "$OIDC_CONFIG" </dev/null >>"$OIDC_LOG" 2>&1 &
# Liveness gate mirrors the server's: the discovery doc serves as soon as the
# mux is up.
for _ in $(seq 1 30); do
curl -fsS -o /dev/null "http://127.0.0.1:9111/.well-known/openid-configuration" 2>/dev/null && break
sleep 0.5
done
if ! curl -fsS -o /dev/null "http://127.0.0.1:9111/.well-known/openid-configuration" 2>/dev/null; then
echo "deploy: eitri-oidc did not come up (see $OIDC_LOG)" >&2
exit 1
fi
echo "eitri-oidc up (issuer http://127.0.0.1:9111)"
# The deploy identity drives the boot-gate's headless sign-in. Generate its
# password once (kept beside the other deploy secrets, 0600), then (re-)register
# it — the flat-file add is idempotent-by-replace and keeps the user's stable
# subject.
if [[ ! -f "$CI_PASSWORD_FILE" ]]; then
( umask 077; openssl rand -hex 16 >"$CI_PASSWORD_FILE" )
chmod 600 "$CI_PASSWORD_FILE"
echo "generated deploy-identity password: $CI_PASSWORD_FILE"
fi
# Flags MUST precede the positional email: Go's flag package stops parsing at
# the first non-flag argument, so `user add <email> --flags` would swallow the
# flags as positionals and fail.
"$OIDC_BIN" user add --config "$OIDC_CONFIG" --password-file "$CI_PASSWORD_FILE" "$CI_USER"
echo "deploy identity registered: $CI_USER"
# The identity's tenant is JIT-provisioned on its first sign-in (nothing to
# assert here); the gate's VM-lifecycle PAT comes from CI_PAT_FILE.
# ── 2. Hosts (remote eitri-agent) ─────────────────────────────────────────────
# Shared agent launch flags. TOMBSTONE_GRACE / VANISH_GRACE / EXTRA are optional.
agent_flags="--state-dir $AGENT_STATE_DIR --ch-bin $CH_BIN --firmware $FIRMWARE"
[[ -n "${TOMBSTONE_GRACE:-}" ]] && agent_flags+=" --tombstone-grace $TOMBSTONE_GRACE"
[[ -n "${VANISH_GRACE:-}" ]] && agent_flags+=" --vanish-grace $VANISH_GRACE"
[[ -n "${AGENT_EXTRA_FLAGS:-}" ]] && agent_flags+=" $AGENT_EXTRA_FLAGS"
for entry in $AGENT_HOSTS; do
# entry = user@host[:port]
userhost="${entry%%:*}"
port="${entry##*:}"; [[ "$port" == "$entry" ]] && port=22
bold "Rolling agent -> $userhost (port $port)"
scp -q -P "$port" -o ConnectTimeout=10 bin/eitri-agent "$userhost:/tmp/eitri-agent-new"
scp -q -P "$port" -o ConnectTimeout=10 scripts/eitri-agent.service "$userhost:/tmp/eitri-agent.service"
# Optionally ship the guest firmware (e.g. CLOUDHV.fd) so a firmware change
# rolls WITH the deploy instead of needing a manual per-host install. When
# FIRMWARE_SRC is unset, hosts keep whatever firmware is already at $FIRMWARE.
fw_install=""
if [[ -n "${FIRMWARE_SRC:-}" ]]; then
scp -q -P "$port" -o ConnectTimeout=10 "$FIRMWARE_SRC" "$userhost:/tmp/eitri-firmware-new"
fw_install="sudo install -Dm0644 /tmp/eitri-firmware-new \"$FIRMWARE\" && rm -f /tmp/eitri-firmware-new"
fi
# Unquoted heredoc: local vars expand here; \$(...) runs on the remote.
ssh -p "$port" -o BatchMode=yes "$userhost" bash -s <<REMOTE
set -e
# Stop whichever agent shape is running: the systemd unit if adopted, plus any
# pre-unit setsid agent. SIGTERM is a "clean" signal to systemd, so a
# unit-managed agent is not resurrected by Restart=on-failure mid-roll.
sudo systemctl stop eitri-agent 2>/dev/null || true
sudo kill -TERM \$(pgrep -x eitri-agent) 2>/dev/null || true
for _ in \$(seq 1 20); do pgrep -x eitri-agent >/dev/null || break; sleep 0.5; done
if pgrep -x eitri-agent >/dev/null; then echo "agent did not stop on \$(hostname)" >&2; exit 1; fi
sudo install -m 0755 /tmp/eitri-agent-new "$AGENT_BIN"
rm -f /tmp/eitri-agent-new
$fw_install
# The agent runs under systemd. Deploy converges the unit, its flags, and the
# coverage env on every roll so hosts cannot drift from the repo's unit file;
# agent logs live in journald (journalctl -u eitri-agent).
sudo install -m 0644 /tmp/eitri-agent.service /etc/systemd/system/eitri-agent.service
rm -f /tmp/eitri-agent.service
echo "EITRI_AGENT_FLAGS=\"$agent_flags\"" | sudo tee /etc/default/eitri-agent >/dev/null
sudo rm -rf "$AGENT_GOCOVERDIR"; sudo mkdir -p "$AGENT_GOCOVERDIR"
sudo mkdir -p /etc/systemd/system/eitri-agent.service.d
printf '[Service]\nEnvironment=GOCOVERDIR=%s\n' "$AGENT_GOCOVERDIR" | sudo tee /etc/systemd/system/eitri-agent.service.d/coverage.conf >/dev/null
sudo systemctl daemon-reload
sudo systemctl enable eitri-agent >/dev/null 2>&1
sudo systemctl start eitri-agent
sleep 2
sudo systemctl is-active --quiet eitri-agent || { echo "agent FAILED to start on \$(hostname) (journalctl -u eitri-agent)" >&2; exit 1; }
pgrep -x eitri-agent >/dev/null || { echo "agent process missing on \$(hostname)" >&2; exit 1; }
echo "agent up: \$(sha256sum $AGENT_BIN | cut -c1-12) on \$(hostname) (systemd)"
REMOTE
done
# ── 3. Post-deploy verification ───────────────────────────────────────────────
bold "Verifying fleet on $SHA"
# Readiness gate: /readyz 200s only when the server's dependencies (DB open) are
# healthy. Poll briefly — the DB is local and up whenever the box is, so this
# settles fast; a persistent 503 names the failed dependency in its JSON body.
ready=""
for _ in $(seq 1 15); do
ready="$(curl -fsS "$SERVER_URL/readyz" 2>/dev/null)" && break
sleep 1
done
if [[ -n "$ready" ]]; then
echo "server ready: $ready"
else
# curl -f makes a 503 a non-zero exit, so a stuck-unready server lands here.
echo "WARNING: server not ready after deploy: $(curl -s "$SERVER_URL/readyz" 2>/dev/null)" >&2
fi
# Authenticated API verification (hosts-online, lifecycle field) used to run here
# with the admin token. That token is gone, and re-implementing a headless
# sign-in in shell would be fragile — so the authenticated proof now lives in the
# boot-gate below: eitri-smoke signs in through the real OIDC flow, lists hosts,
# and creates a VM (exercising the lifecycle field) with a minted PAT. A failing
# authed API surfaces as a boot-gate failure, which fails the deploy.
# ── 4. Boot gate ──────────────────────────────────────────────────────────────
# readyz proves the CONTROL PLANE is up, but a deploy can still
# ship a change that breaks VM BOOT (e.g. an unknown cloud-hypervisor --disk
# option that makes CH refuse to start — exactly how the image_type=raw
# regression reached the fleet). The only way to catch that is to boot a real
# VM. This gate does so and FAILS the deploy if it can't. Set SKIP_BOOT_SMOKE=1
# to bypass for a server-only / docs-only change.
if [[ "${SKIP_BOOT_SMOKE:-0}" == "1" ]]; then
echo "(boot gate skipped: SKIP_BOOT_SMOKE=1)"
else
bold "Boot gate: prove the credential chain, then create a throwaway VM and confirm it boots"
# Like a browser, the OIDC flow only works when the client visits the server
# at exactly oidc.public_url (the state cookie is host-scoped and the issuer
# redirects to the registered public_url callback), so the gate targets
# public_url rather than the loopback health-check URL.
smoke_server_url="$(jq -r '.oidc.public_url // empty' "$SERVER_CONFIG")"
smoke_server_url="${smoke_server_url:-$SERVER_URL}"
# The lifecycle half needs an operator-minted, tenant-scoped PAT: the fleet's
# hosts belong to the operator's tenant, so the machine identity's JIT tenant
# owns none. Minting the PAT is a one-time human step — fail with instructions
# rather than a confusing 403 deep in the gate.
if [[ ! -f "$CI_PAT_FILE" ]]; then
cat >&2 <<-EOF
DEPLOY BLOCKED: operator PAT not found at $CI_PAT_FILE
The boot-gate's VM-lifecycle check runs as the tenant operator, not as the
machine identity. Mint that PAT once:
1. Sign in at $smoke_server_url in a browser as the operator whose
tenant owns the fleet's hosts.
2. Open Settings and mint a NON-EXPIRING personal access token named
"deploy".
3. Save its value to $CI_PAT_FILE and lock it down:
umask 077; printf %s '<token>' > $CI_PAT_FILE; chmod 600 $CI_PAT_FILE
4. Re-run: make deploy
EOF
exit 1
fi
# eitri-smoke reads its config from the environment, so export what it needs.
# Phase 1 signs in as the machine identity through the real OIDC flow
# (CI_USER/CI_PASSWORD_FILE), mints a short-lived PAT, and proves it resolves
# to a tenant. Phase 2 authenticates the VM lifecycle with the operator PAT in
# CI_PAT_FILE and derives its tenant via Me() — no admin token, no hardcoded
# tenant.
# COVER_OUT triggers the post-gate coverage merge (server+agent); AGENT_HOSTS
# is here for that merge alone — the gate itself proves everything through
# the API, the console and the gate, and logs into no host in the fleet.
export SERVER_URL="$smoke_server_url"
export CI_USER CI_PASSWORD_FILE CI_PAT_FILE AGENT_HOSTS SERVER_GOCOVERDIR AGENT_GOCOVERDIR
export SMOKE_GATE SMOKE_VM_USER SMOKE_USER_CA_FILE
if COVER_OUT="$REPO_ROOT/coverage/integration" "$REPO_ROOT/bin/eitri-smoke"; then
echo "boot gate: PASS"
else
echo "DEPLOY FAILED boot gate: sign-in or VM boot failed after this deploy." >&2
echo " The fleet is likely serving a boot-breaking change — investigate before relying on it." >&2
exit 1
fi
fi
bold "Deploy complete: $SHA"