scripts/ship.sh
Ref: Size: 48.4 KiB History
#!/usr/bin/env bash
#
# Deploy a tagged tree to one plane. A release is this script run twice: first
# against stg with a pre-release tag, then — same script, same stages, the same
# artifacts rebuilt from the same tag — against prod. Nothing reaches prod that
# a machine has not already done to stg.
#
# scripts/ship.sh --target <stg|prod> --tag <vX.Y.Z[-pre.N]> [options]
#
# --from <n> resume at stage n after a partial failure
# --skip-smoke deploy and converge the fleet, but do not prove it
# (stage 10 still reports). On a machine whose ship.env
# names no credential, stage 8 is skipped with a warning
# and the fleet stays on the release before this one.
# --render-only print the rendered manifests and exit; touches nothing
#
# Every stage is idempotent: re-running from the top is always safe and is the
# documented default. Every remote or destructive action is gated on --target,
# and the plane's values come from deploy/server/plane.<target>.env — the
# hostnames, the host-port triple, and which plane gets backups.
#
# A tag publishes once. scripts/release.sh builds byte-reproducibly, so a
# re-run rebuilds the artifacts the target already serves; stage 3 recognises
# them, leaves the published site image where it is, and carries on. What it
# will not do is put different bytes behind a URL the world has already
# fetched: /dl/<tag>/ is served immutable, so those bytes are the tag, and
# changing them means cutting a new one.
#
# The pipeline reads the plane's config Secret and validates it. It never
# writes one: a script that can write config secrets is a script that can
# overwrite prod's.
#
# Site-specific values (registries, credential paths, the plane's fleet) come
# from ~/eitri-deploy/<target>/ship.env — see scripts/ship.env.example. No
# secrets and no site-specific values live in the repo.
set -euo pipefail
TARGET=""
TAG=""
FROM=1
SKIP_SMOKE=0
RENDER_ONLY=0
usage() {
sed -n '2,35p' "$0" | sed 's/^#\{1,2\} \{0,1\}//'
exit "${1:-1}"
}
while [[ $# -gt 0 ]]; do
case "$1" in
--target) TARGET="${2:-}"; shift 2 ;;
--tag) TAG="${2:-}"; shift 2 ;;
--from) FROM="${2:-}"; shift 2 ;;
--skip-smoke) SKIP_SMOKE=1; shift ;;
--render-only) RENDER_ONLY=1; shift ;;
-h | --help) usage 0 ;;
*) echo "ship: unknown argument: $1" >&2; usage ;;
esac
done
case "$TARGET" in
stg | prod) ;;
*) echo "ship: --target must be stg or prod (got '${TARGET}')" >&2; usage ;;
esac
[[ -n "$TAG" ]] || { echo "ship: --tag is required" >&2; usage; }
[[ "$FROM" =~ ^([1-9]|10)$ ]] || { echo "ship: --from must be a stage number 1-10" >&2; exit 1; }
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$REPO_ROOT"
bold() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; }
fail() { printf '\nship: %s\n' "$*" >&2; exit 1; }
warn() { printf 'ship: WARNING — %s\n' "$*" >&2; }
# ── Plane values (committed) and site values (not) ────────────────────────────
PLANE_ENV="$REPO_ROOT/deploy/server/plane.$TARGET.env"
[[ -f "$PLANE_ENV" ]] || fail "no plane file for target $TARGET at $PLANE_ENV"
# shellcheck disable=SC1090
source "$PLANE_ENV"
SHIP_ENV="${EITRI_SHIP_ENV:-$HOME/eitri-deploy/$TARGET/ship.env}"
if [[ ! -f "$SHIP_ENV" ]]; then
fail "config not found: $SHIP_ENV
cp scripts/ship.env.example \"$SHIP_ENV\" and edit it."
fi
# shellcheck disable=SC1090
source "$SHIP_ENV"
# The build scripts (release/site-image/server-image) read the registry names
# from $EITRI_DEPLOY_ENV. Point them at THIS plane's file so a prod run can
# never pick up the dev fleet's deploy.env by accident.
export EITRI_DEPLOY_ENV="$SHIP_ENV"
# The variables the templates may reference. envsubst is called with exactly
# this list, so an unrelated $FOO added to a manifest later is passed through
# rather than blanked — and every name here is checked non-empty first, because
# envsubst's own answer to an unset variable is to substitute nothing at all.
RENDER_VARS=(
NAMESPACE NODE_NAME IMAGE_ARCH
CONSOLE_HOST API_HOST GATE_HOST SYNC_HOST SITE_HOST
HTTP_PORT SYNC_PORT GATE_PORT
CONFIG_SECRET PVC_NAME PVC_SIZE TLS_SECRET SITE_TLS_SECRET
SERVER_IMAGE SITE_IMAGE TAG
)
# The backup image is referenced only by the CronJob, which is applied only
# where BACKUPS=1 — so a plane without backups never has to name one. It carries
# its own tag rather than $TAG: it holds sqlite and nothing of eitri's, and
# rebuilding it every release would be work with no output.
if [[ "${BACKUPS:-0}" == "1" ]]; then
RENDER_VARS+=(BACKUP_IMAGE)
fi
# The bundled issuer's values join the list only on a plane that runs one, so a
# plane without it never carries them. An ${OIDC_*} reference added to a SHARED
# template later would then survive rendering as literal text and fail the apply
# loudly, rather than being blanked into a plausible-looking manifest.
if [[ "$LOCAL_OIDC" == "1" ]]; then
RENDER_VARS+=(OIDC_HOST OIDC_PORT OIDC_TLS_SECRET OIDC_CONFIG_SECRET OIDC_PVC_NAME OIDC_PVC_SIZE)
fi
export TAG
require_render_vars() {
local missing=() name
for name in "${RENDER_VARS[@]}"; do
[[ -n "${!name:-}" ]] || missing+=("$name")
done
if [[ ${#missing[@]} -gt 0 ]]; then
fail "unset or empty: ${missing[*]}
plane values live in $PLANE_ENV; site values in $SHIP_ENV"
fi
export "${RENDER_VARS[@]?}"
}
# The manifest sets, in the order they are applied. The shape goes on first; the
# two Deployments follow in separate stages because their ORDER is load-bearing
# (see stage 6).
SHAPE_MANIFESTS=(
namespace.yaml middleware.yaml
pvc.yaml service.yaml certificate.yaml ingressroute.yaml
site-service.yaml site-certificate.yaml site-ingressroute.yaml
)
if [[ "$BACKUPS" == "1" ]]; then
SHAPE_MANIFESTS+=(backup-cronjob.yaml)
fi
if [[ "$LOCAL_OIDC" == "1" ]]; then
SHAPE_MANIFESTS+=(oidc.yaml)
fi
# The Deployments that carry the release tag, rolled in stage 7. The issuer runs
# from the server's image so the two are always the same build.
SERVER_MANIFESTS=(deployment.yaml)
if [[ "$LOCAL_OIDC" == "1" ]]; then
SERVER_MANIFESTS+=(oidc-deployment.yaml)
fi
# render prints one manifest with this plane's values substituted.
render() {
local vars="" name
for name in "${RENDER_VARS[@]}"; do vars+="\${$name} "; done
envsubst "$vars" <"$REPO_ROOT/deploy/server/$1"
}
render_all() {
local f
for f in "${SHAPE_MANIFESTS[@]}" site-deployment.yaml "${SERVER_MANIFESTS[@]}"; do
echo "---"
echo "# deploy/server/$f"
render "$f"
done
}
# ── --render-only: read the manifests, touch nothing ──────────────────────────
if [[ "$RENDER_ONLY" == "1" ]]; then
require_render_vars
render_all
exit 0
fi
bold "Shipping $TAG to $TARGET (namespace $NAMESPACE)"
echo "console https://$CONSOLE_HOST api https://$API_HOST site https://$SITE_HOST"
echo "gate $GATE_HOST:$GATE_PORT sync $SYNC_HOST:$SYNC_PORT http :$HTTP_PORT"
[[ "$FROM" -gt 1 ]] && echo "(resuming at stage $FROM)"
# ── Preflight: what the run will need, before it changes anything ─────────────
#
# Stages 8 and 9 are the two that authenticate to the plane, and both of them
# run AFTER stages 6 and 7 have rolled it. A run that cannot finish has to say
# so while the plane is still serving what it served this morning: the state to
# avoid is a site and a server at $TAG with the fleet a release behind, which is
# the half-shipped state stage 8 exists to end.
#
# What is required depends on which stages this run will actually reach. The
# smoke cannot be driven without a credential at all, so a missing one is fatal
# here. Convergence is not: rolling a plane from a machine that holds no
# credential, with the proof left to someone who does, is a workflow this
# script has always allowed — and stage 8 skips itself and says loudly what the
# fleet is still running, rather than dying with the plane already rolled.
CONVERGE_WILL_RUN=0
SMOKE_WILL_RUN=0
CONVERGE_SKIPPED=0
[[ "$FROM" -le 8 ]] && CONVERGE_WILL_RUN=1
[[ "$FROM" -le 9 && "$SKIP_SMOKE" != "1" ]] && SMOKE_WILL_RUN=1
HAVE_CREDENTIAL=0
[[ -n "${CI_USER:-}" || -n "${CI_PAT_FILE:-}" ]] && HAVE_CREDENTIAL=1
if [[ "$SMOKE_WILL_RUN" == "1" && "$HAVE_CREDENTIAL" == "0" ]]; then
fail "$SHIP_ENV names no credential, so nothing in this run could authenticate
to the plane — the smoke at stage 9 needs one, as does the convergence at
stage 8 before it. Nothing has been deployed.
Set CI_USER + CI_PASSWORD_FILE for a plane with a password issuer, or
CI_PAT_FILE for one fronted by an external identity provider — or pass
--skip-smoke to roll this plane from a machine that holds neither, in
which case stage 8 is skipped and the fleet stays where it is."
fi
# A credential that is named but unreachable fails for the same reason and at
# the same cost, so it is established here too. These are file tests: the
# pipeline never reads a credential it does not have to.
if [[ "$CONVERGE_WILL_RUN" == "1" || "$SMOKE_WILL_RUN" == "1" ]]; then
if [[ -n "${CI_PAT_FILE:-}" ]]; then
[[ -r "$CI_PAT_FILE" ]] ||
fail "CI_PAT_FILE names $CI_PAT_FILE, which cannot be read. Nothing has been deployed."
fi
# CI_USER's password is what the smoke signs in with whenever CI_USER is
# named, and what stage 8 signs in with only when there is no PAT for it to
# prefer. Required exactly where it is used.
if [[ -n "${CI_USER:-}" ]] && [[ "$SMOKE_WILL_RUN" == "1" || -z "${CI_PAT_FILE:-}" ]]; then
[[ -n "${CI_PASSWORD_FILE:-}" ]] ||
fail "$SHIP_ENV sets CI_USER=$CI_USER but no CI_PASSWORD_FILE, so nothing can
sign in as that identity. Nothing has been deployed."
[[ -r "$CI_PASSWORD_FILE" ]] ||
fail "CI_PASSWORD_FILE names $CI_PASSWORD_FILE, which cannot be read. Nothing has been deployed."
fi
fi
# ── 1. Verify the tag ─────────────────────────────────────────────────────────
if [[ "$FROM" -le 1 ]]; then
bold "1. Verify the tag"
git diff --quiet || fail "working tree has unstaged changes; ship from a clean tagged tree"
git diff --cached --quiet || fail "working tree has staged changes; ship from a clean tagged tree"
git rev-parse "$TAG^{commit}" >/dev/null 2>&1 || fail "tag $TAG does not resolve to a commit"
described="$(git describe --tags --exact-match 2>/dev/null || true)"
[[ "$described" == "$TAG" ]] || fail "HEAD is at '${described:-no tag}', not $TAG — check out the tag first"
# The version must be parsable by the SAME code the fleet orders versions
# with, because an unparsable one silently disables the upgrade button
# everywhere at once. Ask that code rather than re-deriving its rule here:
# internal/version owns the ordering both the server and the agent use, and
# Ordered is that package's own name for "this string has a place in it".
probe="$(mktemp -d "$REPO_ROOT/.shipcheck.XXXXXX")"
trap 'rm -rf "$probe"' EXIT
cat >"$probe/main.go" <<-'PROBE'
package main
import (
"fmt"
"os"
"github.com/a73x/eitri/internal/version"
)
func main() {
if !version.Ordered(os.Args[1]) {
fmt.Println("unparsable")
return
}
fmt.Println("parsable")
}
PROBE
parsable="$(go run "./$(basename "$probe")" "$TAG")"
rm -rf "$probe"
trap - EXIT
if [[ "$parsable" != "parsable" ]]; then
fail "$TAG is unparsable to internal/version.Ordered.
Every agent orders versions with that code, so shipping this tag would
disable the upgrade button fleet-wide. Releases are vX.Y.Z and
pre-releases are vX.Y.Z-pre.N."
fi
echo "tag ok: $TAG at $(git rev-parse --short HEAD)"
fi
# ── 2. Build the release artifacts ────────────────────────────────────────────
# The manifest base is the one build input the two planes legitimately differ
# on, and it is why the stg run PROVES the download and upgrade paths rather
# than merely rehearsing them: the artifacts stg publishes name stg's own /dl.
if [[ "$FROM" -le 2 ]]; then
bold "2. Build release artifacts -> dist/$TAG"
MANIFEST_BASE="https://$SITE_HOST/dl/$TAG" "$REPO_ROOT/scripts/release.sh"
fi
# ── 3. Build and push the images ──────────────────────────────────────────────
#
# The site image bakes dist/$TAG in, so the artifacts stage 2 built are what
# /dl serves — the two image builds are called directly rather than through
# `make site-image`, whose own `make release` would rebuild those artifacts
# against the default manifest base and undo stage 2.
if [[ "$FROM" -le 3 ]]; then
bold "3. Build and push images at $TAG"
[[ -d "dist/$TAG" ]] || fail "dist/$TAG missing — run from stage 2"
# Set by the publish check below when the plane already serves exactly
# these artifacts, which is the one case where the site image is left
# alone.
site_published=0
# Publish once, per plane. The site image about to be pushed bakes
# dist/$TAG in, so this is the last moment before those bytes become
# https://$SITE_HOST/dl/$TAG/ — a URL served immutable and cached
# accordingly, downloaded by hand and by every agent taking the upgrade.
# Ask the plane what it already serves for this tag and compare.
#
# The two planes legitimately hold different bytes for one tag: the
# artifacts name their own plane's /dl (stage 2), which is what makes the
# stg run a proof rather than a rehearsal. So the comparison is against
# THIS target and no other.
sums_url="https://$SITE_HOST/dl/$TAG/SHA256SUMS"
published="$(mktemp)"
# curl writes 000 for a request that never got a status line; the explicit
# assignment on failure keeps a non-zero exit from appending a second one.
code="$(curl -sS --max-time 30 -o "$published" -w '%{http_code}' "$sums_url")" || code=000
case "$code" in
200)
if cmp -s "dist/$TAG/SHA256SUMS" "$published"; then
echo "publish ok: $TAG is already published at $SITE_HOST with these exact bytes"
site_published=1
else
rm -f "$published"
fail "$TAG is ALREADY PUBLISHED at $SITE_HOST, with different bytes.
$sums_url does not match dist/$TAG/SHA256SUMS. A versioned download URL
is immutable — someone may already hold the published artifacts, and an
agent verifies its upgrade against the sha the manifest named. The fix
is a new tag, never a new payload under the old one.
(The build is reproducible, so an honest re-run of the same tree
matches. A difference means the tag moved or the tree did.)"
fi
;;
404)
echo "publish ok: $TAG is not yet published at $SITE_HOST — this is its first publish"
;;
000)
warn "$SITE_HOST did not answer for $sums_url, so what it serves for $TAG is
unknown and this run publishes over it. Expected while a plane is being
brought up for the first time; anywhere else, check the site is serving
before letting the push proceed."
;;
*)
rm -f "$published"
fail "$sums_url answered HTTP $code, so whether $TAG is already published
cannot be established. Resolve it before publishing over the answer."
;;
esac
rm -f "$published"
make -C "$REPO_ROOT" web
"$REPO_ROOT/scripts/server-image.sh"
# A site image already published under this tag is left where it is. Its
# content would be identical — the same dist/$TAG baked into the same
# webroot — but a rebuild lands under a new digest, and the registry tag
# for an already-published release quietly pointing somewhere new is the
# confusion this stage exists to end. Stage 6 rolls the digest already
# behind the tag.
#
# The server image is rebuilt and pushed on every branch. Nothing outside
# the cluster holds a contract on its bytes, and making it conditional
# would only complicate what --from resumes into.
if [[ "$site_published" == "1" ]]; then
echo "site image: skipped — $SITE_HOST already serves /dl/$TAG from the image at this tag"
else
make -C "$REPO_ROOT" site SITE_DIST="dist/$TAG"
"$REPO_ROOT/scripts/site-image.sh"
fi
fi
# ── 4. Config schema and value check ──────────────────────────────────────────
# The stage that justifies the script. Two checks, both against the Secret this
# plane is about to run with: does the tagged tree's schema agree with the
# committed contract, and does the Secret agree with the plane's manifests?
#
# The Secret is read, never written, and never printed. Schema failures name
# keys only. The plane-agreement checks below DO print the values they compare —
# listen addresses and public hostnames, none of them secret, and the mismatch
# is the whole point of the check.
if [[ "$FROM" -le 4 ]]; then
bold "4. Check the config schema and the plane's values"
require_render_vars
secret_json="$(kubectl -n "$NAMESPACE" get secret "$CONFIG_SECRET" \
-o jsonpath='{.data.server\.json}' 2>/dev/null | base64 -d)" ||
fail "cannot read secret $CONFIG_SECRET in namespace $NAMESPACE.
Create it once by hand (the pipeline never writes config secrets):
kubectl -n $NAMESPACE create secret generic $CONFIG_SECRET \\
--from-file=server.json=\$HOME/eitri-deploy/$TARGET/server.json"
printf '%s' "$secret_json" | jq -e . >/dev/null 2>&1 ||
fail "secret $CONFIG_SECRET does not hold a server.json key with valid JSON"
CONTRACT="$REPO_ROOT/deploy/server/config.required"
SCHEMA_SRC="$REPO_ROOT/internal/server/config/config.go"
# Inventory the keys the TAGGED TREE declares, as JSON paths. Nested structs
# reach a path through a prefix declared in the contract, so a new nested
# struct cannot slip in unclassified — it is reported here by name.
schema_keys="$(awk '
FNR == NR {
if ($0 ~ /^# struct-prefix:/) {
line = $0
sub(/^# struct-prefix:[ \t]*/, "", line)
eq = index(line, "=")
prefix[substr(line, 1, eq - 1)] = substr(line, eq + 1)
}
next
}
/^type [A-Za-z_]+ struct \{/ { st = $2; next }
st != "" && /^\}/ { st = ""; next }
st != "" && match($0, /json:"[^"]+"/) {
tag = substr($0, RSTART + 6, RLENGTH - 7)
sub(/,.*/, "", tag)
if (tag == "" || tag == "-") next
if (!(st in prefix)) { print "!" st; next }
print prefix[st] tag
}
' "$CONTRACT" "$SCHEMA_SRC")"
undeclared="$(printf '%s\n' "$schema_keys" | grep '^!' | sort -u | tr -d '!' || true)"
if [[ -n "$undeclared" ]]; then
fail "the tagged tree declares config struct(s) with no JSON path:
$(echo "$undeclared" | tr '\n' ' ')
Add a '# struct-prefix: <Struct>=<json.path.>' line to
deploy/server/config.required and classify the keys it carries."
fi
# The contract's own key list, for the two set comparisons below.
contract_keys="$(grep -v '^[[:space:]]*#' "$CONTRACT" | awk 'NF {print $1}')"
# jq_path turns a contract path into a jq expression. A "*" segment stands
# for a map key and iterates every entry.
jq_path() { printf '.%s' "${1//.\*./[].}"; }
# Collect the path's values and require EVERY one of them: a wildcard path
# holds for all the map's entries or for none. A plane with an arm64 image
# and a blank amd64 one is not half-configured, it is broken for half its
# hosts. An empty collection is a missing key.
cfg_has() {
printf '%s' "$secret_json" |
jq -e "[$(jq_path "$1")] | length > 0 and all(. != null and . != \"\")" >/dev/null 2>&1
}
schema_fail=0
while read -r key status; do
[[ -z "$key" || "$key" == \#* ]] && continue
case "$status" in
required)
if ! cfg_has "$key"; then
echo "FAIL: $CONFIG_SECRET does not set required key '$key'" >&2
echo " patch it in (values never come from this repo):" >&2
echo " kubectl -n $NAMESPACE get secret $CONFIG_SECRET -o jsonpath='{.data.server\\.json}' | base64 -d > /tmp/server.json" >&2
echo " # add \"$key\", then:" >&2
echo " kubectl -n $NAMESPACE create secret generic $CONFIG_SECRET --from-file=server.json=/tmp/server.json --dry-run=client -o yaml | kubectl apply -f -" >&2
schema_fail=1
fi
;;
retired)
cfg_has "$key" && warn "$CONFIG_SECRET still sets retired key '$key'; the server ignores it"
;;
optional) ;;
*) fail "$CONTRACT classifies '$key' as '$status'; want required, optional or retired" ;;
esac
# A contract entry the schema no longer declares is stale documentation,
# not a deployment hazard — say so and carry on.
grep -qxF "$key" <<<"$schema_keys" ||
warn "$CONTRACT classifies '$key', which the tagged tree no longer declares"
done < <(grep -v '^[[:space:]]*#' "$CONTRACT" | grep -v '^[[:space:]]*$')
# The other direction, and the one the v0.0.3 incident needed: a key the tree
# grew that nobody classified.
while read -r key; do
[[ -z "$key" ]] && continue
grep -qxF "$key" <<<"$contract_keys" || {
echo "FAIL: the tagged tree grew config key '$key'" >&2
echo " classify it in deploy/server/config.required and, if it is required," >&2
echo " set it in the $NAMESPACE Secret $CONFIG_SECRET before shipping." >&2
schema_fail=1
}
done <<<"$schema_keys"
# Plane agreement. Reported in the same pass as the schema findings above,
# not after a re-run: this stage exists to be hit, and an operator fixing a
# config should see everything wrong with it at once. Each of these disagreeing is a hosted-shape failure that
# presents as something else entirely: a wrong gate domain reads as "pubkey
# denied", a wrong advertise_quic as a host that enrolls and never syncs.
agree_fail=0
assert_cfg() {
local path="$1" want="$2" got
got="$(printf '%s' "$secret_json" | jq -r "$(jq_path "$path") // \"\"")"
if [[ "$got" != "$want" ]]; then
echo "FAIL: $CONFIG_SECRET has $path = '$got', but this plane's manifests say '$want'" >&2
agree_fail=1
fi
}
assert_cfg http_listen ":$HTTP_PORT"
assert_cfg quic_listen ":$SYNC_PORT"
assert_cfg ssh_listen ":$GATE_PORT"
assert_cfg ssh_gate_domain "$GATE_HOST"
assert_cfg advertise_http "https://$CONSOLE_HOST"
assert_cfg advertise_quic "$SYNC_HOST:$SYNC_PORT"
assert_cfg oidc.public_url "https://$CONSOLE_HOST"
# A plane running the bundled issuer has a second config to keep in step, and
# the two ways it drifts both present as something else: a server pointed at
# the wrong issuer fails discovery ("sign-in temporarily unavailable"), and a
# redirect_url that does not match the server's callback exactly is a 400 at
# the end of an otherwise working login. Neither says what is actually wrong.
if [[ "$LOCAL_OIDC" == "1" ]]; then
assert_cfg oidc.issuer "https://$OIDC_HOST"
issuer_json="$(kubectl -n "$NAMESPACE" get secret "$OIDC_CONFIG_SECRET" \
-o jsonpath='{.data.eitri-oidc\.json}' 2>/dev/null | base64 -d)" ||
fail "cannot read secret $OIDC_CONFIG_SECRET in namespace $NAMESPACE.
This plane runs the bundled issuer; create its config once by hand
(see deploy/server/README.md, 'One-time bring-up of a plane')."
assert_issuer() {
local path="$1" want="$2" got
got="$(printf '%s' "$issuer_json" | jq -r "$path // \"\"")"
if [[ "$got" != "$want" ]]; then
echo "FAIL: $OIDC_CONFIG_SECRET has $path = '$got', want '$want'" >&2
agree_fail=1
fi
}
assert_issuer .issuer "https://$OIDC_HOST"
assert_issuer .listen ":$OIDC_PORT"
assert_issuer '.clients[0].redirect_url' "https://$CONSOLE_HOST/auth/callback"
# The client the server presents must be one the issuer knows.
server_client="$(printf '%s' "$secret_json" | jq -r '.oidc.client_id // ""')"
printf '%s' "$issuer_json" | jq -e --arg id "$server_client" \
'[.clients[].id] | index($id) != null' >/dev/null 2>&1 ||
{
echo "FAIL: the server signs in as client '$server_client', which $OIDC_CONFIG_SECRET does not register" >&2
agree_fail=1
}
fi
if [[ "$schema_fail" != "0" || "$agree_fail" != "0" ]]; then
fail "$NAMESPACE's config and the $TAG tree do not agree (see above); nothing was deployed"
fi
echo "config ok: schema classified, required keys present, plane values agree"
fi
# ── 5. Render and apply the shape ─────────────────────────────────────────────
# Everything except the two Deployments: namespace, middleware, storage,
# service, certificates and routes. Applying these is idempotent and cannot
# restart anything.
if [[ "$FROM" -le 5 ]]; then
bold "5. Apply the plane's shape to $NAMESPACE"
require_render_vars
for f in "${SHAPE_MANIFESTS[@]}"; do
echo " $f"
render "$f" | kubectl apply -f -
done
fi
# ── 6. Roll the site ──────────────────────────────────────────────────────────
# ORDER IS LOAD-BEARING. The server fetches the release manifest at boot and
# pins the answer for 24 hours, so a server rolled ahead of its site serves the
# previous release to the whole fleet for a day.
#
# Nothing follows this but the roll. site/nginx.conf states the cache contract
# the site is served under — /dl/v* immutable, everything else no-cache — and
# stage 3 holds the tag to it, so the new URLs were never cached under an older
# answer and the mutable ones are re-fetched on their own.
if [[ "$FROM" -le 6 ]]; then
bold "6. Roll the site to $TAG (before the server)"
require_render_vars
render site-deployment.yaml | kubectl apply -f -
kubectl -n "$NAMESPACE" rollout status deployment/web --timeout=5m
echo "site serving $TAG at https://$SITE_HOST/dl/$TAG/"
fi
# ── 7. Roll the server ────────────────────────────────────────────────────────
# Recreate strategy: a brief control-plane gap. Agents redial on their own.
if [[ "$FROM" -le 7 ]]; then
bold "7. Roll the control plane to $TAG"
require_render_vars
for f in "${SERVER_MANIFESTS[@]}"; do
echo " $f"
render "$f" | kubectl apply -f -
done
kubectl -n "$NAMESPACE" rollout status deployment/eitri-server --timeout=5m
if [[ "$LOCAL_OIDC" == "1" ]]; then
kubectl -n "$NAMESPACE" rollout status deployment/eitri-oidc --timeout=5m
fi
fi
# ── 8. Converge the fleet's agents ────────────────────────────────────────────
#
# Nothing in eitri upgrades itself on a schedule. An agent takes a new binary
# only when an operator offers it one, so a bad release stops at the first host
# instead of running through the fleet. That is deliberate, and it has a
# consequence for this script: the smoke below drives a whole VM cycle — create,
# boot, gate SSH, a published port — on whichever host the plane places it on,
# and a leg this release adds does not exist on an agent from the release
# before. On v0.0.5 that is what happened, on both planes. The plane served the
# new tag, the hosts still ran the old agent, the smoke failed on the leg their
# agents did not have, and the release waited while an operator upgraded three
# machines by hand and resumed the run.
#
# So the click lives here now, once per release, between the roll and the proof.
# Nothing about an offer changed: it is still per host, still audited, still
# refused for a host that is offline or not behind. What changed is that the
# pipeline no longer hands the smoke a fleet it already knows is too old.
#
# The stage runs even under --skip-smoke. A fleet running the release is part of
# shipping it, not part of proving it — a plane at $TAG with agents a release
# behind is exactly the half-shipped state this stage exists to end. Deploy and
# converge, do not prove: that is what --skip-smoke means here.
#
# The one thing that stops it is having nothing to act as. Convergence is an
# authenticated act, and rolling a plane from a machine that holds no credential
# is a workflow this script allowed before this stage existed. It still does —
# the stage steps aside and says, in the warning and again in stage 10's report,
# exactly what the fleet is still running and how to finish it. What it will not
# do is take the plane to $TAG and then fail for want of a token, which would
# leave behind the very state described above. The preflight has already refused
# the run outright if the smoke was going to need that credential.
if [[ "$FROM" -le 8 && "$HAVE_CREDENTIAL" == "1" ]]; then
bold "8. Converge the fleet's agents to $TAG"
# How long a host may take to land the new agent before the ship gives up on
# it. The work behind an offer is a download, a sha256, a binary swap and a
# re-exec, and the offer itself rides the host's next snapshot — one agent
# tick, ~10s — so on a healthy link the whole thing is done in tens of
# seconds. The console stops calling an outstanding offer "in flight" and
# starts calling it stuck at 180s (UPGRADE_STUCK_S, web/src/lib/fleet.svelte.ts),
# and a release must not fail before the console would even raise an eyebrow.
# 300s is that threshold plus a whole further offer cycle of slack for a slow
# link or a large artifact, and still inside the attention of the person who
# started the ship.
CONVERGE_TIMEOUT_S=300
CONVERGE_POLL_S=5
# How long a DARK host is waited on before it is written off as parked. This
# stage runs seconds after stage 7 rolled the server, and that roll ended
# every agent's sync session at once — so for as long as it takes the new
# pod to serve and the agents to come back (a 5s reconnect backoff, several
# attempts while the plane is still starting), the whole fleet reads as
# disconnected here. A host that is dark inside this window is waited on, not
# skipped; past it, the warn-and-skip below stands, because a genuinely
# parked machine must never hold a release. 90s is the roll plus a handful of
# backoffs, and well inside CONVERGE_TIMEOUT_S so the grace can never be what
# ends the loop.
DARK_GRACE_S=90
API="https://$CONSOLE_HOST"
# The credential is the one the plane already names for the smoke, chosen the
# same way the smoke chooses it for the VM lifecycle: the operator PAT from
# CI_PAT_FILE where there is one, and otherwise a console session signed in
# as CI_USER. The order matters — a plane may name both, and the PAT is then
# the identity whose tenant owns the fleet's hosts, which is exactly the
# identity that must offer them an upgrade and the one the smoke will place a
# VM as. A stage with a credential of its own would be a third token to
# issue, rotate and lose.
#
# Everything the credential touches is a file in one 0700 directory that goes
# away with the run: the bearer header and the cookie jar are files rather
# than arguments, because an argument is visible in ps to every user on this
# machine.
cdir="$(mktemp -d)"
chmod 700 "$cdir"
trap 'rm -rf "$cdir"' EXIT
curl_auth=()
if [[ -n "${CI_PAT_FILE:-}" ]]; then
[[ -r "$CI_PAT_FILE" ]] || fail "CI_PAT_FILE names $CI_PAT_FILE, which cannot be read"
(umask 077; printf 'Authorization: Bearer %s\n' "$(cat "$CI_PAT_FILE")" >"$cdir/auth")
curl_auth=(-H "@$cdir/auth")
echo "acting as the operator whose PAT CI_PAT_FILE names"
else
: "${CI_PASSWORD_FILE:?set it alongside CI_USER in $SHIP_ENV}"
[[ -r "$CI_PASSWORD_FILE" ]] || fail "CI_PASSWORD_FILE names $CI_PASSWORD_FILE, which cannot be read"
# The same headless sign-in the smoke's credential chain walks, for the
# same reason it works there: GET /auth/login lands on the issuer's login
# form after the redirects, and the form takes the credentials posted back
# to the URL we landed on. Success is the session cookie in the jar and
# nothing else — a wrong password re-renders the form as a plain 200.
(umask 077; printf '%s' "$(cat "$CI_PASSWORD_FILE")" >"$cdir/pw")
# Retried, because stage 7 rolled BOTH the server and the issuer seconds
# ago and a sign-in is only as settled as the pair of them: an issuer still
# coming up walks the redirect chain and sets no session at the end of it —
# which is, by the line above, indistinguishable from a refused password.
# So the retry is blind on purpose. There is nothing in the answer to tell
# a plane that is not ready from a password that is wrong, and the wrong
# password costs a minute before printing the same failure this stage
# always printed. A stg ship lost a run to that race and then signed in,
# unchanged, a minute later.
SIGNIN_ATTEMPTS=6
SIGNIN_RETRY_S=10
signin_attempt=1
while :; do
# A jar from a failed attempt must never be what the next one reads:
# the session cookie in it is the only thing that says "signed in".
rm -f "$cdir/jar"
signin_err=""
form_url="$(curl -sS -L --max-time 30 -c "$cdir/jar" -b "$cdir/jar" \
-o /dev/null -w '%{url_effective}' "$API/auth/login")" ||
signin_err="GET $API/auth/login did not answer, so nothing can sign in to the plane just rolled"
if [[ -z "$signin_err" ]]; then
curl -sS -L --max-time 30 -c "$cdir/jar" -b "$cdir/jar" -o /dev/null \
--data-urlencode "email=$CI_USER" --data-urlencode "password@$cdir/pw" "$form_url" ||
signin_err="posting $CI_USER's credentials to the issuer's login form at $form_url failed"
fi
if [[ -z "$signin_err" ]]; then
grep -q eitri_session "$cdir/jar" 2>/dev/null ||
signin_err="signing in as $CI_USER left no session — check the identity exists in this
plane's issuer and that CI_PASSWORD_FILE holds its current password."
fi
[[ -n "$signin_err" ]] || break
[[ "$signin_attempt" -lt "$SIGNIN_ATTEMPTS" ]] ||
fail "$signin_err
This was attempt $signin_attempt of $SIGNIN_ATTEMPTS, over $(( (SIGNIN_ATTEMPTS - 1) * SIGNIN_RETRY_S ))s, so the issuer has
had time to settle after the stage-7 roll and the answer is not changing."
echo " sign-in attempt $signin_attempt of $SIGNIN_ATTEMPTS did not take — the issuer and the server were rolled"
echo " seconds ago and a plane still starting cannot hand out a session; retrying in ${SIGNIN_RETRY_S}s"
signin_attempt=$((signin_attempt + 1))
sleep "$SIGNIN_RETRY_S"
done
# The session in the jar IS the credential from here: the API takes the
# console cookie exactly where it takes a bearer token, so this stage needs
# no PAT of its own and leaves no token behind it.
curl_auth=(-b "$cdir/jar")
echo "signed in as $CI_USER"
fi
# One round trip: the body lands in api_body and the status in api_status,
# 000 when there was no answer at all (curl's own convention, and stage 3's).
api_body=""
api_status=0
api_call() {
api_status="$(curl -sS --max-time 30 -X "$1" "${curl_auth[@]}" \
-o "$cdir/resp" -w '%{http_code}' "$API$2")" || api_status=000
api_body="$(cat "$cdir/resp" 2>/dev/null || true)"
}
# A host has converged when it reports THIS tag, compared as a string: the
# agent's version is stamped from the tag it was built at (scripts/release.sh)
# and stage 1 has already refused to ship a HEAD that is not exactly $TAG, so
# the two are the same characters or the host is not running this release.
# Every ordering question — is this host behind, may it be offered anything —
# stays with the plane, which answers it with internal/server/release and
# tells us in a status code.
started=$SECONDS
last_note=$SECONDS
offered=" " # hosts this stage has offered the upgrade to
announced=" " # hosts already reported as converged
skipped=" " # disconnected hosts, warned about once
notbehind=" " # hosts the plane called not-behind, noted once
refusal="" # the last thing the plane refused, for the timeout message
converged=0
while :; do
api_call GET /api/v1/hosts
[[ "$api_status" == "200" ]] ||
fail "GET /api/v1/hosts answered HTTP $api_status: ${api_body:-<empty>}
The plane is serving $TAG and the fleet has not been converged."
fleet="$(printf '%s' "$api_body" | jq -r '.[] | [
.id, .name, (.online|tostring), (.agent_version // "-"),
(.pending_upgrade.version // "-"), (.pending_upgrade.age_s // 0 | tostring)
] | @tsv')" ||
fail "GET /api/v1/hosts did not answer with a list of hosts: ${api_body:-<empty>}"
# A plane with no hosts is a plane nobody has joined yet — a fresh one, or
# one whose fleet is still being built. There is nothing to converge and
# nothing wrong. The tenant is worth saying out loud: hosts are listed for
# the credential that asked, so a plane with a fleet and a credential from
# some other tenant looks exactly like this.
if [[ -z "$fleet" ]]; then
echo "no hosts are enrolled here for this credential's tenant — nothing to converge"
converged=0
break
fi
converged=0
waiting=""
while IFS=$'\t' read -r id name online version pending_v pending_age; do
[[ -n "$id" ]] || continue
if [[ "$version" == "$TAG" ]]; then
converged=$((converged + 1))
case "$announced" in
*" $id "*) ;;
*)
case "$offered" in
*" $id "*) echo " $name: converged, now reporting $TAG" ;;
*) echo " $name: already at $TAG" ;;
esac
announced="$announced$id " ;;
esac
continue
fi
# A host that is STILL dark once the grace has passed blocks nothing. It
# cannot be handed an offer — the plane refuses one outright — and the
# smoke cannot place a VM on it either, so waiting for it would hold a
# release for a machine that has no part in proving it. It comes back to
# a fleet a release ahead of it and the console offers it the upgrade
# then.
#
# Inside the grace, though, dark means nothing at all. Stage 7 rolled the
# server moments ago and every agent's sync session went with it, so the
# first listing after it shows a fleet that is entirely dark. Writing
# hosts off on that listing is how a release shipped with ZERO converged
# hosts and still exited green — twice, v0.0.6 and v0.0.7 on prod, both
# recovered by re-running --from 8 once the agents were back. So a host
# that is dark this early is waited on exactly like one with an offer
# outstanding: the loop polls on, and when it reconnects it flows down
# this same list and is offered $TAG like any other host.
if [[ "$online" != "true" ]]; then
if [[ $((SECONDS - started)) -lt $DARK_GRACE_S ]]; then
waiting="$waiting $name (dark; agents reconnect after the stage-7 roll)"
continue
fi
case "$skipped" in
*" $id "*) ;;
*)
reported="$version"
if [[ "$reported" == "-" ]]; then reported="no version"; fi
warn "host $name is disconnected (last reported $reported) — skipped.
It cannot take an offer while it is dark, and the smoke cannot place a VM
on it either. Offer it the upgrade from the console when it returns."
skipped="$skipped$id " ;;
esac
continue
fi
# Connected and saying nothing about its version: an agent too old to
# name itself. The plane will not offer it anything (it has no version
# to order), and it would refuse to create a VM there as well, because
# that guest would get no certified host key. Neither waiting nor the
# smoke can improve on that, so say it now.
if [[ "$version" == "-" ]]; then
fail "host $name ($id) is connected but reports no agent version, so the plane
cannot offer it $TAG — and it would refuse to create a VM there. Upgrade
that agent by hand (docs/upgrade.md, 'Agents, by hand'), then resume:
scripts/ship.sh --target $TARGET --tag $TAG --from 8"
fi
# An offer for some other version is the plane naming a release that is
# not the one being shipped, and no amount of waiting turns one into the
# other. The usual cause is the manifest: the server reads it at boot
# and pins it for 24 hours, which is why stage 6 rolls the site first.
if [[ "$pending_v" != "-" && "$pending_v" != "$TAG" ]]; then
fail "host $name ($id) has been offered $pending_v, not $TAG: this plane's release
manifest does not name the tag being shipped. Check that the site rolled
(stage 6) and that the server restarted after it (stage 7)."
fi
if [[ "$pending_v" == "$TAG" ]]; then
waiting="$waiting $name (offered ${pending_age}s ago)"
continue
fi
# No offer standing: make one. This is also the re-offer path — a server
# restart forgets every outstanding offer, so a host we offered earlier
# can show up here again, and offering twice is exactly what a human
# would do.
api_call POST "/api/v1/hosts/$id/upgrade-agent"
case "$api_status" in
202)
echo " $name: $version → $TAG (offered)"
offered="$offered$id "
waiting="$waiting $name (offered just now)"
;;
503)
# The plane has not fetched its release manifest yet, which is
# ordinary in the first seconds after stage 7 rolled the server.
# Wait for it rather than failing a release on a cold start.
refusal="$name: $api_body"
waiting="$waiting $name (plane has no release manifest yet)"
;;
409)
# internal/server/api/upgrade.go refuses three ways, and two of
# them are races against the listing this round was built from —
# the state changed between the GET and this POST.
#
# "host is offline": the next round sees the host dark and skips
# it.
#
# "not behind the latest release": the host converged in that same
# gap. An operator clicked Upgrade in the console, or an offer from
# an earlier round landed and the agent re-exec'd while this round
# was still walking the list. There is nothing to offer a host that
# is already there, and nothing wrong either — so the offer is
# dropped and the version the host REPORTS decides it, on the next
# round, exactly as it decides it for every other host. If it never
# reports $TAG, the timeout below says so.
#
# That leaves the third refusal, no artifact for this os/arch, for
# which waiting out the timeout would only delay telling a person.
# Nothing here weakens the stale-manifest catch: a plane offering
# some OTHER version is caught by name, above, before any of this.
if [[ "$api_body" == *"host is offline"* ]]; then
refusal="$name: $api_body"
waiting="$waiting $name (went offline mid-offer)"
elif [[ "$api_body" == *"not behind the latest release"* ]]; then
refusal="$name: $api_body"
case "$notbehind" in
*" $id "*) ;;
*)
echo " $name: not behind the release the plane names — reading back the version it reports"
notbehind="$notbehind$id " ;;
esac
waiting="$waiting $name (offer declined as not-behind; awaiting its reported version)"
else
fail "the plane refuses to offer $TAG to host $name ($id): $api_body"
fi
;;
*)
fail "POST /api/v1/hosts/$id/upgrade-agent answered HTTP $api_status: ${api_body:-<empty>}"
;;
esac
done <<<"$fleet"
[[ -n "$waiting" ]] || break
if [[ $((SECONDS - started)) -ge $CONVERGE_TIMEOUT_S ]]; then
fail "the fleet did not converge on $TAG within ${CONVERGE_TIMEOUT_S}s —
still waiting on:$waiting${refusal:+
last refusal — $refusal}
These hosts are connected and not running $TAG. Where the offer was taken
and did not land, the reason is on the host, in the agent's log
(journalctl -u eitri-agent, or ~/Library/Logs/eitri-agent.log on a Mac);
where the plane refused it, the refusal above says why. The plane is
serving $TAG and is not proven. Resume with --from 8."
fi
if [[ $((SECONDS - last_note)) -ge 30 ]]; then
echo " waiting on:$waiting"
last_note=$SECONDS
fi
sleep "$CONVERGE_POLL_S"
done
skipped_n="$(printf '%s' "$skipped" | wc -w | tr -d ' ')"
summary="fleet: $converged host(s) reporting $TAG"
if [[ "$skipped_n" != "0" ]]; then
summary="$summary, $skipped_n disconnected and skipped"
fi
echo "$summary"
if [[ "$converged" == "0" && "$skipped_n" != "0" ]]; then
warn "no connected host is running $TAG, so the smoke has nowhere to place a VM."
fi
elif [[ "$FROM" -le 8 ]]; then
bold "8. Converge the fleet's agents SKIPPED (no credential in ship.env)"
CONVERGE_SKIPPED=1
warn "the fleet was NOT converged: its agents still run the release before $TAG,
while this plane now serves $TAG. Nothing offers an agent an upgrade on
its own, so they will stay there until someone does.
Finish it either way:
- the console's fleet page, host by host, from any browser signed in as
the tenant that owns them; or
- this same run from a machine that holds the credential:
scripts/ship.sh --target $TARGET --tag $TAG --from 8
$SHIP_ENV names neither CI_PAT_FILE nor CI_USER, which is why this stage
had nothing to act as."
fi
# ── 9. Hosted smoke ───────────────────────────────────────────────────────────
# The same binary the branch gate runs, pointed at this plane's public names.
# It needs nothing but those names and a credential: every leg goes through the
# plane's own front door, so a hosted run logs into no host in the fleet. What
# IS new is that the gate leg crosses the internet, so GATE_HOST:GATE_PORT must
# be reachable from here.
#
# Which credentials the run uses is the plane's business, not this script's: it
# forwards whatever ship.env names and nothing else. A plane with a password
# issuer sets CI_USER and gets the credential-chain proof plus a freshly minted
# PAT; a plane fronted by a real identity provider sets an operator PAT instead,
# because there is no password to post at Google. No COVER_OUT either way — the
# hosted binaries are not coverage-instrumented.
if [[ "$FROM" -le 9 && "$SKIP_SMOKE" != "1" ]]; then
bold "9. Hosted smoke against $TARGET"
if [[ -z "${CI_USER:-}" && -z "${CI_PAT_FILE:-}" ]]; then
fail "$SHIP_ENV names no credential for the smoke.
Set CI_USER + CI_PASSWORD_FILE for a plane with a password issuer, or
CI_PAT_FILE for one fronted by an external identity provider."
fi
go build -o "$REPO_ROOT/bin/eitri-smoke" ./cmd/eitri-smoke
# Both origins by default: the console host is the one a proxy fronts and
# carries the long call, and the API host proves the /mcp-only rule routes
# and authenticates with nothing in front of it.
SMOKE_MCP_URL="${SMOKE_MCP_URL:-https://$CONSOLE_HOST https://$API_HOST}"
SMOKE_USER_CA_FILE="${SMOKE_USER_CA_FILE:-$HOME/eitri-deploy/$TARGET/smoke_user_ca}"
# SMOKE_EXPECT_AGENT_VERSION is what makes this a proof OF $TAG rather than a
# proof of whatever the fleet happens to be running: the smoke refuses to
# place its VM on a host whose agent is not the release being shipped. A ship
# has twice reached this stage with an unconverged fleet, and the smoke passed
# both times — every leg it proves, a release-old agent satisfies just as well,
# so the run exited green with the plane at $TAG and the hosts a release
# behind. Stage 8 is what should prevent that; this is what catches it.
smoke_env=(
SERVER_URL="https://$CONSOLE_HOST"
SMOKE_GATE="$GATE_HOST:$GATE_PORT"
SMOKE_MCP_URL="$SMOKE_MCP_URL"
SMOKE_USER_CA_FILE="$SMOKE_USER_CA_FILE"
SMOKE_EXPECT_AGENT_VERSION="$TAG"
)
if [[ -n "${CI_USER:-}" ]]; then
: "${CI_PASSWORD_FILE:?set it alongside CI_USER in $SHIP_ENV}"
smoke_env+=(CI_USER="$CI_USER" CI_PASSWORD_FILE="$CI_PASSWORD_FILE")
fi
if [[ -n "${CI_PAT_FILE:-}" ]]; then
smoke_env+=(CI_PAT_FILE="$CI_PAT_FILE")
fi
if [[ -n "${SMOKE_VM_USER:-}" ]]; then
smoke_env+=(SMOKE_VM_USER="$SMOKE_VM_USER")
fi
# Clear the trio first so an ambient value from this shell can never stand in
# for one the plane did not name.
env -u CI_USER -u CI_PASSWORD_FILE -u CI_PAT_FILE "${smoke_env[@]}" \
"$REPO_ROOT/bin/eitri-smoke" ||
fail "hosted smoke FAILED against $TARGET — the plane is serving $TAG and is not proven."
elif [[ "$SKIP_SMOKE" == "1" ]]; then
bold "9. Hosted smoke SKIPPED (--skip-smoke)"
fi
# ── 10. Report ────────────────────────────────────────────────────────────────
bold "10. $TARGET is serving $TAG"
report_deployments=(eitri-server web)
if [[ "$LOCAL_OIDC" == "1" ]]; then
report_deployments+=(eitri-oidc)
fi
kubectl -n "$NAMESPACE" get deployment "${report_deployments[@]}" \
-o custom-columns='DEPLOYMENT:.metadata.name,IMAGE:.spec.template.spec.containers[0].image,READY:.status.readyReplicas' ||
true
echo "console https://$CONSOLE_HOST"
if [[ "$LOCAL_OIDC" == "1" ]]; then
echo "issuer https://$OIDC_HOST"
fi
echo "mcp https://$CONSOLE_HOST/mcp (proxied) and https://$API_HOST/mcp"
echo "downloads https://$SITE_HOST/dl/$TAG/"
# An if, not a trailing &&: as the script's last command, a false conditional
# would be the whole run's exit status.
if [[ "$CONVERGE_SKIPPED" == "1" ]]; then
echo "fleet: NOT CONVERGED — the agents still run the release before $TAG;"
echo " converge them from the console, or re-run with --from 8 from a"
echo " machine whose ship.env names a credential"
fi
if [[ "$SKIP_SMOKE" == "1" ]]; then
echo "smoke: SKIPPED — this plane is deployed, not proven"
fi