#!/usr/bin/env bash
# scripts/deploy — non-interactive prod deploy: reinstall precis-mcp@<ref>
# into every cluster venv and bounce every daemon.
#
# The deterministic deploy backbone (the twin of scripts/ship). No LLM in the
# loop: it wraps `ansible-playbook redeploy-precis.yml` from the in-repo
# `deploy/` tree (default since the slice-12a cutover). The private overlay
# (real inventory + `.vault-pass`) is gitignored and LOCAL-only; it lives in the
# main checkout's `deploy/inventory` and is resolved from there even when a
# deploy runs from a worktree (e.g. /go), so there are no symlinks and no
# per-worktree secret copies. Chained after a green ship by the /go command.
#
# Steps:
#   1. reachability — `ansible all -m ping`; abort (non-zero) if any host is
#      down (a partial redeploy leaves nodes on mixed code — never do that).
#   2. deploy       — run the redeploy playbook for the ref.
#   3. report       — the play's final debug task prints installed sha +
#                     bounced daemons; a `failed=` on any host exits non-zero.
#
# Usage:  scripts/deploy            # deploy main (default)
#         scripts/deploy some-branch
#         scripts/deploy <old-sha> --force-rollback   # deliberate rollback
#
# gr338201 (2026-09-13 incident): a stale worktree pinned to an ancestor
# commit of what was already live ran scripts/deploy and rolled the whole
# cluster BACKWARD — the freshness guard below only catches drift on the
# *local checkout* (`git rev-parse "$REF"` vs origin), and no-ops the moment
# REF is a literal sha (there is no `origin/<sha>` ref to compare against),
# which is exactly the shape of that incident. A second, independent guard
# (below the freshness guard) resolves the target sha and refuses when it is
# a strict ancestor of either (a) the sha recorded as currently deployed
# (the shared deploy-state marker — see scripts/lib/deploy-state.sh) or (b)
# origin/main freshly fetched. Equal-sha (no-op redeploy) is allowed; a
# missing marker (first-ever deploy) is not a refusal. Deliberate rollback:
# pass --force-rollback (anywhere in argv).
#
# --pinned (anywhere in argv): the target is deliberately behind origin/main.
# /go pins the deploy to the sha its gate actually validated (scripts/ship
# writes it to .ship-sha), because `main` is a BRANCH NAME resolved at deploy
# time — a sibling --quick ship landing in between substitutes an ungated tree
# for the gated one, and the deploy reports success over code nobody ran the
# suite against. Being behind origin/main is therefore the expected steady
# state, not a rollback, so --pinned drops that leg of the guard only; the
# deployed-sha leg stays (it is the one that catches the real gr338201 shape)
# and an absent marker refuses outright. Expect a nonzero deploy-lag footer
# from the next scripts/ship: that count IS the ungated backlog on main.
#
# PRECIS_DEPLOY_EXTRA_VARS — optional extra `-e key=value` pairs passed through
# to ansible-playbook, space-separated (e.g. "precis_embedder_trace=true" for
# the gr172390 pass-1 diagnosis runbook). Playbook-run only, never persisted.
#
# Every run's full output (incl. the profile_tasks timing table) is tee'd to
# the main checkout's .deploy-logs/<timestamp>-<ref>.log (gitignored, newest
# 50 kept) for later analysis. PRECIS_DEPLOY_NO_LOG=1 opts out;
# PRECIS_DEPLOY_LOG_DIR overrides the directory.
#
# PRECIS_DEPLOY_CANARY=<ansible-host-name> — opt-in canary staging (unset ⇒
# byte-identical single-pass behavior, the default). When set: resolve the
# target sha once (a literal sha this checkout contains is taken as-is;
# a ref name goes through git ls-remote, the same resolution the playbook's
# step-0 pin does), run the playbook against `--limit <canary>` ONLY first, poll
# `scripts/prod-psql` for that host's `host_heartbeat` row to freshen (proof
# the deployed worker came up on the new code), then fan out to the rest of
# the fleet with `--limit '!<canary>'` — same explicit sha pins on both
# phases, so the two-run pin race (redeploy-precis.yml §step 0) can't bite. A
# canary that never freshens aborts non-zero BEFORE the fleet phase and
# prints a rollback line; a `scripts/prod-psql` failure during verify is
# treated as red (fail closed).
#   PRECIS_DEPLOY_CANARY_TIMEOUT_S  — verify timeout, seconds (default 300).
#   PRECIS_DEPLOY_CANARY_DB_HOST    — override the host_heartbeat.host value
#     to poll, if it differs from the ansible host name (fqdn vs short, or a
#     custom `precis heartbeat --host`); default is the canary name itself.
set -euo pipefail

# --help / -h : print usage and exit BEFORE any side effect (gr344875). Scans
# the whole argv (not just $1) since --force-rollback can precede or follow
# the ref positional — must stay the first thing checked, ahead of the
# lock acquire, the freshness/rollback guards, and every ansible/ssh call.
usage() {
    cat <<'USAGE'
scripts/deploy — non-interactive prod deploy: reinstall precis-mcp@<ref>
into every cluster venv and bounce every daemon.

Usage:
  scripts/deploy                              # deploy main (default)
  scripts/deploy <ref>                        # deploy a branch/tag/sha
  scripts/deploy <old-sha> --force-rollback   # deliberate rollback
  scripts/deploy "$(cat .ship-sha)" --pinned  # /go: deploy the GATED sha
  scripts/deploy <ref> --ignore-pin           # deliberately go off-pin
  scripts/deploy --help | -h                  # this message; no reachability
                                               # check, no ansible, no ssh

<ref> is a positional git ref (branch/tag/sha); default 'main'.
--force-rollback bypasses the ancestor-sha rollback guard (gr338201) — use
only for a deliberate rollback.
--pinned says "this target is deliberately behind origin/main": it drops ONLY
the origin/main leg of the rollback guard, keeps the currently-deployed-sha
leg, and refuses outright when no deploy-state marker exists (fail closed —
with no marker that leg has nothing to compare against).

A .ship-sha in this worktree is an UNCONSUMED gated pin: any target that is
not that sha is refused until the pin is deployed (which removes it) or the
run passes --ignore-pin. --force-rollback also bypasses.

Notable env vars (see the header comment in this script for full docs):
  PRECIS_CLUSTER_DIR, PRECIS_DEPLOY_EXTRA_VARS, PRECIS_DEPLOY_ALLOW_STALE,
  PRECIS_DEPLOY_NO_LOG, PRECIS_DEPLOY_LOG_DIR, PRECIS_DEPLOY_FROM_TREE,
  PRECIS_OVERLAY_DIR, PRECIS_DEPLOY_SKIP_CATPATH_WHEEL, PRECIS_CATPATH_DIR,
  PRECIS_DEPLOY_LOCK_WAIT, PRECIS_DEPLOY_CANARY, PRECIS_DEPLOY_CANARY_TIMEOUT_S,
  PRECIS_DEPLOY_CANARY_DB_HOST.
USAGE
}
for _a in "$@"; do
    case "$_a" in
        --help|-h) usage; exit 0 ;;
    esac
done

FORCE_ROLLBACK=0
PINNED=0
IGNORE_PIN=0
# A scalar, not ${#_POSITIONAL[@]}: bash 3.2 (the macOS system bash) errors on
# the length of an empty array under `set -u`, which is why the REF line below
# uses the ${arr[0]:-default} idiom too.
_HAVE_TARGET=0
_POSITIONAL=()
for _a in "$@"; do
    case "$_a" in
        --force-rollback) FORCE_ROLLBACK=1 ;;
        --pinned) PINNED=1 ;;
        --ignore-pin) IGNORE_PIN=1 ;;
        *) _POSITIONAL+=("$_a"); _HAVE_TARGET=1 ;;
    esac
done
REF="${_POSITIONAL[0]:-main}"
EXTRA_ARGS=()
for _v in ${PRECIS_DEPLOY_EXTRA_VARS:-}; do
    EXTRA_ARGS+=(-e "$_v")
done
CLUSTER_DIR="${PRECIS_CLUSTER_DIR:-${HOME}/work/cluster}"
# Repo root captured BEFORE any `cd` below (install-from-tree changes CLUSTER_DIR
# to a subdir, and legacy mode's CLUSTER_DIR can be an entirely different
# checkout) — the deploy-state marker always belongs at THIS repo's root, the
# one scripts/ship reads it from, regardless of where ansible itself runs.
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# A short sha is expanded to the full 40-char commit here: the playbook's pin
# step treats anything that is not ^[0-9a-f]{40}$ as a ref NAME and resolves
# it with `git ls-remote`, which never matches a bare abbreviated sha, so
# `scripts/deploy 08f79a3a` died in preflight after the wheel build. Ref
# names (main, a tag) are left alone; the playbook resolves those itself.
# Resolved against REPO_ROOT, not the cwd, so a deploy launched from another
# directory does not silently skip the expansion.
if [[ "$REF" =~ ^[0-9a-f]{7,39}$ ]]; then
    _full="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    [[ -n "$_full" ]] && REF="$_full"
fi

say() { printf '\n\033[1m▶ %s\033[0m\n' "$*"; }
die() { printf '\n\033[31m✖ %s\033[0m\n' "$*" >&2; exit 1; }

# ── the gated pin is a ONE-SHOT TOKEN: ship mints it, deploy consumes it ───
# `--pinned` only helps if the caller actually types the pinned sha, and /go
# is executed by an agent following prose in .claude/commands/go.md. One run
# that types a bare `scripts/deploy` instead resolves `main` at deploy time
# and ships whatever ungated sibling qland landed since the gate — silently,
# reporting success. So the check does not depend on anyone remembering:
# while an UNCONSUMED .ship-sha sits in this worktree, any target that is not
# that sha is refused outright. Loud-refuse, never silent-substitute: an
# operator who asked for `main` and got the pin instead would be the same
# class of bug pointed the other way.
#
# Consumed (removed) once a deploy of that exact sha records success — both
# at step 3 and at the gr332009 convergence graft, since either can be the
# write that means "the fleet is on this sha now". Without the removal a
# spent pin would block every later ordinary `scripts/deploy` from the tree
# that shipped it.
#
# Bypasses: --ignore-pin (deliberate off-pin deploy, e.g. a cluster-admin
# redeploy from a tree with a pin still pending) and --force-rollback (an
# incident hatch must not be gated behind a second flag).
#
# `--pinned` with no positional is refused outright. The documented recovery
# command is `scripts/deploy "$(cat .ship-sha)" --pinned`, and a `cat` of a
# missing/emptied pin expands to nothing — which `REF="${_POSITIONAL[0]:-main}"`
# would read as "no target given" and silently substitute `main` for, WITH
# --pinned still set. That is the exact substitution this whole mechanism
# exists to prevent, arriving through its own retry path.
if [[ "$PINNED" == 1 && "$_HAVE_TARGET" != 1 ]]; then
    die "--pinned with no target. It means 'deploy exactly this sha'; there is no default for that. If this came from scripts/deploy \"\$(cat .ship-sha)\" --pinned, the pin file is gone — the deploy that consumed it already put that sha on the fleet, so check the deploy-state marker before re-running anything."
fi

SHIP_PIN_FILE="${REPO_ROOT}/.ship-sha"
SHIP_PIN=""
if [[ -f "$SHIP_PIN_FILE" ]]; then
    SHIP_PIN="$(tr -d '[:space:]' < "$SHIP_PIN_FILE" 2>/dev/null || true)"
fi
# An unusable pin must not become an unrecoverable refusal. If the file holds
# something this repo cannot resolve to a commit (a partial write from a
# killed ship, a disk-full truncation), then the target can never equal it and
# the generic refusal below would print a remedy that replays the same garbage
# and dies identically — an operator following the instructions loops forever.
# Still refuse (a pin we cannot read is not evidence that deploying is safe),
# but say the thing that actually works.
if [[ -n "$SHIP_PIN" && "$IGNORE_PIN" != 1 && "$FORCE_ROLLBACK" != 1 ]] \
    && ! git -C "$REPO_ROOT" rev-parse -q --verify "${SHIP_PIN}^{commit}" >/dev/null 2>&1; then
    die "this worktree's .ship-sha is unusable — it does not resolve to a commit here (contents: '${SHIP_PIN:0:40}'). It cannot be deployed and it is blocking other targets. Delete it (rm .ship-sha) and re-run the gate, or pass --ignore-pin if you know what you are deploying."
fi
if [[ -n "$SHIP_PIN" && "$IGNORE_PIN" != 1 && "$FORCE_ROLLBACK" != 1 ]]; then
    _pin_target="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    if [[ "$_pin_target" != "$SHIP_PIN" ]]; then
        die "this worktree holds an unconsumed gated pin (.ship-sha = ${SHIP_PIN:0:8}) but the target is '${REF}'${_pin_target:+ (${_pin_target:0:8})}. The gate validated ${SHIP_PIN:0:8}; deploying anything else ships code no gate ran against. Deploy the pin:
    scripts/deploy \"\$(cat .ship-sha)\" --pinned
or, if going off-pin is deliberate, re-run with --ignore-pin."
    fi
fi

# Remove the pin once this exact sha is recorded as deployed. Called from
# both success-marker writers; a no-op when there is no pin, when the pin is
# for a different sha, or when the target could not be resolved.
_consume_ship_pin() {
    local sha="$1"
    [[ -n "$SHIP_PIN" && -n "$sha" && "$sha" == "$SHIP_PIN" ]] || return 0
    rm -f "$SHIP_PIN_FILE" 2>/dev/null || true
}

# ── gr332009 rollout-convergence graft (on top of the attempt-stamp/no-legacy
# design above) — write the SUCCESS marker (and clear the attempt stamp) as
# soon as the code ROLLOUT itself converges, even if a LATER, unrelated play
# in the same `ansible-playbook` invocation dies ───────────────────────────
# redeploy-precis.yml runs its residual, non-rollout plays (agent-sandbox
# image rebuilds etc.) in the SAME invocation as the core rollout (venv
# installs, daemon bounce, convergence assert) — so a persistent failure on a
# host those residual plays alone touch (the known balthazar sandbox
# podman-pull residual, commit 5fa7ff29) makes the whole command exit non-zero even
# though every host that matters to "what is the fleet actually running"
# landed cleanly. Without this graft, step 3 below never runs on that red
# exit, the attempt stamp never clears, and every ship footer says "deploy
# state uncertain" forever — alarm fatigue in the other direction. A genuine
# rollout-host failure must still leave the stamp in place (fail-safe).
#
# _write_deploy_state_marker duplicates step 3's success write (marker +
# clear-stamp) on purpose rather than factoring it out — step 3 itself stays
# byte-identical to the no-graft design so the ordinary all-green path is
# untouched. _rollout_converged reads the ALREADY-CAPTURED ansible-playbook
# output for the PLAY RECAP and requires every rollout host (ROLLOUT_HOSTS,
# set below from the reachability ping's own output) to show
# `failed=0 unreachable=0` — a host missing from the recap entirely, or
# showing any failure, means NOT converged. Scoped to the single-pass path
# only (PRECIS_DEPLOY_CANARY unset, the default) — the canary path's marker
# already means "the WHOLE fleet is on this sha", and writing it after only
# the canary phase converges would be actively wrong, so that path's existing
# all-or-nothing `|| die` behavior is left untouched.
_write_deploy_state_marker() {
    local sha
    sha="$(git -C "$REPO_ROOT" rev-parse "$REF" 2>/dev/null || true)"
    [[ -n "$sha" ]] || return 0
    . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
    printf '%s %s %s\n' "$sha" "$(date +%s)" "success" > "$(deploy_state_path "$REPO_ROOT")"
    rm -f "$(deploy_attempt_path "$REPO_ROOT")" 2>/dev/null
    _consume_ship_pin "$sha"
}

_rollout_converged() {
    local play_out="$1" host line failed unreachable recap
    [[ -n "${ROLLOUT_HOSTS:-}" ]] || return 1
    recap="$(awk '/^PLAY RECAP/{f=1; next} f' "$play_out")"
    [[ -n "$recap" ]] || return 1
    for host in $ROLLOUT_HOSTS; do
        line="$(awk -v h="$host" '$1==h' <<< "$recap")"
        [[ -n "$line" ]] || return 1
        failed="$(grep -oE 'failed=[0-9]+' <<< "$line" | cut -d= -f2)"
        unreachable="$(grep -oE 'unreachable=[0-9]+' <<< "$line" | cut -d= -f2)"
        [[ "${failed:-1}" == 0 && "${unreachable:-1}" == 0 ]] || return 1
    done
    return 0
}

# Run one redeploy-precis.yml invocation. Its combined output is ALSO tee'd
# to $play_out (a scratch capture, separate from the whole-script tee already
# flowing to console + the deploy log) purely so _rollout_converged can parse
# the PLAY RECAP — nothing about what reaches the console/log changes. The
# `if !` form (not `|| die`) is deliberate: it lets this function inspect the
# recap and possibly write the marker BEFORE the caller's own `|| die` fires.
_run_rollout_playbook() {
    local play_out rc=0
    play_out="$(mktemp)"
    if ! ansible-playbook "$@" 2>&1 | tee "$play_out"; then
        rc=1
    fi
    if [[ "$rc" -ne 0 ]] && _rollout_converged "$play_out"; then
        say "redeploy-precis.yml exited non-zero, but every rollout host (${ROLLOUT_HOSTS}) converged clean in its own PLAY RECAP line — recording the deploy-state marker before reporting the failure (gr332009)"
        _write_deploy_state_marker
    fi
    rm -f "$play_out"
    return "$rc"
}

# ── mutual exclusion: one deploy at a time, machine-wide (gr203786) ──────
# Two concurrent deploys (e.g. sibling /go sessions in different worktrees)
# interleave their installs: each run pins its own sha, so the later run's
# installs move the venvs under the earlier run's convergence assert →
# spurious "DEPLOY DID NOT CONVERGE" on a healthy cluster (observed twice
# 2026-08-11). The playbook's per-run pinning is correct; the missing piece
# is that runs must not overlap. Lock dir lives in the git COMMON dir so
# every worktree of this repo shares one lock. mkdir is the atomic acquire
# (macOS ships no flock); a dead holder's lock is stolen.
_LOCK_DIR="$(git -C "$REPO_ROOT" rev-parse --path-format=absolute --git-common-dir)/precis-deploy.lock"
_acquire_deploy_lock() {
    local wait="${PRECIS_DEPLOY_LOCK_WAIT:-1800}" waited=0 holder
    while ! mkdir "$_LOCK_DIR" 2>/dev/null; do
        holder="$(cat "${_LOCK_DIR}/pid" 2>/dev/null || true)"
        if [[ -n "$holder" ]] && ! kill -0 "$holder" 2>/dev/null; then
            say "stealing deploy lock from dead pid ${holder}"
            rm -rf "$_LOCK_DIR"
            continue
        fi
        (( waited < wait )) || die "another deploy (pid ${holder:-unknown}) still holds the lock after ${wait}s — investigate it, or rm -rf ${_LOCK_DIR}"
        (( waited % 60 == 0 )) && say "another deploy (pid ${holder:-starting}) is in flight — waiting (${waited}s/${wait}s)"
        sleep 10; waited=$((waited + 10))
    done
    echo $$ > "${_LOCK_DIR}/pid"
    trap 'rm -rf "$_LOCK_DIR"' EXIT
    trap 'exit 130' INT TERM
}
_acquire_deploy_lock

# ── deploy log: persist every run's full output — including the per-task
# profile_tasks timing table (deploy/ansible.cfg) — so deploys can be compared
# later instead of dying with the terminal scrollback. One timestamped file per
# run, in the MAIN checkout's .deploy-logs/ (gitignored) so every worktree's
# deploys land in one place. Deliberately NOT under deploy/: the logs name real
# hosts and tests/test_deploy_tree_no_secrets.py walks that whole tree. Keep
# the newest 50. PRECIS_DEPLOY_NO_LOG=1 opts out (e.g. for probe runs).
if [[ -z "${PRECIS_DEPLOY_NO_LOG:-}" ]]; then
    _main_root="$(dirname "$(git -C "$REPO_ROOT" rev-parse --path-format=absolute --git-common-dir)")"
    DEPLOY_LOG_DIR="${PRECIS_DEPLOY_LOG_DIR:-${_main_root}/.deploy-logs}"
    mkdir -p "$DEPLOY_LOG_DIR"
    DEPLOY_LOG="${DEPLOY_LOG_DIR}/$(date -u +%Y%m%d-%H%M%S)-${REF//\//_}.log"
    exec > >(tee -a "$DEPLOY_LOG") 2>&1
    say "logging this deploy to ${DEPLOY_LOG}"
    ls -1t "$DEPLOY_LOG_DIR"/*.log 2>/dev/null | tail -n +51 | xargs rm -f 2>/dev/null || true
fi

# ── freshness guard: ansible renders roles/templates from THIS checkout, while
# the venvs install precis-mcp@<ref> from git — a local tree behind origin/<ref>
# deploys NEW code with STALE units (bit us 2026-08-03: workers restarted
# without the §F-b env exports). Refuse the skew unless explicitly overridden.
if [[ -z "${PRECIS_DEPLOY_ALLOW_STALE:-}" ]]; then
    git -C "$REPO_ROOT" fetch -q origin "$REF" 2>/dev/null || true
    _remote="$(git -C "$REPO_ROOT" rev-parse -q --verify "origin/${REF}" 2>/dev/null || true)"
    if [[ -n "$_remote" ]] && ! git -C "$REPO_ROOT" merge-base --is-ancestor "$_remote" HEAD; then
        die "local tree does not contain origin/${REF} ($(git -C "$REPO_ROOT" rev-parse --short HEAD) vs ${_remote:0:8}) — templates would render stale. Sync this checkout (git merge origin/${REF}) or set PRECIS_DEPLOY_ALLOW_STALE=1."
    fi
    # REF given as a literal sha (or a tag): `origin/<sha>` does not resolve, so
    # the branch comparison above no-ops ENTIRELY and a sha deploy gets no skew
    # check at all — the hole the gr338201 header calls out, and now the normal
    # path since /go pins the gated sha. Compare the target commit against this
    # checkout directly instead: ansible renders deploy/ templates from HEAD
    # while the venvs install the target, so HEAD must at minimum CONTAIN the
    # target. Any divergence beyond that is said out loud rather than refused —
    # deploying an older-than-HEAD sha is exactly what --pinned is for.
    if [[ -z "$_remote" ]]; then
        _fr_target="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
        _fr_head="$(git -C "$REPO_ROOT" rev-parse -q --verify HEAD 2>/dev/null || true)"
        if [[ -n "$_fr_target" ]]; then
            git -C "$REPO_ROOT" merge-base --is-ancestor "$_fr_target" HEAD 2>/dev/null \
                || die "local tree does not contain ${REF} (${_fr_target:0:8}; HEAD is ${_fr_head:0:8}) — templates would render from a tree that never saw this commit. Sync this checkout, or set PRECIS_DEPLOY_ALLOW_STALE=1."
            [[ "$_fr_target" == "$_fr_head" ]] \
                || echo "NOTE: installing ${_fr_target:0:8} but rendering deploy/ templates from HEAD ${_fr_head:0:8} — fine when the delta touches no template, otherwise check out the target sha first."
        fi
    fi
    _dirty="$(git -C "$REPO_ROOT" status --porcelain -- deploy/ 2>/dev/null || true)"
    [[ -z "$_dirty" ]] || die "deploy/ tree has uncommitted changes — templates would render unshipped state. Commit/ship first, or set PRECIS_DEPLOY_ALLOW_STALE=1."
fi

# ── rollback guard (gr338201): refuse an ancestor-sha deploy ─────────────
# Independent of the freshness guard above (which only compares the LOCAL
# checkout to origin/$REF and no-ops for a literal-sha REF). This compares
# the resolved TARGET sha itself against the sha already recorded as
# deployed (the shared deploy-state marker) and against a freshly-fetched
# origin/main — the two things that jointly define "already ahead of this".
# Not gated by PRECIS_DEPLOY_ALLOW_STALE (a different problem: template
# skew, not direction). Only --force-rollback bypasses it.
if [[ "$FORCE_ROLLBACK" != 1 ]]; then
    _rb_target="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    if [[ -n "$_rb_target" ]]; then
        . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
        _rb_marker="$(deploy_state_read_path "$REPO_ROOT")"
        _rb_deployed=""
        if [[ -n "$_rb_marker" && -f "$_rb_marker" ]]; then
            _rb_deployed="$(awk '{print $1}' "$_rb_marker" 2>/dev/null || true)"
            git -C "$REPO_ROOT" cat-file -e "${_rb_deployed}^{commit}" 2>/dev/null || _rb_deployed=""
        fi
        git -C "$REPO_ROOT" fetch -q origin main 2>/dev/null || true
        _rb_origin_main="$(git -C "$REPO_ROOT" rev-parse -q --verify origin/main 2>/dev/null || true)"

        # --pinned: the target is deliberately behind origin/main (it is the sha
        # a full gate validated; siblings have qlanded ungated commits on top
        # since). Drop ONLY the origin/main leg — the deployed-sha leg below
        # still catches the actual gr338201 incident shape, which was a stale
        # worktree deploying an ancestor of WHAT WAS ALREADY LIVE. With no
        # marker that leg has nothing to compare against, so fail closed rather
        # than let --pinned degrade into a guard-free deploy.
        if [[ "$PINNED" == 1 && -z "$_rb_deployed" ]]; then
            die "--pinned needs the deploy-state marker to check direction, and there is none (no successful deploy on record). Deploy the branch normally first, or pass --force-rollback if you know this target is correct."
        fi

        _rb_against=""
        if [[ -n "$_rb_deployed" && "$_rb_target" != "$_rb_deployed" ]] \
            && git -C "$REPO_ROOT" merge-base --is-ancestor "$_rb_target" "$_rb_deployed" 2>/dev/null; then
            _rb_against="the currently-deployed sha ${_rb_deployed:0:8} (deploy-state marker)"
        elif [[ "$PINNED" != 1 ]] && [[ -n "$_rb_origin_main" && "$_rb_target" != "$_rb_origin_main" ]] \
            && git -C "$REPO_ROOT" merge-base --is-ancestor "$_rb_target" "$_rb_origin_main" 2>/dev/null; then
            _rb_against="origin/main (${_rb_origin_main:0:8})"
        fi

        # Say how much ungated trunk this pin is leaving behind — the whole
        # point of pinning is that main moved on, and the operator should see
        # by how much rather than infer it from a silent success.
        if [[ "$PINNED" == 1 && -z "$_rb_against" && -n "$_rb_origin_main" && "$_rb_target" != "$_rb_origin_main" ]]; then
            _rb_behind="$(git -C "$REPO_ROOT" rev-list --count "${_rb_target}..${_rb_origin_main}" 2>/dev/null || true)"
            # An `if`, not a trailing `[[ … ]] && echo`: nothing behind main is
            # an ordinary pin (a /go that won the CAS race cleanly, or a target
            # ahead of origin/main), and as the last command of this block the
            # AND-list form returns non-zero on that path. Harmless where this
            # sits today — `set -e` exempts it at top level — but it exits the
            # moment anyone wraps this code in a function.
            if [[ -n "$_rb_behind" && "$_rb_behind" != 0 ]]; then
                echo "NOTE: --pinned — deploying the gated ${_rb_target:0:8}, which is ${_rb_behind} commit(s) behind origin/main (${_rb_origin_main:0:8}). Those commits are NOT going out; the next full gate certifies them."
            fi
        fi

        if [[ -n "$_rb_against" ]]; then
            {
                . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
                printf '%s %s %s\n' "$_rb_target" "$(date +%s)" "refused" \
                    > "$(deploy_attempt_path "$REPO_ROOT")"
            } 2>/dev/null || true
            die "REFUSING TO DEPLOY: target '${REF}' (${_rb_target:0:8}) is an ANCESTOR of ${_rb_against} — this would roll the cluster BACKWARD (gr338201: this is exactly the 2026-09-13 stale-worktree incident shape). deployed=${_rb_deployed:-unknown} target=${_rb_target} origin/main=${_rb_origin_main:-unknown}. If this rollback is deliberate, re-run with --force-rollback."
        fi
    fi
fi

# ── slice 12a: install-from-tree is the DEFAULT (cutover complete 2026-07-19).
# Deploy runs ansible from the portable in-repo `deploy/` tree. Overlay (private
# inventory + vault) resolution, symlink-free and checkout-independent:
#   • if THIS checkout has deploy/inventory, use it (ansible.cfg's relative paths);
#   • else fall back to the CANONICAL overlay in the main checkout (found via
#     git --git-common-dir), or $PRECIS_OVERLAY_DIR — so a deploy works from any
#     worktree (/go ships+deploys from one) with the overlay stored in ONE place.
# Legacy escape (pre-cutover private checkout): `PRECIS_DEPLOY_FROM_TREE=`.
INV_ARGS=()
PRECIS_DEPLOY_FROM_TREE="${PRECIS_DEPLOY_FROM_TREE-1}"
if [[ -n "${PRECIS_DEPLOY_FROM_TREE}" ]]; then
    _script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    CLUSTER_DIR="$(cd "${_script_dir}/.." && pwd)/deploy"
    if [[ -d "${CLUSTER_DIR}/inventory" ]]; then
        say "install-from-tree: deploying from ${CLUSTER_DIR} (local overlay)"
    else
        _main="$(dirname "$(git -C "${CLUSTER_DIR}" rev-parse --path-format=absolute --git-common-dir 2>/dev/null)")"
        _overlay="${PRECIS_OVERLAY_DIR:-${_main}/deploy/inventory}"
        _vpass="$(dirname "${_overlay}")/.vault-pass"
        [[ -d "${_overlay}" && -f "${_vpass}" ]] || die "install-from-tree: no overlay — this checkout's ${CLUSTER_DIR}/inventory is empty and the canonical overlay (${_overlay}) is missing. Put your private inventory in the main checkout's deploy/inventory, or set PRECIS_OVERLAY_DIR (see deploy/README.md)."
        INV_ARGS=(-i "${_overlay}/hosts.yml" --vault-password-file "${_vpass}")
        say "install-from-tree: deploying from ${CLUSTER_DIR} (canonical overlay: ${_overlay})"
    fi
fi

[[ -d "$CLUSTER_DIR" ]] || die "cluster dir not found at ${CLUSTER_DIR} (set PRECIS_CLUSTER_DIR)."
[[ -f "${CLUSTER_DIR}/redeploy-precis.yml" ]] || die "redeploy-precis.yml not in ${CLUSTER_DIR}."

cd "$CLUSTER_DIR"

# ── 1. reachability ─────────────────────────────────────────────────────
# Ping only the groups redeploy-precis.yml actually targets, NOT `all` — the
# inventory can carry bootstrap-only hosts with no `deploy` SSH user that would
# abort a deploy they play no part in (gripe 196641). `serving` (castor/pollux)
# IS a redeploy target now — 20c provisions the heartbeat-only unit there — so
# it must be pinged too: a twin down mid-deploy should abort, not half-
# provision. Keep this in sync with the plays' `hosts:` in redeploy-precis.yml.
PING_TARGETS="gateway:scheduler:data:inference:serving"
say "checking cluster reachability (ansible ping — ${PING_TARGETS})"
_PING_OUT="/tmp/precis-deploy-ping.$$"
ansible "$PING_TARGETS" -m ping ${INV_ARGS[@]+"${INV_ARGS[@]}"} >"$_PING_OUT" 2>&1 || {
    cat "$_PING_OUT" >&2
    rm -f "$_PING_OUT"
    die "a host is unreachable — aborting (a partial redeploy mixes code versions)."
}
# gr332009: the rollout-host set _run_rollout_playbook checks convergence
# against below — every host that just answered the ping, read straight back
# out of ansible's own SUCCESS lines rather than re-querying the inventory.
ROLLOUT_HOSTS="$(grep -oE '^[^[:space:]]+ \| SUCCESS' "$_PING_OUT" | awk '{print $1}')"
rm -f "$_PING_OUT"

# ── 1a. autocatpath wheel (gr263082) ────────────────────────────────────
# Same reason the ping above aborts: a partial redeploy mixes code versions.
# autocatpath is release-gated off PyPI past 0.13.0, so the only channel to a
# host is `--find-links /opt/precis/wheels`, seeded from a controller-side
# wheel passed as `-e autocatpath_wheel=<path>`. Nothing ever set that, so a
# floor bump in THIS repo's pyproject (the >=0.18.0 move) silently outran the
# newest built wheel (0.17.0) and the `precis-mcp[paper,catalyst]` install
# died on the autocatpath_plugin hosts alone, mid-run, after four other hosts
# had already moved. Resolve the wheel here, before touching anything.
#
# Building is in scope on purpose. The floor lives in this repo and the
# artifact lives in the catpath checkout, so they drift by construction; a fix
# that still needs a hand-run `uv build` after every bump just relocates the
# same failure to the next one. Version equality with the floor is what keeps
# this honest — a checkout too old to satisfy the floor is reported, never
# built and hoped over.
#
#   PRECIS_CATPATH_DIR            — catpath checkout. Unset, the sibling
#                                   layout is probed: <repo>/../catpath,
#                                   ~/work/projects/code/catpath, ~/catpath
#   PRECIS_DEPLOY_SKIP_CATPATH_WHEEL=1 — bypass entirely (hosts already seeded
#                                   by hand, or a fleet with no plugin hosts)
if [[ -z "${PRECIS_DEPLOY_SKIP_CATPATH_WHEEL:-}" \
    && "${PRECIS_DEPLOY_EXTRA_VARS:-}" != *autocatpath_wheel* ]]; then
    source "${REPO_ROOT}/scripts/lib/autocatpath-wheel.sh"
    # Default search, first hit wins. `~/catpath` was the only candidate and it
    # does not exist on this controller — the real checkout lives beside the
    # other repos — so every deploy died in preflight with "no wheel that new
    # exists ... and there is no catpath checkout there", naming a path the
    # operator had never used. Probing the conventional sibling locations makes
    # the common layout work with no env var; PRECIS_CATPATH_DIR still wins.
    if [[ -n "${PRECIS_CATPATH_DIR:-}" ]]; then
        _CATPATH_DIR="$PRECIS_CATPATH_DIR"
    else
        _CATPATH_DIR="${HOME}/catpath"
        for _cand in \
            "${REPO_ROOT}/../catpath" \
            "${HOME}/work/projects/code/catpath" \
            "${HOME}/catpath"; do
            if [[ -f "${_cand}/pyproject.toml" ]]; then
                _CATPATH_DIR="$(cd "$_cand" && pwd)"
                break
            fi
        done
    fi
    _FLOOR="$(autocatpath_floor "${REPO_ROOT}/pyproject.toml" || true)"

    if [[ -z "$_FLOOR" ]]; then
        : # no autocatpath floor declared — nothing to seed
    else
        _WHEEL="$(newest_autocatpath_wheel "${_CATPATH_DIR}/dist" || true)"
        if [[ -z "$_WHEEL" ]] || ! version_ge "$(autocatpath_wheel_version "$_WHEEL")" "$_FLOOR"; then
            # No local wheel clears the floor — try to build one, but only
            # from a checkout that actually declares a satisfying version.
            _PROJ_V="$(autocatpath_project_version "${_CATPATH_DIR}/pyproject.toml" || true)"
            [[ -n "$_PROJ_V" ]] || die "autocatpath floor is >=${_FLOOR} but no wheel that new exists in ${_CATPATH_DIR}/dist and there is no catpath checkout there to build one from. Set PRECIS_CATPATH_DIR, or PRECIS_DEPLOY_SKIP_CATPATH_WHEEL=1 if the hosts are already seeded. (gr263082)"
            # The checkout's own version is NOT the gate: uv.lock may pin a
            # newer catpath commit than the checkout has checked out (the
            # throwaway-worktree path below builds the pin, not HEAD), so the
            # floor is checked once, after the build source is resolved.

            # Only build from a checkout that matches what precis actually
            # depends on. catpath reuses one version number across many
            # commits — 0.18.0 already spans nine, including a minimum-image
            # convention fix — so the wheel FILENAME cannot tell a stale build
            # from a current one. Two wheels named autocatpath-0.18.0 can carry
            # different code, and the wheelhouse keeps whichever landed last.
            # A deploy is the wrong place to be relaxed about that: refuse
            # rather than ship a plausible-looking wheel nobody can identify.
            git -C "$_CATPATH_DIR" fetch -q origin 2>/dev/null || true
            [[ -z "$(git -C "$_CATPATH_DIR" status --porcelain 2>/dev/null)" ]] \
                || die "catpath at ${_CATPATH_DIR} has uncommitted changes — a wheel built from it would be named autocatpath-${_PROJ_V} but contain code no one else has. Commit/stash there, or build the wheel on the release machine and pass -e autocatpath_wheel=<path>. (gr263082)"

            # uv.lock already names the exact commit precis resolved against,
            # so ask the precise question — "would this tree build the code we
            # locked?" — instead of the proxy "is this tree level with its
            # upstream?". The proxy is not the same question and gets it wrong
            # in the normal case: catpath moves ahead between `uv lock -P`
            # runs, so a checkout at the pinned commit is CORRECT while being
            # behind upstream, and the proxy would refuse it.
            #
            # Compare only what ships. Commits touching docs/ or tests/ cannot
            # change the wheel, and refusing them would be the same false
            # alarm in a new place; src/ plus pyproject.toml (metadata, deps,
            # packaging config) is what a build actually reads.
            _BUILD_DIR="$_CATPATH_DIR"
            _PIN_WT=""
            _LOCKED="$(autocatpath_locked_sha "${REPO_ROOT}/uv.lock" || true)"
            if [[ -n "$_LOCKED" ]]; then
                git -C "$_CATPATH_DIR" cat-file -e "${_LOCKED}^{commit}" 2>/dev/null \
                    || die "uv.lock pins autocatpath at ${_LOCKED}, but that commit is not in ${_CATPATH_DIR} even after a fetch — the checkout is on a different remote, or the pin references an unpushed commit. Fix the checkout, or build on the release machine and pass -e autocatpath_wheel=<path>. (gr263082)"
                if ! git -C "$_CATPATH_DIR" diff --quiet "$_LOCKED" HEAD -- src pyproject.toml 2>/dev/null; then
                    # The checkout is clean but its packaged code is not the
                    # pinned commit's: catpath moved ahead of the pin, or the
                    # operator is mid-work on another branch. Refusing here
                    # was the first answer, and it turned every deploy after
                    # a catpath release into a "check the pin out, deploy,
                    # check main back out" dance done by hand. The pin IS
                    # what precis depends on, so build exactly that from a
                    # throwaway detached worktree and remove it afterwards.
                    # The dirty-tree check above still applies — a worktree
                    # cannot rescue code nobody has committed.
                    _PIN_WT="$(mktemp -d "${TMPDIR:-/tmp}/autocatpath-pin.XXXXXX")"
                    git -C "$_CATPATH_DIR" worktree add --detach -q "$_PIN_WT" "$_LOCKED" 2>/dev/null \
                        || die "could not check the pinned autocatpath commit ${_LOCKED} out into a throwaway worktree at ${_PIN_WT}. (gr263082)"
                    _BUILD_DIR="$_PIN_WT"
                    _PIN_V="$(autocatpath_project_version "${_PIN_WT}/pyproject.toml" || true)"
                    if [[ -z "$_PIN_V" ]] || ! version_ge "$_PIN_V" "$_FLOOR"; then
                        git -C "$_CATPATH_DIR" worktree remove --force "$_PIN_WT" 2>/dev/null || rm -rf "$_PIN_WT"
                        die "uv.lock pins autocatpath at ${_LOCKED}, which declares version ${_PIN_V:-?} — below the floor >=${_FLOOR} this repo's pyproject demands. Run 'uv lock -P autocatpath' here so the lock and the floor agree. (gr263082)"
                    fi
                    _PROJ_V="$_PIN_V"
                    say "catpath at ${_CATPATH_DIR} is not at the pinned commit — building ${_PROJ_V} from a throwaway worktree at ${_LOCKED:0:12}"
                fi
            else
                # No git pin to check against (autocatpath resolved from an
                # index, or a stale lock). Fall back to the weaker signal
                # rather than dropping provenance checking entirely.
                _BEHIND="$(git -C "$_CATPATH_DIR" rev-list --count 'HEAD..@{upstream}' 2>/dev/null || echo 0)"
                [[ "$_BEHIND" == "0" ]] \
                    || die "catpath at ${_CATPATH_DIR} is ${_BEHIND} commit(s) behind its upstream, all still labelled ${_PROJ_V} — building here would ship a wheel that silently differs from the released one. Pull catpath first, or build on the release machine and pass -e autocatpath_wheel=<path>. (gr263082)"
            fi

            version_ge "$_PROJ_V" "$_FLOOR" || die "autocatpath floor is >=${_FLOOR} but the catpath source to build from (${_BUILD_DIR}) declares only ${_PROJ_V} — pull/bump catpath (or 'uv lock -P autocatpath' so the pin clears the floor) and release the wheel first. (gr263082)"

            say "building autocatpath ${_PROJ_V} wheel (floor >=${_FLOOR}, nothing that new in ${_CATPATH_DIR}/dist)"
            # The wheel always lands in the checkout's own dist/ — a worktree
            # build's dist/ would vanish with the worktree.
            if (cd "$_BUILD_DIR" && uv build --wheel --out-dir "${_CATPATH_DIR}/dist"); then
                _BUILD_RC=0
            else
                _BUILD_RC=$?
            fi
            if [[ -n "$_PIN_WT" ]]; then
                git -C "$_CATPATH_DIR" worktree remove --force "$_PIN_WT" 2>/dev/null || rm -rf "$_PIN_WT"
            fi
            [[ "$_BUILD_RC" == "0" ]] \
                || die "autocatpath wheel build failed in ${_BUILD_DIR} — the cluster cannot resolve autocatpath>=${_FLOOR} without it. (gr263082)"
            _WHEEL="$(newest_autocatpath_wheel "${_CATPATH_DIR}/dist" || true)"
            [[ -n "$_WHEEL" ]] && version_ge "$(autocatpath_wheel_version "$_WHEEL")" "$_FLOOR" \
                || die "built autocatpath in ${_CATPATH_DIR} but still no wheel >=${_FLOOR} in dist/. (gr263082)"
        fi
        say "autocatpath wheel → $(basename "$_WHEEL") (floor >=${_FLOOR})"
        EXTRA_ARGS+=(-e autocatpath_wheel="$_WHEEL")
    fi
fi

# ── 2. deploy ───────────────────────────────────────────────────────────
# Attempt stamp (gr332009): past this point hosts get touched. Record the
# attempt now; the success marker at step 3 replaces it and removes this. If
# any play dies red, the stamp survives with no newer marker and scripts/ship
# reports "deploy state uncertain" instead of counting lag from a stale sha.
# Best-effort — never blocks a deploy.
{
    . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
    printf '%s %s %s\n' "$(git -C "$REPO_ROOT" rev-parse "$REF" 2>/dev/null || printf '%s' "$REF")" "$(date +%s)" "attempt" \
        > "$(deploy_attempt_path "$REPO_ROOT")"
} 2>/dev/null || true

CANARY="${PRECIS_DEPLOY_CANARY:-}"
if [[ -z "$CANARY" ]]; then
    # unchanged single-pass path (PRECIS_DEPLOY_CANARY unset). Routed through
    # _run_rollout_playbook (gr332009 graft) so a rollout that genuinely
    # converged still gets its success marker written (and attempt stamp
    # cleared) even if a later, unrelated play in this same invocation dies —
    # see that function's header comment.
    if [[ "$REF" == "main" ]]; then
        say "deploying main to the cluster"
        _run_rollout_playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
            ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
            || die "redeploy failed — see the failing task above (nodes may be on mixed versions; re-run once fixed)."
    else
        say "deploying ref '${REF}' to the cluster"
        _run_rollout_playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
            -e precis_worker_git_ref="$REF" -e precis_web_git_ref="$REF" \
            ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
            || die "redeploy failed — see the failing task above (nodes may be on mixed versions; re-run once fixed)."
    fi

    say "deployed — cluster is running '${REF}'."
else
    # ── canary path (PRECIS_DEPLOY_CANARY set): one host first, verified,
    # then the fleet. Resolve the sha ONCE (same ls-remote the playbook's
    # step-0 pin does) and pass it as explicit `-e` pins to BOTH phases — an
    # explicit `-e` wins over the playbook's own set_fact pin, so both
    # ansible-playbook invocations below install the identical sha even if
    # main advances between phase 1 and phase 2.
    #
    # `git ls-remote` matches ref NAMES, so a literal commit sha — the COMMON
    # target since /go deploys `$(cat .ship-sha) --pinned` — never resolved
    # and the canary path died before touching a host (gr346747). A REF this
    # checkout already contains as a commit is taken from the local object
    # store instead; it must also be reachable from a remote-tracking branch,
    # since the hosts install precis-mcp@<sha> from GitHub.
    _target_sha=""
    if [[ "$REF" =~ ^[0-9a-f]{7,40}$ ]]; then
        _target_sha="$(git -C "$REPO_ROOT" rev-parse -q --verify "${REF}^{commit}" 2>/dev/null || true)"
    fi
    if [[ ${#_target_sha} -eq 40 ]]; then
        [[ -n "$(git -C "$REPO_ROOT" branch -r --contains "$_target_sha" 2>/dev/null)" ]] \
            || die "'${REF}' (${_target_sha:0:8}) is not on any remote-tracking branch — the hosts install from GitHub, so it must be pushed first."
        say "canary deploy: '${REF}' is a commit this checkout contains — pinned ${_target_sha:0:8}"
    else
        say "canary deploy: resolving '${REF}' to a single commit (git ls-remote)"
        _target_sha="$(git ls-remote https://github.com/retospect/precis-mcp "$REF" | awk 'NR==1{print $1}')"
        [[ ${#_target_sha} -eq 40 ]] || die "could not resolve '${REF}' to a commit via git ls-remote (got '${_target_sha:-empty}') — refusing to deploy an unpinned/unresolvable ref."
    fi
    SHA_ARGS=(-e precis_worker_git_ref="$_target_sha" -e precis_web_git_ref="$_target_sha" -e precis_embedder_git_ref="$_target_sha")
    _rollback_hint="rollback: scripts/deploy <previous-sha> with PRECIS_DEPLOY_CANARY=${CANARY} (targets just the canary)."

    say "canary deploy: phase 1 — ${CANARY} only (pinned ${_target_sha:0:8})"
    _phase1_start="$(date +%s)"
    ansible-playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
        --limit "$CANARY" \
        "${SHA_ARGS[@]}" \
        ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
        || die "canary phase 1 (${CANARY}) failed — see the failing task above; the rest of the fleet was never touched."

    CANARY_TIMEOUT_S="${PRECIS_DEPLOY_CANARY_TIMEOUT_S:-300}"
    DB_HOST="${PRECIS_DEPLOY_CANARY_DB_HOST:-$CANARY}"
    [[ "$DB_HOST" =~ ^[A-Za-z0-9_.-]+$ ]] || die "PRECIS_DEPLOY_CANARY_DB_HOST/'${DB_HOST}': not a plausible hostname"
    say "canary deploy: verifying ${CANARY}'s heartbeat (host_heartbeat.host='${DB_HOST}') freshens (timeout ${CANARY_TIMEOUT_S}s)"
    _elapsed=0
    _green=""
    while (( _elapsed < CANARY_TIMEOUT_S )); do
        _ts_epoch="$(PRECIS_PROD_PSQL_OPTS="-At" scripts/prod-psql \
            "SELECT extract(epoch from ts)::bigint FROM host_heartbeat WHERE host = '${DB_HOST}';")" \
            || die "canary verify: scripts/prod-psql failed reaching prod — treated as red (fail closed).
mixed state: ${CANARY} may be on new code (${_target_sha}), the rest of the fleet is untouched (still on old code).
${_rollback_hint}"
        _now="$(date +%s)"
        if [[ -n "$_ts_epoch" ]] && (( _ts_epoch > _phase1_start )) && (( _now - _ts_epoch < 120 )); then
            _green=1
            break
        fi
        sleep 15
        _elapsed=$((_elapsed + 15))
    done

    [[ -n "$_green" ]] || die "canary verify: ${CANARY}'s host_heartbeat (host='${DB_HOST}') did not freshen within ${CANARY_TIMEOUT_S}s.
mixed state: ${CANARY} may be on new code (${_target_sha}), the rest of the fleet is untouched (still on old code).
${_rollback_hint}"

    say "canary deploy: ${CANARY} verified fresh on new code — phase 2 (rest of the fleet)"
    ansible-playbook redeploy-precis.yml ${INV_ARGS[@]+"${INV_ARGS[@]}"} \
        --limit "all:!${CANARY}" \
        "${SHA_ARGS[@]}" \
        ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
        || die "canary phase 2 (fleet) failed — see the failing task above. ${CANARY} is on new code (${_target_sha}); the rest of the fleet may be mixed — re-run once fixed."

    REF="$_target_sha"
    say "deployed — cluster is running '${REF}' (canary ${CANARY} verified first, then the fleet)."
fi

# ── 3. record the deploy-state marker (for scripts/ship's lag report and
# scripts/deploy's own rollback guard) ────────────────────────────────────
# Gitignored, local-only: `<sha> <epoch> <outcome>` of what's now actually
# running on the cluster, so a later `scripts/ship` can tell how many commits
# + how much time have accumulated since, and a later `scripts/deploy` can
# refuse to roll it backward (gr338201). Best-effort — never fails a
# successful deploy.
{
    . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
    DEPLOYED_SHA="$(git -C "$REPO_ROOT" rev-parse "$REF" 2>/dev/null || true)"
    if [[ -n "$DEPLOYED_SHA" ]]; then
        # Shared across worktrees (git common dir): the fleet's sha is a global
        # fact, and a per-worktree marker made every OTHER tree's ship
        # over-report the lag. See scripts/lib/deploy-state.sh.
        . "${REPO_ROOT}/scripts/lib/deploy-state.sh"
        printf '%s %s %s\n' "$DEPLOYED_SHA" "$(date +%s)" "success" \
            > "$(deploy_state_path "$REPO_ROOT")"
    fi
    # Success: clear the attempt stamp so ship stops reporting "uncertain"
    # (gr332009). Unconditional — a deploy that got here ran green.
    rm -f "$(deploy_attempt_path "$REPO_ROOT")" 2>/dev/null
    # The pin has done its job: this sha is what the fleet runs now.
    _consume_ship_pin "$DEPLOYED_SHA"
} || true
