#!/bin/bash
################################################################################
##                         EnvoyAnalyze
################################################################################
## Version 3.1
## Developed by SageCor Solutions, Annapolis Junction, MD
################################################################################
##
## Description: Generate a mix of useful and throw-away analyses for CSV/TSV
##              datasets. Designed to be exploratory like MFE: run once to see
##              what looks good, then re-run with a selection list or a saved
##              profile.
##
################################################################################
##
## Usage: envoy-analyze [OPTIONS] [FILE] [ANALYSIS-LIST]
##        cat file | envoy-analyze [OPTIONS] [ANALYSIS-LIST]
##
################################################################################

set -euo pipefail

# ENV-181: force LC_NUMERIC=C so awk's numeric parsing is locale-
# independent. On systems set to a comma-decimal locale (de_DE.*,
# fr_FR.*, …) the awk numeric_combined.awk helper's
# `split("30.1 17.6 …", expected, " ")` would mis-parse and emit
# garbage statistics. C locale uses `.` for the decimal separator
# and matches the format every awk script in the suite assumes.
export LC_NUMERIC=C

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROFILE_DIR_DEFAULT="$SCRIPT_DIR/profiles"
if [[ -r "$SCRIPT_DIR/../lib/envoy_version.sh" ]]; then
    # shellcheck disable=SC1091
    source "$SCRIPT_DIR/../lib/envoy_version.sh"
elif [[ -r "${ENVOY_HOME:-}/lib/envoy_version.sh" ]]; then
    # shellcheck disable=SC1091
    source "${ENVOY_HOME}/lib/envoy_version.sh"
fi
if declare -F envoy_suite_version >/dev/null; then
    VERSION="$(envoy_suite_version "$SCRIPT_DIR")"
else
    VERSION="0.0.0+unknown"
fi

MAX_CORR_COLS=8
MAX_GROUP_UNIQUE=12
MAX_GROUPS=5
MAX_GROUP_NUM_COLS=4
SPARK_SAMPLES=24

########################################
## SOURCE LIBRARIES

# ENV-137: shared typo / suggestion helpers (lib/envoy_suggest.sh at suite
# root). Tolerate missing file when running from an in-tree dev checkout
# without the suite root on disk; the typo path is best-effort.
if [[ -r "$SCRIPT_DIR/../lib/envoy_suggest.sh" ]]; then
    # shellcheck disable=SC1091
    source "$SCRIPT_DIR/../lib/envoy_suggest.sh"
elif [[ -r "${ENVOY_HOME:-}/lib/envoy_suggest.sh" ]]; then
    # shellcheck disable=SC1091
    source "${ENVOY_HOME}/lib/envoy_suggest.sh"
fi

# ENV-143: shared pager helper.  Tolerate missing file (partial dev checkout).
if [[ -r "$SCRIPT_DIR/../lib/envoy_pager.sh" ]]; then
    # shellcheck disable=SC1091
    source "$SCRIPT_DIR/../lib/envoy_pager.sh"
elif [[ -r "${ENVOY_HOME:-}/lib/envoy_pager.sh" ]]; then
    # shellcheck disable=SC1091
    source "${ENVOY_HOME}/lib/envoy_pager.sh"
fi
if ! declare -F envoy_maybe_page >/dev/null; then
    envoy_maybe_page() { cat; }
    envoy_pager_disable() { :; }
    envoy_pager_active() { return 1; }
fi

# ENV-152: shared output-file helper (atomic write + --force + parent-dir
# creation + `-` for stdout).  Tolerate missing file: a fallback that just
# redirects stdout achieves a usable subset.
if [[ -r "$SCRIPT_DIR/../lib/envoy_output.sh" ]]; then
    # shellcheck disable=SC1091
    source "$SCRIPT_DIR/../lib/envoy_output.sh"
elif [[ -r "${ENVOY_HOME:-}/lib/envoy_output.sh" ]]; then
    # shellcheck disable=SC1091
    source "${ENVOY_HOME}/lib/envoy_output.sh"
fi
if ! declare -F envoy_output_filter >/dev/null; then
    envoy_output_filter() { cat > "$1"; }
    envoy_output_validate() { :; }
fi

# ENV-135: shared progress helper. Tolerate missing file with no-op shims.
if [[ -r "$SCRIPT_DIR/../lib/envoy_progress.sh" ]]; then
    # shellcheck disable=SC1091
    source "$SCRIPT_DIR/../lib/envoy_progress.sh"
elif [[ -r "${ENVOY_HOME:-}/lib/envoy_progress.sh" ]]; then
    # shellcheck disable=SC1091
    source "${ENVOY_HOME}/lib/envoy_progress.sh"
fi
if ! declare -F envoy_progress_init >/dev/null; then
    envoy_progress_init() { :; }
    envoy_progress_active() { return 1; }
    envoy_progress_tick() { :; }
    envoy_progress_done() { :; }
fi

# ENV-154: suite-wide config layer (XDG → ~/.envoyrc → upward-walked).
# Loaded before flag parsing so .envoyrc-set keys are visible to argv
# defaults; flags still override via the loader's snapshot semantics.
if [[ -r "$SCRIPT_DIR/../lib/envoy_envoyrc.sh" ]]; then
    # shellcheck disable=SC1091
    source "$SCRIPT_DIR/../lib/envoy_envoyrc.sh"
elif [[ -r "${ENVOY_HOME:-}/lib/envoy_envoyrc.sh" ]]; then
    # shellcheck disable=SC1091
    source "${ENVOY_HOME}/lib/envoy_envoyrc.sh"
fi
if declare -F envoy_envoyrc_load >/dev/null; then
    envoy_envoyrc_load
fi

source "$SCRIPT_DIR/lib/_helpers.sh"
source "$SCRIPT_DIR/lib/_csv.sh"
source "$SCRIPT_DIR/lib/_profile.sh"
source "$SCRIPT_DIR/lib/_format.sh"
source "$SCRIPT_DIR/lib/_numeric.sh"
source "$SCRIPT_DIR/lib/_categorical.sh"
source "$SCRIPT_DIR/lib/_correlation.sh"
source "$SCRIPT_DIR/lib/_quality.sh"
source "$SCRIPT_DIR/lib/_compare.sh"
source "$SCRIPT_DIR/lib/_threshold.sh"
source "$SCRIPT_DIR/lib/_summary.sh"

########################################
## USAGE

usage () {
    cat <<USAGE
EnvoyAnalyze (envoy-analyze) v${VERSION}

Usage: envoy-analyze [OPTIONS] [FILE] [ANALYSIS-LIST]
       cat file | envoy-analyze [OPTIONS] [ANALYSIS-LIST]

Generate basic analyses for CSV/TSV datasets. Outputs a numbered list of
analyses, including useful items and noisy throw-aways. Re-run with a list of
analysis numbers to select the "good stuff".

Options:
  -h, --help            Show this help message
  -V, --version         Show version
  -v, --verbose         Enable verbose output (sets ENVOY_VERBOSE=1) — ENV-151
  -q, --quiet           Suppress info / hint output (sets ENVOY_QUIET=1) — ENV-151
  --progress            Emit per-column stderr progress in the stats stage
                        (auto on TTY ≥20 columns; ENV-135)
  --no-progress         Disable progress (also: ENVOY_NO_PROGRESS=1)
                        --verbose and --quiet are mutually exclusive; later flag wins.
  -o, --output FILE     Write output to FILE atomically (ENV-152). "-" or
                        /dev/stdout writes to stdout. Refuses to overwrite
                        an existing FILE without --force. Creates parent
                        directories on demand.  Implies --no-pager.
  -f, --force           Allow --output to overwrite an existing file (ENV-152).
  --delimiter DELIM     Force delimiter (default: auto-detect comma/tab/semicolon)
  --profile NAME        Use a saved selection profile
  --save NAME           Save the provided analysis list as a profile
  --profile-dir DIR     Directory for profiles (default: ./profiles)
  --good                Emit only items tagged GOOD (default)
  --all                 Emit all analyses (including NOISY, ADVANCED, DIAGNOSTIC)
  --tags TAGS           Emit only items with these tags (GOOD, NOISY, ADVANCED, DIAGNOSTIC)
  --kinds KINDS         Emit only items with these kinds (comma or space list)
  --list-items          Print the menu of analysis items for the dataset and exit
  --suggest             Print a suggested analysis list and exit
  --suggest-save NAME   Save a suggested list as a profile
  --format FMT          Output format: plain (default), json, markdown, csv, tsv, html
  --max-corr-cols N     Max columns for correlation matrix (default: 8)
  --max-groups N        Max group-by columns (default: 5)
  --max-group-unique N  Max unique values per group column (default: 12)
  --columns COLS        Select specific columns by number (e.g., 1,3,5)
  --compare FILE ...    Compare stats across multiple datasets
  --threshold SPEC      Alert if column stat violates threshold. Supports
                        AND/OR composition and parentheses, e.g.
                        'errors>10 AND latency_p99>500'. May be repeated
                        (multiple --threshold flags combine with OR).
                        Exit code 1 if any threshold fails.
  --mode MODE           Analysis mode: benchmark, monitor, or explore (default)
  --summary             Append a natural-language findings summary
  --timeseries          Promote trend/monotonicity items to GOOD
  --top-n N             Limit output to the N most significant findings
  --numeric-columns LIST    Comma-separated column names to force numeric
                            (overrides the 0.80 auto-detection heuristic)
  --categorical-columns LIST Comma-separated column names to force categorical
  --strict              Exit non-zero when any column classified as numeric
                        has non-numeric rows (rows that would be silently
                        excluded from mean / std). Off by default; intended
                        for CI / scripted consumers that want fail-fast.
  --missing-values LIST Comma-separated strings counted as missing, e.g.
                        'NA,N/A,NULL,UNKNOWN,?'. Empty string is always missing.
  --missing-values-case-insensitive
                        Make --missing-values matching case-insensitive
  --benford-alpha A     Significance level for Benford chi-squared test:
                        0.05 (default, critical=15.507), 0.01 (20.090),
                        0.001 (26.125). df=8.
  --color WHEN          ANSI colour in plain output: auto (default, TTY-detect),
                        always, or never. NO_COLOR=1 forces never.
  --no-pager            Don't auto-page plain output even on a TTY.
                        Also honoured: ENVOY_NO_PAGER=1, PAGER="" or PAGER=cat.
                        Default pager is 'less -FRX'. (ENV-143)
  --head N              Analyse only the first N data rows (header preserved)
  --sample N            Analyse a uniform-random sample of N data rows
  --seed S              Integer seed for reproducible --sample (default: time-based)

Arguments:
  FILE                  Input file (reads stdin if omitted)
  ANALYSIS-LIST         Comma-delimited list of analysis numbers to output

Examples:
  envoy-analyze samples/trees.csv                    Analyze (key findings only, default)
  envoy-analyze --all samples/trees.csv              Show all analyses
  envoy-analyze samples/trees.csv 1,3,5              Show specific analyses
  envoy-analyze --save trees-core samples/trees.csv 1,2,3
  envoy-analyze --profile trees-core samples/trees.csv
  envoy-analyze --suggest samples/trees.csv
  envoy-analyze --compare a.csv b.csv c.csv          Compare datasets
  envoy-analyze --threshold "Index>0" samples/trees.csv
  envoy-analyze --mode benchmark samples/trees.csv
  envoy-analyze --summary samples/trees.csv

Pipeline (stdin piping):
  envoy-extract --format tsv data.log | envoy-analyze
  envoy-extract --format tsv data.log | envoy-analyze --all
  envoy-extract --format tsv data.log | envoy-analyze 1,3,5
USAGE
}

########################################
## ARG PARSING

input_file=""
analysis_list=""
profile_name=""
save_name=""
profile_dir="$PROFILE_DIR_DEFAULT"
forced_delim=""
only_good=1
tag_filter=""
kind_filter=""
suggest_only=0
list_items_only=0
suggest_save_name=""
output_format="plain"
cleanup_files=()
column_selection=""
compare_files=()
compare_mode=0
threshold_specs=()
threshold_exit_code=0
analysis_mode=""
emit_summary=0
timeseries_mode=0
top_n=""
# ENV-025: explicit type overrides. Comma-separated column names; case-insensitive match.
force_numeric_cols=""
force_categorical_cols=""
# ENV-132: --strict makes envoy-analyze exit non-zero if any numeric
# column has impure rows (any value rejected by the numeric parse). Off by
# default; intended for CI / scripted consumers that want fail-fast.
strict_numeric=0
# ENV-026: custom missing-value forms. Comma-separated list; empty string always missing.
custom_missing_values=""
missing_case_insensitive=0
# ENV-058: categorical-column heuristic thresholds
categorical_unique_max="${CATEGORICAL_UNIQUE_MAX:-10}"
categorical_ratio_max="${CATEGORICAL_RATIO_MAX:-0.5}"
# ENV-067: Benford significance level (alpha). Chi-squared df=8 critical values.
BENFORD_ALPHA="0.05"
BENFORD_CRIT="15.507"
# ENV-065: colour control for plain output. auto = TTY detect, honour NO_COLOR.
color_when="auto"
# ENV-063: row-sampling for fast exploration of large files.
head_n=""
sample_n=""
sample_seed=""
# ENV-152: -o / --output FILE redirects the structured-analysis output to
# FILE (atomic write).  --force allows overwriting an existing file.
output_file=""
output_force=0

while [[ $# -gt 0 ]]; do
    case "$1" in
        -h|--help)
            usage
            exit 0
            ;;
        -V|--version)
            echo "envoy-analyze ${VERSION} — Developed by SageCor Solutions, Annapolis Junction, MD"
            exit 0
            ;;
        -v|--verbose)
            # ENV-151: enable verbose info output (gates _log_info elsewhere
            # in the suite).  Last-flag-wins against --quiet.
            ENVOY_VERBOSE=1; export ENVOY_VERBOSE
            unset ENVOY_QUIET
            shift
            ;;
        -q|--quiet)
            # ENV-151: suppress info / hint output (errors still print).
            ENVOY_QUIET=1; export ENVOY_QUIET
            unset ENVOY_VERBOSE
            shift
            ;;
        --progress)
            # ENV-135: opt-in stderr progress (works on non-TTY too).
            export ENVOY_PROGRESS_OPT_IN=1; unset ENVOY_PROGRESS_DISABLE
            shift
            ;;
        --no-progress)
            # ENV-135: explicit suppression. Also: ENVOY_NO_PROGRESS=1.
            export ENVOY_PROGRESS_DISABLE=1; unset ENVOY_PROGRESS_OPT_IN
            shift
            ;;
        -o|--output)
            # ENV-152: redirect output to a file.  "-" / "/dev/stdout" pass
            # through; otherwise atomic write to FILE.tmp + rename.
            output_file="${2:-}"
            [[ -n "$output_file" ]] || err "--output requires a path (use - for stdout)"
            shift 2
            ;;
        -f|--force)
            # ENV-152: opt-in overwrite of the --output target.
            output_force=1
            shift
            ;;
        --delimiter)
            forced_delim="${2:-}"
            [[ -n "$forced_delim" ]] || err "--delimiter requires a value"
            shift 2
            ;;
        --profile)
            profile_name="${2:-}"
            [[ -n "$profile_name" ]] || err "--profile requires a name"
            shift 2
            ;;
        --save)
            save_name="${2:-}"
            [[ -n "$save_name" ]] || err "--save requires a name"
            shift 2
            ;;
        --profile-dir)
            profile_dir="${2:-}"
            [[ -n "$profile_dir" ]] || err "--profile-dir requires a value"
            shift 2
            ;;
        --good)
            only_good=1
            shift
            ;;
        --all)
            only_good=0
            shift
            ;;
        --tags)
            tag_filter="${2:-}"
            [[ -n "$tag_filter" ]] || err "--tags requires a value"
            shift 2
            ;;
        --kinds)
            kind_filter="${2:-}"
            [[ -n "$kind_filter" ]] || err "--kinds requires a value"
            shift 2
            ;;
        --suggest)
            suggest_only=1
            shift
            ;;
        --list-items)
            list_items_only=1
            shift
            ;;
        --suggest-save)
            suggest_save_name="${2:-}"
            [[ -n "$suggest_save_name" ]] || err "--suggest-save requires a name"
            shift 2
            ;;
        --format)
            output_format="${2:-}"
            [[ -n "$output_format" ]] || err "--format requires a value"
            shift 2
            ;;
        --no-pager)
            # ENV-143: per-process disable of auto-paging.
            envoy_pager_disable
            shift
            ;;
        --max-corr-cols)
            MAX_CORR_COLS="${2:-}"
            [[ -n "$MAX_CORR_COLS" ]] || err "--max-corr-cols requires a value"
            shift 2
            ;;
        --max-groups)
            MAX_GROUPS="${2:-}"
            [[ -n "$MAX_GROUPS" ]] || err "--max-groups requires a value"
            shift 2
            ;;
        --max-group-unique)
            MAX_GROUP_UNIQUE="${2:-}"
            [[ -n "$MAX_GROUP_UNIQUE" ]] || err "--max-group-unique requires a value"
            shift 2
            ;;
        --columns)
            column_selection="${2:-}"
            [[ -n "$column_selection" ]] || err "--columns requires a value (e.g., 1,3,5)"
            shift 2
            ;;
        --compare)
            compare_mode=1
            shift
            while [[ $# -gt 0 && "$1" != --* ]]; do
                compare_files+=("$1")
                shift
            done
            [[ ${#compare_files[@]} -ge 2 ]] || err "--compare requires at least 2 files"
            # Use first compare file as input_file if none set yet
            if [[ -z "$input_file" ]]; then
                input_file="${compare_files[0]}"
            fi
            ;;
        --threshold)
            [[ -n "${2:-}" ]] || err "--threshold requires a value (e.g., 'column>100')"
            threshold_specs+=("$2")
            shift 2
            ;;
        --mode)
            analysis_mode="${2:-}"
            [[ -n "$analysis_mode" ]] || err "--mode requires a value (benchmark|monitor|explore)"
            shift 2
            ;;
        --summary)
            emit_summary=1
            shift
            ;;
        --timeseries)
            timeseries_mode=1
            shift
            ;;
        --top-n)
            top_n="${2:-}"
            [[ -n "$top_n" ]] || err "--top-n requires a numeric value"
            [[ "$top_n" =~ ^[0-9]+$ ]] || err "--top-n requires a non-negative integer, got: $top_n"
            shift 2
            ;;
        --numeric-columns)
            force_numeric_cols="${2:-}"
            [[ -n "$force_numeric_cols" ]] || err "--numeric-columns requires a comma-separated column list"
            shift 2
            ;;
        --categorical-columns)
            force_categorical_cols="${2:-}"
            [[ -n "$force_categorical_cols" ]] || err "--categorical-columns requires a comma-separated column list"
            shift 2
            ;;
        --strict)
            # ENV-132: fail-fast when any numeric column has non-numeric rows.
            strict_numeric=1
            shift
            ;;
        --missing-values)
            custom_missing_values="${2:-}"
            [[ -n "$custom_missing_values" ]] || err "--missing-values requires a comma-separated list"
            shift 2
            ;;
        --missing-values-case-insensitive)
            missing_case_insensitive=1
            shift
            ;;
        --categorical-unique-max)
            categorical_unique_max="${2:-}"
            [[ "$categorical_unique_max" =~ ^[0-9]+$ ]] || err "--categorical-unique-max requires a non-negative integer"
            shift 2
            ;;
        --categorical-ratio-max)
            categorical_ratio_max="${2:-}"
            [[ -n "$categorical_ratio_max" ]] || err "--categorical-ratio-max requires a value between 0 and 1"
            shift 2
            ;;
        --color)
            color_when="${2:-}"
            case "$color_when" in
                auto|always|never) ;;
                *) err "--color must be auto, always, or never (got: ${color_when:-<empty>})" ;;
            esac
            shift 2
            ;;
        --head)
            head_n="${2:-}"
            [[ "$head_n" =~ ^[1-9][0-9]*$ ]] || err "--head requires a positive integer, got: ${head_n:-<empty>}"
            shift 2
            ;;
        --sample)
            sample_n="${2:-}"
            [[ "$sample_n" =~ ^[1-9][0-9]*$ ]] || err "--sample requires a positive integer, got: ${sample_n:-<empty>}"
            shift 2
            ;;
        --seed)
            sample_seed="${2:-}"
            [[ "$sample_seed" =~ ^[0-9]+$ ]] || err "--seed requires a non-negative integer, got: ${sample_seed:-<empty>}"
            shift 2
            ;;
        --benford-alpha)
            BENFORD_ALPHA="${2:-}"
            case "$BENFORD_ALPHA" in
                0.05)  BENFORD_CRIT="15.507" ;;
                0.01)  BENFORD_CRIT="20.090" ;;
                0.001) BENFORD_CRIT="26.125" ;;
                *) err "--benford-alpha must be 0.05, 0.01, or 0.001 (got: ${BENFORD_ALPHA:-<empty>})" ;;
            esac
            export BENFORD_ALPHA BENFORD_CRIT
            shift 2
            ;;
        --)
            shift
            break
            ;;
        -* )
            err "Unknown option: $1"
            ;;
        *)
            if [[ -z "$input_file" ]]; then
                # When stdin is piped and the argument looks like an analysis
                # list (comma/space-separated numbers), treat it as the list
                # rather than a filename.  This enables:
                #   envoy-extract --format tsv file | envoy-analyze 1,3,5
                if ! [[ -t 0 ]] && [[ "$1" =~ ^[0-9][0-9,\ ]*$ ]]; then
                    analysis_list="$1"
                else
                    input_file="$1"
                fi
            elif [[ -z "$analysis_list" ]]; then
                analysis_list="$1"
            else
                err "Unexpected argument: $1"
            fi
            shift
            ;;
    esac
done

# If user provided specific analysis numbers or a profile, don't filter by --good
if [[ -n "$analysis_list" ]] || [[ -n "$profile_name" ]]; then
    only_good=0
fi

case "$output_format" in
    plain|json|markdown|csv|tsv|html) ;;
    *) err "Unknown format: $output_format" ;;
esac

# ENV-152: pre-flight the output destination before doing the analysis
# work.  Fails fast on overwrite-without-force / bad parent dir.
if [[ -n "$output_file" ]]; then
    envoy_output_validate "$output_file" "$output_force" || exit 1
    # When writing to a file, the pager would buffer content the user
    # never sees.  Disable it explicitly so `-o file --no-pager` is the
    # implicit pairing.
    envoy_pager_disable
fi

# ENV-065: resolve final colour decision. NO_COLOR always wins; only plain format
# ever emits ANSI (JSON/CSV/TSV/HTML/markdown stay clean regardless).
COLOR_ENABLED=0
if [[ "$output_format" == "plain" ]]; then
    if [[ -n "${NO_COLOR:-}" ]]; then
        COLOR_ENABLED=0
    elif [[ "$color_when" == "always" ]]; then
        COLOR_ENABLED=1
    elif [[ "$color_when" == "auto" ]]; then
        if [[ -t 1 && "${TERM:-dumb}" != "dumb" ]]; then
            COLOR_ENABLED=1
        fi
    fi
fi
export COLOR_ENABLED

case "$analysis_mode" in
    ""|benchmark|monitor|explore) ;;
    *) err "Unknown mode: $analysis_mode (use benchmark, monitor, or explore)" ;;
esac

# ENV-058: let --categorical-unique-max act as the canonical version of
# --max-group-unique. If the user set --categorical-unique-max but not
# --max-group-unique, propagate the value through.
if [[ -n "${categorical_unique_max:-}" ]]; then
    MAX_GROUP_UNIQUE="$categorical_unique_max"
fi

# ENV-056: detect non-GNU awk and warn. GNU-only features (gensub, asort,
# array semantics) mean mawk/BSD awk will silently produce empty output.
if [[ "${ENVOY_ANALYZE_SUPPRESS_AWK_WARNING:-0}" != "1" ]]; then
    # Don't pipe `awk --version` through `head -1`: under `set -o
    # pipefail` the closed-stdin SIGPIPE from `head` propagates as the
    # pipeline's exit status (141), which `set -e` then turns into an
    # abort of the whole script. Race-prone when awk finishes writing
    # AFTER head closes (mawk on busy systems triggers it ~5% of runs;
    # surfaced via the EnvoyAnalyze test suite, ENV-125 follow-up).
    # Read the full output and slice the first line in pure bash.
    _awk_version="$(awk --version 2>&1 || true)"
    _awk_version="${_awk_version%%$'\n'*}"
    if [[ "$_awk_version" != *"GNU Awk"* && "$_awk_version" != *"gawk"* ]]; then
        echo "envoy-analyze: warning: requires GNU awk (gawk); detected: ${_awk_version:-unknown}" >&2
        echo "envoy-analyze: warning: some analyses may produce empty or malformed output. See README § AWK Requirements." >&2
        echo "envoy-analyze: warning: suppress with ENVOY_ANALYZE_SUPPRESS_AWK_WARNING=1." >&2
    fi
fi

# ENV-026: replace AWK_ISMISSING with a user-defined form list when provided.
# Empty string is always missing (non-overridable). Case-insensitive matching
# is opt-in via --missing-values-case-insensitive.
if [[ -n "$custom_missing_values" ]]; then
    _mv_conds='v==""'
    _old_ifs="$IFS"
    IFS=',' read -ra _mv_items <<< "$custom_missing_values"
    IFS="$_old_ifs"
    for _mv in "${_mv_items[@]}"; do
        [[ -z "$_mv" ]] && continue
        # Escape backslashes and double-quotes for embedding in awk string literal
        _esc="${_mv//\\/\\\\}"
        _esc="${_esc//\"/\\\"}"
        if [[ "$missing_case_insensitive" -eq 1 ]]; then
            _mv_conds+=" || tolower(v)==tolower(\"$_esc\")"
        else
            _mv_conds+=" || v==\"$_esc\""
        fi
    done
    AWK_ISMISSING="function ismissing(v) { return ($_mv_conds) }"
fi

if [[ -n "$profile_name" ]] && [[ -z "$analysis_list" ]]; then
    analysis_list=$(load_profile "$profile_name" "$profile_dir")
fi

if [[ -z "$input_file" ]]; then
    if [[ -t 0 ]]; then
        usage
        err "No input file provided and stdin is empty"
    fi
    tmp_input="$(mktemp -t mda-input.XXXXXX)"
    cat > "$tmp_input"
    input_file="$tmp_input"
    cleanup_files+=("$tmp_input")
fi

if [[ ! -f "$input_file" ]]; then
    # ENV-137: distinguish between "you typo'd a flag / subcommand" and a
    # genuine missing-file error. The typo case is much more common in
    # interactive use and the old "File not found" message blamed the
    # filesystem instead of the input.
    if declare -f _envoy_arg_looks_like_typo >/dev/null 2>&1 \
       && _envoy_arg_looks_like_typo "$input_file"; then
        echo "envoy-analyze: error: '$input_file' is not a known option or file" >&2
        echo "  If you meant a file, check the path." >&2
        echo "  If you meant a flag or subcommand, run 'envoy-analyze --help'." >&2
        exit 2
    fi
    err "File not found: $input_file"
fi

########################################
## SETUP

cleanup () {
    for f in "${cleanup_files[@]:-}"; do
        [[ -n "$f" && -f "$f" ]] && rm -f "$f"
    done
    return 0
}
trap cleanup EXIT
# ENV-109: explicit INT / TERM / HUP traps ensure deterministic exit codes
# (130 / 143 / 129) on the non-exit signal paths. Each calls the same cleanup
# function so temp files are removed regardless of how we exit.
trap 'cleanup; exit 130' INT
trap 'cleanup; exit 143' TERM
trap 'cleanup; exit 129' HUP

# Strip UTF-8 BOM if present (works on a copy; never modifies the original)
strip_bom "$input_file"
input_file="$_STRIP_BOM_RESULT"

# CSV preprocessing: convert quoted CSV to clean TSV
if [[ -n "$forced_delim" ]]; then
    delim="$forced_delim"
else
    delim="$(detect_delim "$input_file")"
fi

if needs_csv_preprocess "$input_file"; then
    tmp_tsv="$(mktemp -t mda-tsv.XXXXXX)"
    cleanup_files+=("$tmp_tsv")
    csv_to_tsv "$input_file" "$delim" > "$tmp_tsv"
    input_file="$tmp_tsv"
    delim=$'\t'
fi

# ENV-063: apply --head / --sample after preprocessing. Mutually exclusive; both
# write a new temp file that the rest of the pipeline treats as the input.
if [[ -n "$head_n" && -n "$sample_n" ]]; then
    err "--head and --sample are mutually exclusive"
fi
SAMPLE_NOTE=""
SAMPLE_TOTAL=0
if [[ -n "$head_n" || -n "$sample_n" ]]; then
    SAMPLE_TOTAL=$(awk 'NR>1{c++} END{print c+0}' "$input_file")
    tmp_sample="$(mktemp -t mda-sample.XXXXXX)"
    cleanup_files+=("$tmp_sample")
    if [[ -n "$head_n" ]]; then
        awk -v n="$head_n" 'NR==1{print; next} NR<=n+1{print}' "$input_file" > "$tmp_sample"
        SAMPLE_NOTE="(first $head_n of $SAMPLE_TOTAL rows)"
    else
        # Reservoir sampling: deterministic when sample_seed is set.
        # ENV-205: do NOT use awk's srand()/rand() — mawk (1.3.4) does not
        # produce a reproducible sequence from srand(constant), so the same
        # --seed gave different samples under mawk vs gawk. Use a self-contained
        # Park–Miller minimal-standard LCG in pure awk arithmetic instead: its
        # output is identical on every awk (multiplier*state stays < 2^53, so no
        # double-precision loss) and fully reproducible for a fixed seed.
        local_seed="${sample_seed:-$(date +%s%N)}"
        awk -v k="$sample_n" -v seed="$local_seed" '
            function rnd(){ _s=(16807*_s)%2147483647; return _s/2147483647 }
            BEGIN{ _s=(seed+0)%2147483647; if(_s<=0)_s+=2147483646; if(_s==0)_s=1; n=0 }
            NR==1{ print; next }
            { n++
              if(n<=k){ R[n]=$0 }
              else { j=int(rnd()*n)+1; if(j<=k) R[j]=$0 }
            }
            END{ m = (n<k? n : k); for(i=1;i<=m;i++) print R[i] }
        ' "$input_file" > "$tmp_sample"
        seed_note=""
        [[ -n "$sample_seed" ]] && seed_note=", seed=$sample_seed"
        SAMPLE_NOTE="(random sample $sample_n of $SAMPLE_TOTAL rows${seed_note})"
    fi
    input_file="$tmp_sample"
fi

COL_NAMES=()
while IFS= read -r name; do
    COL_NAMES+=("$name")
done < <(awk -v FS="$delim" 'NR==1{for(i=1;i<=NF;i++){gsub(/^[ \t]+|[ \t]+$/, "", $i); print $i}}' "$input_file")
COL_COUNT=${#COL_NAMES[@]}
ROW_COUNT=$(awk 'NR>1{c++} END{print c+0}' "$input_file")

# ENV-129: reject empty / header-only input before anything downstream tries
# to index COL_NAMES. The upstream `[[ -t 0 ]]` guard at line ~458 only fires
# when stdin is a TTY; a closed pipe (`< /dev/null`, `cat empty.csv | ...`,
# any broken upstream) still lands here with COL_COUNT=0, which previously
# crashed later with "COL_NAMES[$((col_idx-1))]: unbound variable". Fail
# fast instead, with a user-visible message that doesn't leak internals.
if (( COL_COUNT == 0 )); then
    err "Input contained no data (no header row detected). Pipe a CSV/TSV with at least a header row, or pass a file."
fi
if (( ROW_COUNT == 0 )); then
    err "Input contained no data rows (only a header was found). Nothing to analyze."
fi

# ENV-025: validate explicit type-override lists. Unknown names fail fast so users
# don't silently get their override ignored. Overlapping names between
# --numeric-columns and --categorical-columns are rejected.
_validate_col_list () {
    local label="$1"
    local csv="$2"
    [[ -z "$csv" ]] && return 0
    local old_ifs="$IFS"
    IFS=',' read -ra _names <<< "$csv"
    IFS="$old_ifs"
    local n
    for n in "${_names[@]}"; do
        n="${n#"${n%%[![:space:]]*}"}"
        n="${n%"${n##*[![:space:]]}"}"
        [[ -z "$n" ]] && continue
        local found=0 col
        for col in "${COL_NAMES[@]}"; do
            if [[ "${col,,}" == "${n,,}" ]]; then
                found=1
                break
            fi
        done
        if [[ $found -eq 0 ]]; then
            err "$label references unknown column '$n'. Available columns: $(IFS=, ; echo "${COL_NAMES[*]}")"
        fi
    done
}
_validate_col_list "--numeric-columns" "$force_numeric_cols"
_validate_col_list "--categorical-columns" "$force_categorical_cols"
if [[ -n "$force_numeric_cols" && -n "$force_categorical_cols" ]]; then
    IFS=',' read -ra _num_items <<< "$force_numeric_cols"
    for n in "${_num_items[@]}"; do
        n="${n#"${n%%[![:space:]]*}"}"
        n="${n%"${n##*[![:space:]]}"}"
        [[ -z "$n" ]] && continue
        if _name_in_csv_list_ci "$n" "$force_categorical_cols"; then
            err "Column '$n' cannot appear in both --numeric-columns and --categorical-columns"
        fi
    done
fi

ITEM_TITLES=()
ITEM_BODIES=()
ITEM_TAGS=()
ITEM_KINDS=()

add_item () {
    local title="$1"
    local body="$2"
    local tag="$3"
    local kind="${4:-generic}"
    ITEM_TITLES+=("$title")
    ITEM_BODIES+=("$body")
    ITEM_TAGS+=("$tag")
    ITEM_KINDS+=("$kind")
}

########################################
## CORE ITEMS

file_size=$(wc -c < "$input_file" | tr -d ' ')

delim_name="comma"
if [[ "$delim" == $'\t' ]]; then
    delim_name="tab"
elif [[ "$delim" == "|" ]]; then
    delim_name="|"
elif [[ "$delim" == ";" ]]; then
    delim_name="semicolon"
elif [[ "$delim" == ":" ]]; then
    delim_name="colon"
fi

_overview_body="Rows: $ROW_COUNT
Columns: $COL_COUNT
Delimiter: $delim_name
File: $input_file
Size: ${file_size} bytes"
if [[ -n "${SAMPLE_NOTE:-}" ]]; then
    _overview_body+=$'\n'"Sampling: $SAMPLE_NOTE"
fi
add_item "Dataset overview" "$_overview_body" "GOOD" "overview"

# Column types and missing counts
col_types=""
missing_report=""

COL_TYPE=()
COL_NUMERIC=()
# ENV-132: parallel-array of non-numeric value counts per column. Set for
# every column from the awk pass; reads only after the column type is
# decided, so we know whether the count represents "rows excluded from
# numeric stats" (col_type=numeric, count > 0) or "fixture detail no one
# cares about" (col_type=categorical).
COL_TEXT=()
COL_MEAN=()
COL_STD=()
COL_SKEW=()
COL_KURT=()
COL_MIN=()
COL_MAX=()
COL_UNIQUE=()
COL_MISSING=()
COL_Q1=()
COL_Q3=()

# Single-pass: compute basic stats + moments for ALL columns simultaneously
all_col_stats=$(awk -F"$delim" -v ncols="$COL_COUNT" "$AWK_ISMISSING"'
    function trim(s){ gsub(/^[ \t]+|[ \t]+$/, "", s); return s }
    function isnum(x){ return (x ~ /^-?[0-9]+([.][0-9]+)?([eE][-+]?[0-9]+)?$/) }
    NR==1{next}
    {
        for(i=1; i<=ncols; i++){
            v=trim($i)
            if(ismissing(v)) {missing[i]++; continue}
            total[i]++
            if(isnum(v)){
                num[i]++
                vn=v+0
                # Welford online algorithm for numerically stable moments
                n1=num[i]
                delta=vn - wm[i]
                delta_n=delta/n1
                delta_n2=delta_n*delta_n
                term1=delta*delta_n*(n1-1)
                wm4[i]+=term1*delta_n2*(n1*n1 - 3*n1 + 3) + 6*delta_n2*wm2[i] - 4*delta_n*wm3[i]
                wm3[i]+=term1*delta_n*(n1-2) - 3*delta_n*wm2[i]
                wm2[i]+=term1
                wm[i]+=delta_n
                if(num[i]==1 || vn<mn[i]) mn[i]=vn
                if(num[i]==1 || vn>mx[i]) mx[i]=vn
            } else {
                text[i]++
            }
        }
    }
    END{
        for(i=1; i<=ncols; i++){
            n=num[i]+0; t=total[i]+0
            mean=0; std=0; skew=0; kurt=0
            if(n>0){
                mean=wm[i]
                m2=wm2[i]/n
                if(m2<0) m2=0
                std=sqrt(m2)
                m3r=wm3[i]/n
                m4r=wm4[i]/n
                skew=(m2>0? m3r/(m2^(1.5)) : 0)
                kurt=(m2>0? (m4r/(m2*m2)-3) : 0)
            }
            printf "%d\t%d\t%d\t%d\t%s\t%s\t%.6f\t%.6f\t%.6f\t%.6f\n",
                t, missing[i]+0, n, text[i]+0,
                (n>0? sprintf("%g", mn[i]) : "NA"), (n>0? sprintf("%g", mx[i]) : "NA"),
                mean, std, skew, kurt
        }
    }
' "$input_file")

col_idx=0
while IFS=$'\t' read -r total missing num text min max mean std skew kurt; do
    col_idx=$((col_idx + 1))
    col_name="${COL_NAMES[$((col_idx-1))]}"

    ratio="0.00"
    if [[ "$total" -gt 0 ]]; then
        _r100=$(( num * 100 / total ))
        ratio="$(( _r100 / 100 )).$(printf '%02d' $(( _r100 % 100 )))"
    fi

    # ENV-025: --numeric-columns / --categorical-columns override the heuristic.
    # Match is case-insensitive against the header row. Columns in both lists are
    # rejected earlier at startup.
    col_type=""
    if [[ -n "$force_numeric_cols" ]] && _name_in_csv_list_ci "$col_name" "$force_numeric_cols"; then
        col_type="numeric"
    elif [[ -n "$force_categorical_cols" ]] && _name_in_csv_list_ci "$col_name" "$force_categorical_cols"; then
        col_type="categorical"
    elif [[ "$num" -gt 0 ]] && float_cmp "$ratio" ">=" "0.80"; then
        col_type="numeric"
    else
        col_type="categorical"
    fi

    COL_TYPE+=("$col_type")
    COL_NUMERIC+=("$num")
    COL_TEXT+=("$text")
    COL_MIN+=("$min")
    COL_MAX+=("$max")
    COL_MISSING+=("$missing")

    if [[ "$col_type" == "numeric" ]]; then
        COL_MEAN+=("$mean")
        COL_STD+=("$std")
        COL_SKEW+=("$skew")
        COL_KURT+=("$kurt")
        if [[ -n "$force_numeric_cols" ]] && _name_in_csv_list_ci "$col_name" "$force_numeric_cols"; then
            col_types+="$col_name: numeric (forced via --numeric-columns)"$'\n'
        elif float_cmp "$ratio" "<" "1.00"; then
            col_types+="$col_name: numeric (ratio $ratio, $text non-numeric values excluded)"$'\n'
        else
            col_types+="$col_name: numeric (ratio $ratio)"$'\n'
        fi
    else
        COL_MEAN+=("")
        COL_STD+=("")
        COL_SKEW+=("")
        COL_KURT+=("")
        # Note in the column-types report whether the categorical classification
        # came from the user's override or from the heuristic.
        if [[ -n "$force_categorical_cols" ]] && _name_in_csv_list_ci "$col_name" "$force_categorical_cols"; then
            col_types+="$col_name: categorical (forced via --categorical-columns)"$'\n'
        else
            col_types+="$col_name: categorical (ratio $ratio)"$'\n'
            # Warn when a column has significant numeric content but falls below 80% threshold.
            # Suppressed when the user explicitly forced categorical via override (signal, not noise).
            if [[ "$num" -gt 0 ]] && float_cmp "$ratio" ">=" "0.50"; then
                echo "DIAGNOSTIC: column '$col_name' has ${num}/${total} numeric values (ratio $ratio) but falls below the 80% threshold; treating as categorical. All $num numeric values will be ignored." >&2
                add_item "Numeric exclusion warning: $col_name" "Column '$col_name' is classified as categorical but has ${num}/${total} numeric values (ratio $ratio).
The 80% numeric threshold was not met, so all $num numeric values are being ignored.
Consider: is this column actually numeric with non-numeric outliers?" "DIAGNOSTIC" "numeric_exclusion"
            fi
        fi
    fi

    missing_report+="$col_name: $missing missing"$'\n'
    COL_UNIQUE+=("0")
done <<< "$all_col_stats"

add_item "Column types (guessed)" "${col_types%$'\n'}" "GOOD" "col_types"
add_item "Missing values" "${missing_report%$'\n'}" "GOOD" "missing"

# Missingness ranking
if [[ "$ROW_COUNT" -gt 0 ]]; then
    missing_lines=""
    for ((i=1; i<=COL_COUNT; i++)); do
        miss="${COL_MISSING[$((i-1))]}"
        if [[ -n "$miss" && "$miss" -gt 0 ]]; then
            _p100=$(( miss * 10000 / ROW_COUNT ))
            pct="$(( _p100 / 100 )).$(printf '%02d' $(( _p100 % 100 )))"
            missing_lines+="$miss"$'\t'"$pct"$'\t'"${COL_NAMES[$((i-1))]}"$'\n'
        fi
    done
    if [[ -n "$missing_lines" ]]; then
        top_missing=$(printf "%s" "$missing_lines" | sort -nr | awk 'NR<=5')
        miss_body=""
        while IFS=$'\t' read -r miss pct name; do
            miss_body+="$name: $miss missing (${pct}%)"$'\n'
        done <<< "$top_missing"
        add_item "Most missing columns" "${miss_body%$'\n'}" "ADVANCED" "missing_rank"
    fi
fi

########################################
## BUILD COLUMN INDEX LISTS

NUM_COL_INDEXES=()
NUM_COL_NAMES=()
for ((i=1; i<=COL_COUNT; i++)); do
    if [[ "${COL_TYPE[$((i-1))]}" == "numeric" ]]; then
        NUM_COL_INDEXES+=("$i")
        NUM_COL_NAMES+=("${COL_NAMES[$((i-1))]}")
    fi
done

# Apply --columns filter: restrict analysis to selected columns only
if [[ -n "$column_selection" ]]; then
    _SELECTED_COLS=()
    IFS=',' read -r -a _SELECTED_COLS <<< "$column_selection"
    _FILTERED_NUM_IDX=()
    _FILTERED_NUM_NAMES=()
    for ni in "${!NUM_COL_INDEXES[@]}"; do
        col_num="${NUM_COL_INDEXES[$ni]}"
        for sc in "${_SELECTED_COLS[@]}"; do
            if [[ "$col_num" == "$sc" ]]; then
                _FILTERED_NUM_IDX+=("$col_num")
                _FILTERED_NUM_NAMES+=("${NUM_COL_NAMES[$ni]}")
                break
            fi
        done
    done
    NUM_COL_INDEXES=("${_FILTERED_NUM_IDX[@]}")
    NUM_COL_NAMES=("${_FILTERED_NUM_NAMES[@]}")
fi

########################################
## RUN ANALYSIS MODULES

corr_cols=()
corr_names=()
CORR_VALUES=()
_CORR_PAIR_LINES=""

# ENV-135: arm the shared progress helper just before the per-column
# analysis stage (the slowest part on wide CSVs). The helper's TTY +
# threshold gating decides whether anything actually renders; below
# 20 columns we skip ticking even if armed because the loop completes
# faster than a human can read a tick.
envoy_progress_init
_ENV135_TOTAL_COLS=$COL_COUNT
_ENV135_SHOW_PROGRESS=0
if envoy_progress_active && \
   { [[ "${ENVOY_PROGRESS_OPT_IN:-0}" == "1" ]] || [[ "$_ENV135_TOTAL_COLS" -ge 20 ]]; }; then
    _ENV135_SHOW_PROGRESS=1
fi

run_numeric_analyses
run_categorical_analyses
run_group_summaries
run_zero_variance
run_correlation_analyses
run_cramers_v
run_quality_score
run_suggested_drops
run_noisy_items
envoy_progress_done

########################################
## CROSS-DATASET COMPARISON

if [[ $compare_mode -eq 1 ]]; then
    run_cross_dataset_comparison
fi

########################################
## THRESHOLD CHECKS

if [[ ${#threshold_specs[@]} -gt 0 ]]; then
    run_threshold_checks
fi

########################################
## TIME-SERIES PROMOTION & ANALYSIS MODE

detect_and_promote_timeseries () {
    local is_ts=0
    if [[ $timeseries_mode -eq 1 ]]; then
        is_ts=1
    else
        # Auto-detect: look for monotonic index column
        local idx
        for ((idx=0; idx<${#ITEM_KINDS[@]}; idx++)); do
            if [[ "${ITEM_KINDS[$idx]}" == "monotonicity" ]]; then
                if [[ "${ITEM_BODIES[$idx]}" == "perfectly sorted (ascending)"* ]] || \
                   [[ "${ITEM_BODIES[$idx]}" == "perfectly sorted (descending)"* ]]; then
                    is_ts=1
                    break
                fi
            fi
        done
    fi
    if [[ $is_ts -eq 1 ]]; then
        for ((idx=0; idx<${#ITEM_KINDS[@]}; idx++)); do
            local kind="${ITEM_KINDS[$idx]}"
            if [[ "$kind" == "trend" || "$kind" == "monotonicity" ]]; then
                if [[ "${ITEM_TAGS[$idx]}" == "ADVANCED" ]]; then
                    ITEM_TAGS[$idx]="GOOD"
                fi
            fi
        done
    fi
}

apply_analysis_mode () {
    if [[ -z "$analysis_mode" || "$analysis_mode" == "explore" ]]; then
        return
    fi
    declare -A _promote_good=()
    declare -A _demote_advanced=()
    case "$analysis_mode" in
        benchmark)
            for k in cv compare compare_rank compare_summary anomaly monotonicity threshold threshold_summary; do
                _promote_good[$k]=1
            done
            for k in top_values dominance group_summary; do
                _demote_advanced[$k]=1
            done
            ;;
        monitor)
            for k in trend threshold threshold_summary anomaly monotonicity; do
                _promote_good[$k]=1
            done
            for k in histogram corr_matrix corr_pairs sparkline; do
                _demote_advanced[$k]=1
            done
            ;;
    esac
    local idx
    for ((idx=0; idx<${#ITEM_KINDS[@]}; idx++)); do
        local kind="${ITEM_KINDS[$idx]}"
        if [[ -n "${_promote_good[$kind]:-}" ]]; then
            ITEM_TAGS[$idx]="GOOD"
        elif [[ -n "${_demote_advanced[$kind]:-}" && "${ITEM_TAGS[$idx]}" == "GOOD" ]]; then
            ITEM_TAGS[$idx]="ADVANCED"
        fi
    done
}

detect_and_promote_timeseries
apply_analysis_mode

########################################
## SUMMARY / VERDICT

if [[ $emit_summary -eq 1 ]]; then
    run_summary_verdict
fi

########################################
## SUGGESTED LIST

suggested_ids=()
for ((idx=0; idx<${#ITEM_TITLES[@]}; idx++)); do
    if [[ "${ITEM_TAGS[$idx]}" != "GOOD" ]]; then
        continue
    fi
    case "${ITEM_KINDS[$idx]}" in
        overview|col_types|missing|summary|top_values|corr_matrix|corr_pairs|group_summary|compare|compare_rank|compare_summary|verdict|threshold_summary)
            suggested_ids+=("$((idx+1))")
            ;;
    esac
done

suggested_list=$(printf "%s " "${suggested_ids[@]}")
suggested_list=$(normalize_list "$suggested_list")

if [[ $suggest_only -eq 1 ]]; then
    echo "$suggested_list"
    exit 0
fi

# ENV-191: discoverable analysis-item menu — the list of analyses available for
# this dataset (id, tag, kind, title). A leading '*' marks suggested items.
if [[ $list_items_only -eq 1 ]]; then
    printf '%-4s %-3s %-9s %-16s %s\n' "ID" " * " "TAG" "KIND" "TITLE"
    printf '%-4s %-3s %-9s %-16s %s\n' "--" "---" "---" "----" "-----"
    _li_idx=0; _li_mark=""
    for ((_li_idx=0; _li_idx<${#ITEM_TITLES[@]}; _li_idx++)); do
        case " $suggested_list " in
            *" $((_li_idx+1)) "*) _li_mark=" * " ;;
            *) _li_mark="   " ;;
        esac
        printf '%-4s %-3s %-9s %-16s %s\n' \
            "$((_li_idx+1))" "$_li_mark" "${ITEM_TAGS[$_li_idx]}" "${ITEM_KINDS[$_li_idx]}" "${ITEM_TITLES[$_li_idx]}"
    done
    exit 0
fi

if [[ -n "$suggest_save_name" ]]; then
    save_profile "$suggest_save_name" "$profile_dir" "$suggested_list"
    if [[ -z "$analysis_list" && -z "$profile_name" ]]; then
        analysis_list="$suggested_list"
    fi
fi

########################################
## FILTER & SELECT

item_count=${#ITEM_TITLES[@]}

if [[ $only_good -eq 1 && -z "$kind_filter" && -z "$tag_filter" ]]; then
    selected_ids=()
    for ((idx=0; idx<item_count; idx++)); do
        if [[ "${ITEM_TAGS[$idx]}" == "GOOD" ]]; then
            selected_ids+=("$((idx+1))")
        fi
    done
    analysis_list=$(printf "%s " "${selected_ids[@]}")
    analysis_list=$(normalize_list "$analysis_list")
fi

if [[ $only_good -eq 1 && -z "$tag_filter" && -z "$kind_filter" ]]; then
    tag_filter="GOOD"
fi

if [[ -n "$analysis_list" ]]; then
    analysis_list=$(normalize_list "$analysis_list")
    IFS=' ' read -r -a selected <<< "$analysis_list"
else
    selected=()
    for ((i=1; i<=item_count; i++)); do
        selected+=("$i")
    done
fi

if [[ -n "$tag_filter" ]]; then
    tag_filter=$(normalize_list "$tag_filter")
    tag_filter=$(echo "$tag_filter" | tr '[:lower:]' '[:upper:]')
    IFS=' ' read -r -a TAG_FILTERS <<< "$tag_filter"
fi

if [[ -n "$kind_filter" ]]; then
    kind_filter=$(normalize_list "$kind_filter")
    kind_filter=$(echo "$kind_filter" | tr '[:upper:]' '[:lower:]')
    IFS=' ' read -r -a KIND_FILTERS <<< "$kind_filter"
fi

if [[ -n "$tag_filter" || -n "$kind_filter" ]]; then
    filtered=()
    for id in "${selected[@]}"; do
        if ! [[ "$id" =~ ^[0-9]+$ ]]; then
            continue
        fi
        if (( id < 1 || id > item_count )); then
            continue
        fi
        idx=$((id-1))
        tag="${ITEM_TAGS[$idx]}"
        kind="${ITEM_KINDS[$idx]}"
        if [[ -n "$tag_filter" ]] && ! list_contains "$tag" "${TAG_FILTERS[@]}"; then
            continue
        fi
        if [[ -n "$kind_filter" ]] && ! list_contains "$kind" "${KIND_FILTERS[@]}"; then
            continue
        fi
        filtered+=("$id")
    done
    selected=("${filtered[@]}")
fi

if [[ -n "$save_name" ]]; then
    [[ -n "$analysis_list" ]] || err "--save requires an analysis list"
    save_profile "$save_name" "$profile_dir" "$analysis_list"
    echo "Saved profile '$save_name' to $profile_dir" >&2
fi

########################################
## TOP-N FILTERING

if [[ -n "$top_n" ]]; then
    if [[ "$top_n" -eq 0 ]]; then
        selected=()
    elif [[ ${#selected[@]} -gt "$top_n" ]]; then
        # Score each selected item by significance, keep top N
        scored_lines=""
        for id in "${selected[@]}"; do
            idx=$((id - 1))
            score=0
            tag="${ITEM_TAGS[$idx]}"
            kind="${ITEM_KINDS[$idx]}"
            body="${ITEM_BODIES[$idx]}"

            # Base score from tag
            case "$tag" in
                GOOD)       score=100 ;;
                ADVANCED)   score=70  ;;
                NOISY)      score=40  ;;
                DIAGNOSTIC) score=20  ;;
            esac

            # Bonus for specific high-value kinds
            case "$kind" in
                corr_pairs|corr_matrix)   score=$((score + 30)) ;;
                anomaly)                  score=$((score + 25)) ;;
                threshold|threshold_summary) score=$((score + 25)) ;;
                trend|monotonicity)       score=$((score + 20)) ;;
                summary|verdict)          score=$((score + 20)) ;;
                quality)                  score=$((score + 15)) ;;
                compare|compare_rank)     score=$((score + 15)) ;;
                cv)                       score=$((score + 10)) ;;
                overview)                 score=$((score + 5))  ;;
            esac

            # Bonus for strong correlations (look for high absolute values)
            if [[ "$kind" == "corr_pairs" ]]; then
                if echo "$body" | grep -qE '0\.[89][0-9]|1\.00'; then
                    score=$((score + 20))
                elif echo "$body" | grep -qE '0\.7[0-9]'; then
                    score=$((score + 10))
                fi
            fi

            # Bonus for anomalies/outliers detected
            if [[ "$kind" == "anomaly" ]]; then
                if echo "$body" | grep -qE '[0-9]+ outlier'; then
                    score=$((score + 10))
                fi
            fi

            # Bonus for high skewness/kurtosis
            if [[ "$kind" == "shape" ]]; then
                if echo "$body" | grep -qiE 'heavy|extreme|skew'; then
                    score=$((score + 15))
                fi
            fi

            scored_lines+="${score} ${id}"$'\n'
        done

        # Sort by score descending, take top N
        top_ids=$(printf "%s" "$scored_lines" | sort -rn | awk -v n="$top_n" 'NR<=n{print $2}')
        selected=()
        while IFS= read -r id; do
            [[ -n "$id" ]] && selected+=("$id")
        done <<< "$top_ids"
    fi
fi

########################################
## OUTPUT

# ENV-143: only the plain rendering goes through the pager.  Structured
# formats (json/csv/tsv/markdown/html) bypass it so scripted consumers
# get clean bytes.
# ENV-152: when -o FILE is set, route the *structured* output to FILE
# (atomic rename via envoy_output_filter).  Pager bypass already happened
# in pre-flight.  When -o is unset, behaviour is unchanged from ENV-143.
if [[ -n "$output_file" ]]; then
    format_output "${selected[@]}" \
        | envoy_output_filter "$output_file" "$output_force" \
        || exit 1
elif [[ "$output_format" == "plain" ]]; then
    format_output "${selected[@]}" | envoy_maybe_page
else
    format_output "${selected[@]}"
fi

if [[ -z "$analysis_list" && $only_good -eq 0 && "$output_format" == "plain" ]]; then
    echo "Hint: re-run with a list like 'envoy-analyze $input_file 1,3,5' to select only the good stuff." >&2
    echo "Hint: save a profile with '--save NAME' to reuse selections." >&2
    if [[ -n "$suggested_list" ]]; then
        echo "Hint: suggested list: $suggested_list (use --suggest to print)" >&2
    fi
fi

# ENV-132: --strict fails the run if any numeric-classified column had
# non-numeric rows that were silently excluded from mean / std. Runs after
# all output so the user still sees the analysis above the failure line.
if [[ "${strict_numeric:-0}" == "1" ]]; then
    strict_violations=()
    for ((i=1; i<=COL_COUNT; i++)); do
        if [[ "${COL_TYPE[$((i-1))]:-}" == "numeric" ]] \
           && [[ "${COL_TEXT[$((i-1))]:-0}" -gt 0 ]]; then
            strict_violations+=("${COL_NAMES[$((i-1))]} (${COL_TEXT[$((i-1))]} non-numeric)")
        fi
    done
    if (( ${#strict_violations[@]} > 0 )); then
        echo "" >&2
        echo "envoy-analyze: error: --strict: numeric column(s) had non-numeric rows:" >&2
        for v in "${strict_violations[@]}"; do
            echo "  - $v" >&2
        done
        exit 1
    fi
fi

# Threshold exit code (must be after all output)
if [[ ${#threshold_specs[@]} -gt 0 ]]; then
    exit "$threshold_exit_code"
fi
