# llms-full (private-aware)
> Built from GitHub files and website pages. Large files may be truncated.

--- docs/oracle-actions-pack-2026-01-07.md ---
# Oraclepack Stage 3 — Action Pack (Taskify)

Generated: 2026-01-07

Parsed args (resolved):
- stage2_path: auto
- out_dir: auto
- pack_path: docs/oracle-actions-pack-2026-01-07.md
- actions_json: auto/_actions.json
- actions_md: auto/_actions.md
- prd_path: .taskmaster/docs/oracle-actions-prd.md
- tag: oraclepack
- mode: backlog
- top_n: 10
- oracle_cmd: oracle
- task_master_cmd: task-master
- tm_cmd: tm
- extra_files: (none)

This document must contain exactly one bash code fence, and no other code fences.

```bash
# 01) Preflight: resolve Stage-2 source, output dirs, and verify tools (fail fast)
set -euo pipefail

STAGE2_PATH="auto"
OUT_DIR="auto"
MODE="backlog"
TASK_MASTER_CMD="task-master"
ORACLE_CMD="oracle"
TM_CMD="tm"
ACTIONS_JSON="auto/_actions.json"
ACTIONS_MD="auto/_actions.md"
PRD_PATH=".taskmaster/docs/oracle-actions-prd.md"

require_cmd() {
  local cmd="$1"
  if ! command -v "${cmd%% *}" >/dev/null 2>&1; then
    echo "ERROR: required tool missing: ${cmd%% *} (from '${cmd}')" >&2
    exit 10
  fi
}

dir_form_ok() {
  local dir="$1"
  shopt -s nullglob
  for n in 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20; do
    local matches=( "${dir}/${n}-"*.md )
    if [ "${#matches[@]}" -ne 1 ]; then
      return 1
    fi
  done
  return 0
}

assert_dir_form() {
  local dir="$1"
  shopt -s nullglob
  for n in 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20; do
    local matches=( "${dir}/${n}-"*.md )
    if [ "${#matches[@]}" -eq 0 ]; then
      echo "ERROR: missing oracle output for prefix ${n}: expected ${dir}/${n}-*.md" >&2
      exit 3
    fi
    if [ "${#matches[@]}" -gt 1 ]; then
      echo "ERROR: multiple oracle outputs for prefix ${n}; expected exactly one file." >&2
      printf '%s\n' "${matches[@]}" >&2
      exit 4
    fi
  done
}

select_latest_dir_form() {
  local pattern="$1"
  local chosen=""
  mapfile -t dirs < <(ls -1d ${pattern} 2>/dev/null | sort)
  for ((i=${#dirs[@]}-1; i>=0; i--)); do
    local d="${dirs[i]}"
    if [ -d "${d}" ] && dir_form_ok "${d}"; then
      chosen="${d}"
      break
    fi
  done
  echo "${chosen}"
}

select_latest_file() {
  local pattern="$1"
  local chosen=""
  chosen=$(ls -1 ${pattern} 2>/dev/null | sort | tail -n 1 || true)
  echo "${chosen}"
}

stage2_resolved=""
stage2_type=""

if [ "${STAGE2_PATH}" != "auto" ]; then
  if [ -d "${STAGE2_PATH}" ]; then
    stage2_resolved="${STAGE2_PATH}"
    stage2_type="dir"
    assert_dir_form "${stage2_resolved}"
  elif [ -f "${STAGE2_PATH}" ]; then
    stage2_resolved="${STAGE2_PATH}"
    stage2_type="file"
  else
    echo "ERROR: stage2_path '${STAGE2_PATH}' does not exist (expected dir or file)." >&2
    exit 2
  fi
else
  # Directory form candidates (ordered)
  if [ -d "oracle-out" ] && dir_form_ok "oracle-out"; then
    stage2_resolved="oracle-out"
    stage2_type="dir"
  elif [ -d "docs/oracle-out" ] && dir_form_ok "docs/oracle-out"; then
    stage2_resolved="docs/oracle-out"
    stage2_type="dir"
  else
    candidate_dir=$(select_latest_dir_form "docs/oracle-questions-*/oracle-out/")
    if [ -n "${candidate_dir}" ]; then
      stage2_resolved="${candidate_dir}"
      stage2_type="dir"
    else
      candidate_dir=$(select_latest_dir_form "docs/oracle-questions-*/")
      if [ -n "${candidate_dir}" ]; then
        stage2_resolved="${candidate_dir}"
        stage2_type="dir"
      fi
    fi
  fi

  # Single-pack candidates (ordered) if directory form not found
  if [ -z "${stage2_resolved}" ]; then
    candidate_file=$(select_latest_file "docs/oracle-pack-*.md")
    if [ -n "${candidate_file}" ]; then
      stage2_resolved="${candidate_file}"
      stage2_type="file"
    else
      candidate_file=$(select_latest_file "docs/oraclepacks/oracle-pack-*.md")
      if [ -n "${candidate_file}" ]; then
        stage2_resolved="${candidate_file}"
        stage2_type="file"
      else
        candidate_file=$(select_latest_file "docs/oracle-questions-*/oracle-pack-*.md")
        if [ -n "${candidate_file}" ]; then
          stage2_resolved="${candidate_file}"
          stage2_type="file"
        else
          candidate_file=$(select_latest_file "docs/oracle-questions-*/oraclepacks/oracle-pack-*.md")
          if [ -n "${candidate_file}" ]; then
            stage2_resolved="${candidate_file}"
            stage2_type="file"
          fi
        fi
      fi
    fi
  fi
fi

if [ -z "${stage2_resolved}" ]; then
  echo "ERROR: Could not resolve Stage 2 outputs (stage2_path=auto)." >&2
  echo "Searched in order:" >&2
  echo "  - oracle-out/ (dir form)" >&2
  echo "  - docs/oracle-out/ (dir form)" >&2
  echo "  - newest docs/oracle-questions-*/oracle-out/ (dir form)" >&2
  echo "  - newest docs/oracle-questions-*/ (dir form)" >&2
  echo "  - newest docs/oracle-pack-*.md (single-pack)" >&2
  echo "  - newest docs/oraclepacks/oracle-pack-*.md (single-pack)" >&2
  echo "  - newest docs/oracle-questions-*/oracle-pack-*.md (single-pack)" >&2
  echo "  - newest docs/oracle-questions-*/oraclepacks/oracle-pack-*.md (single-pack)" >&2
  exit 2
fi

if [ "${stage2_type}" = "dir" ]; then
  assert_dir_form "${stage2_resolved}"
fi

# Resolve out_dir if auto
if [ "${OUT_DIR}" = "auto" ]; then
  if [ "${stage2_type}" = "dir" ]; then
    OUT_DIR="${stage2_resolved}"
  else
    if [[ "${stage2_resolved}" =~ ^docs/oracle-questions-[0-9]{4}-[0-9]{2}-[0-9]{2}/ ]]; then
      base="${BASH_REMATCH[0]%/}"
      OUT_DIR="${base}/oracle-out"
    else
      OUT_DIR="oracle-out"
    fi
  fi
fi

require_cmd "${TASK_MASTER_CMD}"
require_cmd "${ORACLE_CMD}"
if [ "${MODE}" = "autopilot" ]; then
  require_cmd "${TM_CMD}"
fi

# Re-base derived outputs if they were derived from auto
if [ "${ACTIONS_JSON}" = "auto/_actions.json" ]; then
  if [ -d "${OUT_DIR}" ]; then
    ACTIONS_JSON="${OUT_DIR}/_actions.json"
    ACTIONS_MD="${OUT_DIR}/_actions.md"
  else
    ACTIONS_JSON="docs/_actions.json"
    ACTIONS_MD="docs/_actions.md"
  fi
fi

mkdir -p "$(dirname "${ACTIONS_JSON}")" "$(dirname "${ACTIONS_MD}")" "$(dirname "${PRD_PATH}")" "docs"
if [ -d "${OUT_DIR}" ]; then mkdir -p "${OUT_DIR}"; fi

cat > "_actions.schema.md" <<'SCHEMA'
# Canonical Actions JSON Schema (human-readable)
[... schema details omitted for brevity, see asset ...]
SCHEMA
mv "_actions.schema.md" "docs/_actions.schema.md"

echo "OK: Stage 2 resolved: ${stage2_resolved} (${stage2_type})"
echo "OK: out_dir: ${OUT_DIR}"
echo "OK: Mode: ${MODE}"


# 02) Synthesize canonical actions JSON + summary MD
set -euo pipefail

STAGE2_PATH="auto"
OUT_DIR="auto"
ACTIONS_JSON="auto/_actions.json"
ACTIONS_MD="auto/_actions.md"

# Local helpers (self-contained)
dir_form_ok() {
  local dir="$1"
  shopt -s nullglob
  for n in 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20; do
    local matches=( "${dir}/${n}-"*.md )
    if [ "${#matches[@]}" -ne 1 ]; then
      return 1
    fi
  done
  return 0
}

select_latest_dir_form() {
  local pattern="$1"
  local chosen=""
  mapfile -t dirs < <(ls -1d ${pattern} 2>/dev/null | sort)
  for ((i=${#dirs[@]}-1; i>=0; i--)); do
    local d="${dirs[i]}"
    if [ -d "${d}" ] && dir_form_ok "${d}"; then
      chosen="${d}"
      break
    fi
  done
  echo "${chosen}"
}

select_latest_file() {
  local pattern="$1"
  local chosen=""
  chosen=$(ls -1 ${pattern} 2>/dev/null | sort | tail -n 1 || true)
  echo "${chosen}"
}

stage2_resolved=""
stage2_type=""

if [ "${STAGE2_PATH}" != "auto" ]; then
  if [ -d "${STAGE2_PATH}" ]; then
    stage2_resolved="${STAGE2_PATH}"
    stage2_type="dir"
  elif [ -f "${STAGE2_PATH}" ]; then
    stage2_resolved="${STAGE2_PATH}"
    stage2_type="file"
  else
    echo "ERROR: stage2_path '${STAGE2_PATH}' does not exist (expected dir or file)." >&2
    exit 2
  fi
else
  if [ -d "oracle-out" ] && dir_form_ok "oracle-out"; then
    stage2_resolved="oracle-out"
    stage2_type="dir"
  elif [ -d "docs/oracle-out" ] && dir_form_ok "docs/oracle-out"; then
    stage2_resolved="docs/oracle-out"
    stage2_type="dir"
  else
    candidate_dir=$(select_latest_dir_form "docs/oracle-questions-*/oracle-out/")
    if [ -n "${candidate_dir}" ]; then
      stage2_resolved="${candidate_dir}"
      stage2_type="dir"
    else
      candidate_dir=$(select_latest_dir_form "docs/oracle-questions-*/")
      if [ -n "${candidate_dir}" ]; then
        stage2_resolved="${candidate_dir}"
        stage2_type="dir"
      fi
    fi
  fi

  if [ -z "${stage2_resolved}" ]; then
    candidate_file=$(select_latest_file "docs/oracle-pack-*.md")
    if [ -n "${candidate_file}" ]; then
      stage2_resolved="${candidate_file}"
      stage2_type="file"
    else
      candidate_file=$(select_latest_file "docs/oraclepacks/oracle-pack-*.md")
      if [ -n "${candidate_file}" ]; then
        stage2_resolved="${candidate_file}"
        stage2_type="file"
      else
        candidate_file=$(select_latest_file "docs/oracle-questions-*/oracle-pack-*.md")
        if [ -n "${candidate_file}" ]; then
          stage2_resolved="${candidate_file}"
          stage2_type="file"
        else
          candidate_file=$(select_latest_file "docs/oracle-questions-*/oraclepacks/oracle-pack-*.md")
          if [ -n "${candidate_file}" ]; then
            stage2_resolved="${candidate_file}"
            stage2_type="file"
          fi
        fi
      fi
    fi
  fi
fi

if [ -z "${stage2_resolved}" ]; then
  echo "ERROR: Could not resolve Stage 2 outputs (stage2_path=auto)." >&2
  exit 2
fi

# Resolve out_dir if auto
if [ "${OUT_DIR}" = "auto" ]; then
  if [ "${stage2_type}" = "dir" ]; then
    OUT_DIR="${stage2_resolved}"
  else
    if [[ "${stage2_resolved}" =~ ^docs/oracle-questions-[0-9]{4}-[0-9]{2}-[0-9]{2}/ ]]; then
      base="${BASH_REMATCH[0]%/}"
      OUT_DIR="${base}/oracle-out"
    else
      OUT_DIR="oracle-out"
    fi
  fi
fi

# Re-base derived outputs if they were derived from auto
if [ "${ACTIONS_JSON}" = "auto/_actions.json" ]; then
  if [ -d "${OUT_DIR}" ]; then
    ACTIONS_JSON="${OUT_DIR}/_actions.json"
    ACTIONS_MD="${OUT_DIR}/_actions.md"
  else
    ACTIONS_JSON="docs/_actions.json"
    ACTIONS_MD="docs/_actions.md"
  fi
fi

oracle_file_flags=()
if [ "${stage2_type}" = "file" ]; then
  oracle_file_flags+=( -f "${stage2_resolved}" )
else
  shopt -s nullglob
  for n in 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20; do
    matches=( "${stage2_resolved}/${n}-"*.md )
    if [ "${#matches[@]}" -ne 1 ]; then
      echo "ERROR: expected exactly one match for ${stage2_resolved}/${n}-*.md, got ${#matches[@]}" >&2
      printf '%s\n' "${matches[@]:-}" >&2
      exit 20
    fi
    oracle_file_flags+=( -f "${matches[0]}" )
  done
fi

mkdir -p "$(dirname "${ACTIONS_JSON}")" "$(dirname "${ACTIONS_MD}")"

oracle \
  --write-output "${ACTIONS_JSON}" \
  "${oracle_file_flags[@]}" \
  -p "$(cat <<'PROMPT'
You are producing a SINGLE JSON document and nothing else.

Task: Normalize oraclepack answers into a canonical actionable plan.

Hard requirements:
- Output MUST be valid JSON (no markdown, no prose, no code fences).
- Output MUST follow the schema described below.
- Output MUST be deterministic in ordering:
  - items sorted by id ascending: 01..20
  - arrays use stable ordering (highest priority first; ties lexical)
- Extract only actionable work. Do not invent repo facts. If evidence is missing, record it in missing_artifacts/risk_notes explicitly.

Schema (summarized):
Root object:
- metadata: { generated_at, pack_date, source_out_dir, repo?, tooling?, top_n }
- items: array (max 20)
Each item:
- id: "01".."20"
- source_file: string
- category: string
- priority_score: number
- recommended_next_action: string (single imperative sentence)
- missing_artifacts: string[]
- acceptance_criteria: string[] (testable)
- risk_notes: string[]
- estimated_effort: "XS"|"S"|"M"|"L"|"XL"
- dependencies?: string[] of ids

Repo/run context:
- pack_date: 2026-01-07
- source_out_dir: auto
- top_n: 10
- tag: oraclepack

Output hygiene:
- Do not include backticks or fenced code blocks anywhere.
- Keep strings concise and specific.

Now produce the JSON.
PROMPT
)"

oracle \
  --write-output "${ACTIONS_MD}" \
  -f "${ACTIONS_JSON}" \
  -p "$(cat <<'PROMPT'
Write a human-readable Markdown summary of the canonical actions JSON.

Hard requirements:
- Output MUST be Markdown text with headings/bullets only (no code fences).
- Keep ordering aligned with items id ascending (01..20).
- Include:
  - short executive summary (5–10 bullets)
  - top 10 prioritized list (with id, title inferred from recommended_next_action, category, and why)
  - per-item: recommended_next_action + acceptance_criteria bullets + missing_artifacts bullets
- Do not invent facts; reflect only what is present in the JSON.

Now write the summary Markdown.
PROMPT
)"

echo "OK: Wrote ${ACTIONS_JSON}"
echo "OK: Wrote ${ACTIONS_MD}"


# 03) Generate PRD for Task Master
set -euo pipefail

ACTIONS_JSON="auto/_actions.json"
PRD_PATH=".taskmaster/docs/oracle-actions-prd.md"

# Re-resolve if actions_json path is generic
if [ "${ACTIONS_JSON}" = "auto/_actions.json" ]; then
  if [ -f "docs/_actions.json" ]; then ACTIONS_JSON="docs/_actions.json"; fi
  found=$(find docs -name "_actions.json" | head -n 1)
  if [ -n "${found}" ]; then ACTIONS_JSON="${found}"; fi
fi

mkdir -p "$(dirname "${PRD_PATH}")"

oracle \
  --write-output "${PRD_PATH}" \
  -f "${ACTIONS_JSON}" \
  -p "$(cat <<'PROMPT'
Write a Task Master-compatible PRD (Markdown) derived from the canonical actions JSON.

Hard requirements:
- Output MUST be Markdown (no code fences).
- Be dependency-aware (use dependencies if present; otherwise infer minimal dependencies cautiously).
- Prioritize focus: select the top N items by priority_score (N = TOP_N), but keep a traceability appendix mapping all ids 01..20.
- Every selected item must become an implementation-ready PRD section with:
  - Goal
  - Scope
  - Non-goals
  - Constraints
  - Acceptance criteria (testable)
  - Risks/unknowns
  - Dependencies (explicit)
- Use the tag value "oraclepack" in the PRD text where helpful for grouping.

Constants:
- TOP_N=10
- TAG=oraclepack

Now produce the PRD.
PROMPT
)"

echo "OK: Wrote ${PRD_PATH}"


# 04) Task Master: parse PRD into tasks
set -euo pipefail

PRD_PATH=".taskmaster/docs/oracle-actions-prd.md"
TASK_MASTER_CMD="task-master"
TAG="oraclepack"

if "${TASK_MASTER_CMD}" parse-prd "${PRD_PATH}" --tag "${TAG}" 2>/dev/null; then
  echo "OK: task-master parse-prd (tagged) succeeded."
else
  echo "INFO: task-master parse-prd did not accept --tag; retrying without tag."
  "${TASK_MASTER_CMD}" parse-prd "${PRD_PATH}"
fi

if [ -f ".taskmaster/tasks.json" ]; then
  echo "OK: Found .taskmaster/tasks.json"
elif [ -f "tasks.json" ]; then
  echo "OK: Found tasks.json"
else
  echo "WARN: tasks.json not found at .taskmaster/tasks.json or tasks.json. Check your Task Master configuration/output path."
fi


# 05) Task Master: analyze complexity and save report
set -euo pipefail

TASK_MASTER_CMD="task-master"
OUT_DIR="auto"

# Re-resolve if auto (light version for report output)
if [ "${OUT_DIR}" = "auto" ] || [ ! -e "${OUT_DIR}" ]; then
  OUT_DIR="docs"
fi
if [ -f "${OUT_DIR}" ]; then OUT_DIR="$(dirname "${OUT_DIR}")"; fi

mkdir -p "${OUT_DIR}"

"${TASK_MASTER_CMD}" analyze-complexity --output "${OUT_DIR}/tm-complexity.json"
echo "OK: Wrote ${OUT_DIR}/tm-complexity.json"


# 06) Task Master: expand tasks
set -euo pipefail

TASK_MASTER_CMD="task-master"

"${TASK_MASTER_CMD}" expand --all
echo "OK: Expanded tasks."


# 07) Pipelines (pipelines mode only): generate deterministic pipelines from tasks.json
set -euo pipefail

MODE="backlog"

if [ "${MODE}" != "pipelines" ]; then
  echo "SKIP: mode=${MODE} (pipelines step runs only when mode=pipelines)."
else
  tasks_path=""
  if [ -f ".taskmaster/tasks.json" ]; then
    tasks_path=".taskmaster/tasks.json"
  elif [ -f "tasks.json" ]; then
    tasks_path="tasks.json"
  else
    echo "ERROR: tasks.json not found; cannot generate pipelines." >&2
    exit 70
  fi

  mkdir -p "docs"

  oracle \
    --write-output "docs/oracle-actions-pipelines.md" \
    -f "${tasks_path}" \
    -p "$(cat <<'PROMPT'
Generate deterministic command pipelines from tasks.json.

Hard requirements:
- Output MUST be Markdown (no code fences).
- Include 3–6 pipelines, each a numbered list of shell commands.
- Each pipeline must be tests-first and avoid destructive operations.
- Commands should be generic and repo-agnostic (no invented scripts).
- Include a short “resume strategy” section explaining how to re-run pipelines safely.

Now write docs/oracle-actions-pipelines.md content.
PROMPT
)"

  echo "OK: Wrote docs/oracle-actions-pipelines.md"
fi


# 08) Autopilot (autopilot mode only): branch safety + guarded entrypoint
set -euo pipefail

MODE="backlog"
TM_CMD="tm"
OUT_DIR="auto"
PACK_DATE="2026-01-07"
TAG="oraclepack"

if [ "${MODE}" != "autopilot" ]; then
  echo "SKIP: mode=${MODE} (autopilot step runs only when mode=autopilot)."
else
  if ! command -v git >/dev/null 2>&1; then
    echo "ERROR: autopilot mode requires git on PATH." >&2
    exit 80
  fi

  if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
    echo "ERROR: not inside a git work tree; autopilot mode requires a git repo." >&2
    exit 81
  fi

  if ! git diff --quiet || ! git diff --cached --quiet; then
    echo "ERROR: working tree not clean. Commit/stash before autopilot." >&2
    exit 82
  fi

  current_branch="$(git rev-parse --abbrev-ref HEAD)"
  if [ "${current_branch}" = "main" ] || [ "${current_branch}" = "master" ]; then
    new_branch="oraclepack/${PACK_DATE}-${TAG}"
    echo "INFO: on default-like branch (${current_branch}); creating work branch: ${new_branch}"
    git checkout -b "${new_branch}"
  else
    echo "OK: current branch is ${current_branch}"
  fi

  # Ensure dir exists (re-resolve safe fallback)
  if [ "${OUT_DIR}" = "auto" ] || [ ! -d "${OUT_DIR}" ]; then OUT_DIR="docs"; fi
  mkdir -p "${OUT_DIR}"

  cat > "${OUT_DIR}/tm-autopilot.state.json" <<STATE
{"pack_date":"${PACK_DATE}","tag":"${TAG}","mode":"autopilot","notes":"State file created by Stage-3 Action Pack. Autopilot tooling should resume from this file if supported."}
STATE

  echo "OK: Wrote ${OUT_DIR}/tm-autopilot.state.json"
  echo "INFO: Starting autopilot via: ${TM_CMD} autopilot"
  echo "INFO: If your tm tool uses a different subcommand, edit this step accordingly."

  if ! "${TM_CMD}" --help 2>&1 | grep -qi "autopilot"; then
    echo "ERROR: '${TM_CMD}' does not advertise 'autopilot' in --help output." >&2
    echo "HINT: rerun Stage 3 with mode=backlog if you only want tasks generated." >&2
    exit 83
  fi

  "${TM_CMD}" autopilot
fi
```


--- docs/oracle-pack-2026-01-07.md ---
# Oracle Pack — Unknown (Gold Stage 1)

## Parsed args

- codebase_name: Unknown
- constraints: None
- non_goals: None
- team_size: Unknown
- deadline: Unknown
- out_dir: docs/oracle-questions-2026-01-07
- oracle_cmd: oracle
- oracle_flags: --files-report
- engine: None
- model: None
- extra_files: None

Notes:

- Template is the **contract**. Keep the pack runner-ingestible.
- Exactly one fenced `bash` block in this whole document.
- Exactly 20 steps, numbered `01..20`.
- Each step includes: `ROI= impact= confidence= effort= horizon= category= reference=`
- Categories must be exactly the fixed set used in Coverage check.

## Commands

```bash
# Optional preflight pattern:
# - Add `--dry-run summary` to preview what will be sent, and keep `--files-report` enabled when available.

# 01) ROI=9 impact=high confidence=med effort=low horizon=Immediate category=contracts/interfaces reference=promptbench/cli.py
oracle \
  --files-report \
  --file "promptbench/cli.py" \
  --file "README.md" \
  --write-output "docs/oracle-questions-2026-01-07/01-contracts-interfaces-surface.md" \
  -p "$(cat <<'PROMPT'
Strategist question #01

Reference: promptbench/cli.py
Category: contracts/interfaces
Horizon: Immediate
ROI: 9 (impact=high, confidence=med, effort=low)

Question:
Identify the primary public interface(s) of this system (API endpoints, CLI commands, public SDK surface, event contracts). For each, list the key request/response (or input/output) shapes and where they are defined in the code.

Rationale (one sentence):
We need a trustworthy map of the system’s “outside-facing contract” before deeper planning.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–6 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 02) ROI=8 impact=high confidence=med effort=med horizon=Immediate category=contracts/interfaces reference=promptbench/providers/lmstudio.py
oracle \
  --files-report \
  --file "promptbench/core/config.py" \
  --file "promptbench/providers/lmstudio.py" \
  --write-output "docs/oracle-questions-2026-01-07/02-contracts-interfaces-integration.md" \
  -p "$(cat <<'PROMPT'
Strategist question #02

Reference: promptbench/providers/lmstudio.py
Category: contracts/interfaces
Horizon: Immediate
ROI: 8 (impact=high, confidence=med, effort=med)

Question:
What are the top integration points with external systems (databases, queues, third-party APIs, SSO, storage), and what contract(s) or config declare them? Provide the minimal list of files/locations that define each integration.

Rationale (one sentence):
Integration boundaries drive risk, deployment needs, and test strategy.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–6 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 03) ROI=7 impact=med confidence=med effort=low horizon=NearTerm category=invariants reference=promptbench/core/contract.py
oracle \
  --files-report \
  --file "promptbench/core/contract.py" \
  --file "promptbench/core/types.py" \
  --write-output "docs/oracle-questions-2026-01-07/03-invariants-domain.md" \
  -p "$(cat <<'PROMPT'
Strategist question #03

Reference: promptbench/core/contract.py
Category: invariants
Horizon: NearTerm
ROI: 7 (impact=med, confidence=med, effort=low)

Question:
List the system’s most important invariants (business rules, correctness properties, “must always be true” conditions). For each, show where it is enforced (or where it should be enforced but currently is not).

Rationale (one sentence):
Invariants define correctness and are the backbone of reliable changes.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 04) ROI=7 impact=med confidence=med effort=med horizon=NearTerm category=invariants reference=promptbench/core/validation.py
oracle \
  --files-report \
  --file "promptbench/core/validation.py" \
  --file "promptbench/core/config.py" \
  --write-output "docs/oracle-questions-2026-01-07/04-invariants-validation.md" \
  -p "$(cat <<'PROMPT'
Strategist question #04

Reference: promptbench/core/validation.py
Category: invariants
Horizon: NearTerm
ROI: 7 (impact=med, confidence=med, effort=med)

Question:
Where does validation happen (input validation, schema validation, domain validation)? Identify the validation boundaries and the most likely gaps that could cause inconsistent state.

Rationale (one sentence):
Knowing validation boundaries prevents regressions and reduces security/correctness risk.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 05) ROI=6 impact=med confidence=med effort=med horizon=NearTerm category=caching/state reference=promptbench/core/discovery_cache.py
oracle \
  --files-report \
  --file "promptbench/core/discovery_cache.py" \
  --file "promptbench/core/workspace.py" \
  --write-output "docs/oracle-questions-2026-01-07/05-caching-state-layers.md" \
  -p "$(cat <<'PROMPT'
Strategist question #05

Reference: promptbench/core/discovery_cache.py
Category: caching/state
Horizon: NearTerm
ROI: 6 (impact=med, confidence=med, effort=med)

Question:
What stateful components exist (in-memory state, caches, sessions, client-side state, persisted state)? For each, describe lifecycle, invalidation/expiry, and where it is implemented.

Rationale (one sentence):
State and caching are common sources of subtle bugs and performance issues.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 06) ROI=6 impact=med confidence=med effort=med horizon=NearTerm category=caching/state reference=promptbench/core/discovery.py
oracle \
  --files-report \
  --file "promptbench/core/discovery.py" \
  --file "promptbench/core/discovery_cache.py" \
  --write-output "docs/oracle-questions-2026-01-07/06-caching-state-consistency.md" \
  -p "$(cat <<'PROMPT'
Strategist question #06

Reference: promptbench/core/discovery.py
Category: caching/state
Horizon: NearTerm
ROI: 6 (impact=med, confidence=med, effort=med)

Question:
Identify the top consistency risks between caches/state layers and the source of truth (stale reads, write skew, missing invalidation). Where are the knobs/configs for cache behavior?

Rationale (one sentence):
Consistency failure modes often surface as “random bugs” and are expensive to debug.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 07) ROI=5 impact=med confidence=low effort=med horizon=NearTerm category=background jobs reference=promptbench/runner/executor.py
oracle \
  --files-report \
  --file "promptbench/runner/executor.py" \
  --file "promptbench/runner/eventing_executor.py" \
  --write-output "docs/oracle-questions-2026-01-07/07-background-jobs-discovery.md" \
  -p "$(cat <<'PROMPT'
Strategist question #07

Reference: promptbench/runner/executor.py
Category: background jobs
Horizon: NearTerm
ROI: 5 (impact=med, confidence=low, effort=med)

Question:
What background jobs/workers/scheduled tasks exist? For each, identify trigger mechanism, payload, retries, idempotency, and where it is defined.

Rationale (one sentence):
Background work affects reliability, cost, and operational complexity.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 08) ROI=5 impact=med confidence=low effort=med horizon=NearTerm category=background jobs reference=promptbench/runner/executor.py
oracle \
  --files-report \
  --file "promptbench/runner/executor.py" \
  --file "promptbench/providers/base.py" \
  --write-output "docs/oracle-questions-2026-01-07/08-background-jobs-reliability.md" \
  -p "$(cat <<'PROMPT'
Strategist question #08

Reference: promptbench/runner/executor.py
Category: background jobs
Horizon: NearTerm
ROI: 5 (impact=med, confidence=low, effort=med)

Question:
Where are the main reliability controls for background work (dead-lettering, backoff, concurrency limits, reprocessing), and what is missing or inconsistent?

Rationale (one sentence):
Reliability controls prevent incident loops and data corruption.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 09) ROI=8 impact=high confidence=med effort=low horizon=Immediate category=observability reference=promptbench/core/events.py
oracle \
  --files-report \
  --file "promptbench/core/events.py" \
  --file "promptbench/runner/eventing_executor.py" \
  --write-output "docs/oracle-questions-2026-01-07/09-observability-signals.md" \
  -p "$(cat <<'PROMPT'
Strategist question #09

Reference: promptbench/core/events.py
Category: observability
Horizon: Immediate
ROI: 8 (impact=high, confidence=med, effort=low)

Question:
What observability signals exist (logs/metrics/traces/events), and what are the primary identifiers for correlating a request/job across components? Point to the code/config that defines them.

Rationale (one sentence):
You can’t operate or improve what you can’t measure or debug quickly.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 10) ROI=7 impact=high confidence=med effort=med horizon=Immediate category=observability reference=promptbench/runner/summary.py
oracle \
  --files-report \
  --file "promptbench/runner/summary.py" \
  --file "promptbench/core/events.py" \
  --write-output "docs/oracle-questions-2026-01-07/10-observability-gaps.md" \
  -p "$(cat <<'PROMPT'
Strategist question #10

Reference: promptbench/runner/summary.py
Category: observability
Horizon: Immediate
ROI: 7 (impact=high, confidence=med, effort=med)

Question:
Where are the biggest observability gaps (missing logs around key decisions, missing metrics for SLOs, missing trace spans)? Recommend the smallest additions that would most improve debugging.

Rationale (one sentence):
Targeted observability improvements compound across all future changes.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 11) ROI=6 impact=med confidence=med effort=med horizon=Immediate category=permissions reference=promptbench/core/discovery.py
oracle \
  --files-report \
  --file "promptbench/core/discovery.py" \
  --file "promptbench/cli.py" \
  --write-output "docs/oracle-questions-2026-01-07/11-permissions-model.md" \
  -p "$(cat <<'PROMPT'
Strategist question #11

Reference: promptbench/core/discovery.py
Category: permissions
Horizon: Immediate
ROI: 6 (impact=med, confidence=med, effort=med)

Question:
What is the permission model (roles/scopes/claims/ACLs), and where is it defined? Provide the minimal set of files that encode “who can do what.”

Rationale (one sentence):
Permission rules are a high-risk area with security and product impact.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 12) ROI=6 impact=med confidence=med effort=med horizon=Immediate category=permissions reference=promptbench/core/workspace.py
oracle \
  --files-report \
  --file "promptbench/core/workspace.py" \
  --file "promptbench/core/validation.py" \
  --write-output "docs/oracle-questions-2026-01-07/12-permissions-enforcement.md" \
  -p "$(cat <<'PROMPT'
Strategist question #12

Reference: promptbench/core/workspace.py
Category: permissions
Horizon: Immediate
ROI: 6 (impact=med, confidence=med, effort=med)

Question:
Where are permissions enforced (middleware/guards/policies/service-layer checks), and where are likely bypass risks? Identify the enforcement chokepoints and any inconsistent patterns.

Rationale (one sentence):
Enforcement consistency prevents privilege escalation and policy drift.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 13) ROI=4 impact=med confidence=low effort=med horizon=NearTerm category=migrations reference=Unknown
oracle \
  --files-report \
  --write-output "docs/oracle-questions-2026-01-07/13-migrations-schema.md" \
  -p "$(cat <<'PROMPT'
Strategist question #13

Reference: Unknown
Category: migrations
Horizon: NearTerm
ROI: 4 (impact=med, confidence=low, effort=med)

Question:
How are schema/config migrations handled (DB migrations, data backfills, versioned configs)? Identify the tooling, directories, and how migrations are applied in CI/deploy.

Rationale (one sentence):
Migration mechanics are critical for safe releases and rollbacks.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 14) ROI=4 impact=med confidence=low effort=med horizon=NearTerm category=migrations reference=Unknown
oracle \
  --files-report \
  --write-output "docs/oracle-questions-2026-01-07/14-migrations-compat.md" \
  -p "$(cat <<'PROMPT'
Strategist question #14

Reference: Unknown
Category: migrations
Horizon: NearTerm
ROI: 4 (impact=med, confidence=low, effort=med)

Question:
What are the backward/forward compatibility expectations during migrations (rolling deploys, dual-read/dual-write, feature-flagged schema use)? Identify where compatibility is ensured or currently risky.

Rationale (one sentence):
Compatibility strategy prevents outages during deployments.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–8 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 15) ROI=7 impact=high confidence=med effort=med horizon=NearTerm category=UX flows reference=promptbench/cli.py
oracle \
  --files-report \
  --file "promptbench/cli.py" \
  --file "README.md" \
  --write-output "docs/oracle-questions-2026-01-07/15-ux-flows-primary.md" \
  -p "$(cat <<'PROMPT'
Strategist question #15

Reference: promptbench/cli.py
Category: UX flows
Horizon: NearTerm
ROI: 7 (impact=high, confidence=med, effort=med)

Question:
What are the primary user flows (or primary operator workflows) and their steps? Map each to the main components/modules involved, and note the key state transitions.

Rationale (one sentence):
Flow maps reveal critical paths and help prioritize work with user impact.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–10 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 16) ROI=7 impact=high confidence=med effort=med horizon=NearTerm category=UX flows reference=promptbench/core/validation.py
oracle \
  --files-report \
  --file "promptbench/core/validation.py" \
  --file "promptbench/cli.py" \
  --write-output "docs/oracle-questions-2026-01-07/16-ux-flows-edgecases.md" \
  -p "$(cat <<'PROMPT'
Strategist question #16

Reference: promptbench/core/validation.py
Category: UX flows
Horizon: NearTerm
ROI: 7 (impact=high, confidence=med, effort=med)

Question:
For the primary flows, what are the top edge cases and “gotchas” (validation failures, partial completion, retries, timeouts)? Identify where these cases are handled and where they are missing.

Rationale (one sentence):
Edge-case handling is where many UX and reliability issues originate.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–10 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 17) ROI=8 impact=high confidence=med effort=med horizon=Immediate category=failure modes reference=promptbench/runner/executor.py
oracle \
  --files-report \
  --file "promptbench/runner/executor.py" \
  --file "promptbench/providers/lmstudio.py" \
  --write-output "docs/oracle-questions-2026-01-07/17-failure-modes-taxonomy.md" \
  -p "$(cat <<'PROMPT'
Strategist question #17

Reference: promptbench/runner/executor.py
Category: failure modes
Horizon: Immediate
ROI: 8 (impact=high, confidence=med, effort=med)

Question:
What is the failure-mode taxonomy of this system (timeouts, retries, partial failures, inconsistent state, dependency failures)? Identify where failures are classified/handled and what is surfaced to users/operators.

Rationale (one sentence):
Explicit failure handling prevents incidents and reduces user-facing errors.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–10 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 18) ROI=7 impact=high confidence=med effort=med horizon=Immediate category=failure modes reference=promptbench/runner/executor.py
oracle \
  --files-report \
  --file "promptbench/runner/executor.py" \
  --file "promptbench/providers/gemini_cli.py" \
  --write-output "docs/oracle-questions-2026-01-07/18-failure-modes-resilience.md" \
  -p "$(cat <<'PROMPT'
Strategist question #18

Reference: promptbench/runner/executor.py
Category: failure modes
Horizon: Immediate
ROI: 7 (impact=high, confidence=med, effort=med)

Question:
What resilience mechanisms exist (circuit breakers, bulkheads, retries with jitter, rate limiting, graceful degradation)? Where are they configured, and which critical path lacks them?

Rationale (one sentence):
Resilience patterns determine real-world reliability under stress.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–10 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 19) ROI=4 impact=med confidence=low effort=low horizon=NearTerm category=feature flags reference=Unknown
oracle \
  --files-report \
  --write-output "docs/oracle-questions-2026-01-07/19-feature-flags-inventory.md" \
  -p "$(cat <<'PROMPT'
Strategist question #19

Reference: Unknown
Category: feature flags
Horizon: NearTerm
ROI: 4 (impact=med, confidence=low, effort=low)

Question:
What feature-flag system exists (or how are conditional rollouts handled)? Inventory the flags (or equivalents) and identify where flags are defined, evaluated, and documented.

Rationale (one sentence):
Flags enable safe rollout and experimentation and reduce release risk.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–10 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"

# 20) ROI=4 impact=med confidence=low effort=low horizon=NearTerm category=feature flags reference=Unknown
oracle \
  --files-report \
  --write-output "docs/oracle-questions-2026-01-07/20-feature-flags-rollout.md" \
  -p "$(cat <<'PROMPT'
Strategist question #20

Reference: Unknown
Category: feature flags
Horizon: NearTerm
ROI: 4 (impact=med, confidence=low, effort=low)

Question:
Describe the flag/rollout lifecycle (how flags are created, tested, ramped, monitored, and retired). Identify the minimum guardrails needed to prevent “flag debt.”

Rationale (one sentence):
A disciplined rollout lifecycle reduces long-term complexity and operational risk.

Constraints: None
Non-goals: None

Answer format:
1) Direct answer (1–10 bullets, evidence-cited)
2) Risks/unknowns (bullets)
3) Next smallest concrete experiment (exactly one action)
4) If evidence is insufficient, name the exact missing file/path pattern(s) to attach next.
PROMPT
)"
```

## Coverage check

- contracts/interfaces: OK
- invariants: OK
- caching/state: OK
- background jobs: OK
- observability: OK
- permissions: OK
- migrations: OK
- UX flows: OK
- failure modes: OK
- feature flags: OK


--- docs/oracle-questions-2026-01-07/01-contracts-interfaces-surface.md ---
1) Direct answer (primary public interfaces + shapes + where defined)

Primary user-facing interface: promptbench CLI (default “run benchmark” command)

Invocation shape: promptbench --config <path> [--output <dir>] [--limit N] [--concurrency N] [--providers csv] [--templates csv] [--payloads csv] [--skills csv] [--event-log <file|->] [--force-refresh] (defined in promptbench/cli.py:_build_run_parser() and executed by promptbench/cli.py:_run_benchmark()).

I/O shape: reads TOML config via load_config(args.config) + apply_overrides(...); discovers template/payload/skill file lists; produces per-job artifacts and a summary.json under config.output_root (written by promptbench/runner/summary.py:write_summary via promptbench/cli.py step “Generate summary”).

Where “job matrix” contract is defined: promptbench/runner/matrix.py:build_jobs(...) called with (templates, payloads, skills, config.providers, provider_filter, limit, base_dir) from promptbench/cli.py:_run_benchmark().

Machine-readable “preflight” interface: promptbench discover (JSON to stdout)

Invocation shape: promptbench discover --config <path> [--force-refresh] (defined in promptbench/cli.py:_discover(), dispatched in promptbench/cli.py:main()).

Response shape: prints a single JSON object with keys:
templates (count), payloads (count), skills (count), providers (count), jobs (computed cartesian size), output_root (string), workspace (diagnostics payload) — assembled in promptbench/cli.py:_discover() using diagnostics_payload(config_dir).

Machine-readable “selection data” interface for UI clients: promptbench matrix (JSON to stdout)

Invocation shape: promptbench matrix --config <path> [--force-refresh] (defined in promptbench/cli.py:_matrix(), dispatched in promptbench/cli.py:main()).

Response shape: prints a single JSON object containing:
templates (list of paths), payloads (list), skills (list), template_sets/payload_sets/skill_sets (list of set descriptors via discover_sets(...).__dict__), providers (list of {id,type,model,url}), output_root (string), jobs (computed count). This is built in promptbench/cli.py:_matrix() and is the clearest “public contract” for the TUI/other tools.

Event contract interface: JSONL event stream (--event-log or executor eventing)

Enabling shape: promptbench ... --event-log <file|-> or config-driven use_eventing_executor (wired in promptbench/cli.py:_run_benchmark() via use_eventing = config.use_eventing_executor or bool(args.event_log)).

Event record shape: RunEvent(ts, type, run_id, job_id, provider_id, payload) emitted via emit_event(event_sink, RunEvent(...)) in promptbench/cli.py:_emit_run_event(), with example lifecycle events EventType.CONFIG_LOADED and EventType.DISCOVERY_COMPLETE emitted in _run_benchmark(). The canonical schema/types live in promptbench/core/events.py (imported in promptbench/cli.py).

Workspace scaffolding contract: promptbench init (filesystem layout + sample files)

Invocation shape: promptbench init [--path <dir>] (promptbench/cli.py:_init_workspace()).

Output shape: creates workspace directories via ensure_workspace_structure(base_dir) + ensure_set_dirs(base_dir, ["templates","payloads","skills"]), then writes sample files like templates/example.md, payloads/example.md, skills/example.md, and default sets sets/<category>/default.txt (all in promptbench/cli.py:_init_workspace()).

Validation/diagnostics contract: promptbench doctor

Invocation shape: promptbench doctor [--path <dir> | --config <toml>] [--json] (promptbench/cli.py:_doctor()).

Response shape: either JSON list [{...diag fields...}] (from validate_workspace(base_dir) results) or human-readable LEVEL: message (hint: ...) lines; defined in promptbench/cli.py:_doctor() and promptbench/core/validation.py (imported).

2) Risks/unknowns

Config schema is reference


--- docs/oracle-questions-2026-01-07/02-contracts-interfaces-integration.md ---
1) Direct answer (1–6 bullets, evidence-cited)

LM Studio (HTTP inference API; OpenAI-compatible Chat Completions)

Integration: POST {base}/v1/chat/completions via requests.post(...) with JSON payload {model, messages, temperature, ttl}. Fallback adds a "user": "." message when LM Studio returns 400.

Contract surface: OpenAI-style chat response shape is assumed: data["choices"][0]["message"]["content"], plus optional data["usage"] and data["model"].

Defined in: promptbench/providers/lmstudio.py (LMStudioAdapter.run, _get_bases).

Declared/configured by: ProviderSpec.url + ProviderSpec.model (from TOML providers.<id>.url / providers.<id>.model) parsed in promptbench/core/config.py (load_config builds ProviderSpec(... url=..., model=...)).

LM Studio (HTTP management API; model load/unload)

Integration: POST {base}/api/v1/models/load, POST {base}/api/v1/models/unload, and fallback DELETE {base}/api/v1/models/{model_id}.

Contract surface: expects JSON response with {"status": "loaded"} for load; unload success inferred from HTTP 200.

Defined in: promptbench/providers/lmstudio.py (load_model, unload_model, _get_bases).

Declared/configured by: ProviderSpec.url and ProviderSpec.model from promptbench/core/config.py as above.

Environment-variable based external configuration (provider env passthrough)

Integration point: provider definitions support an env map in config (implying downstream use to configure external systems), though the actual wiring/usage isn’t shown in the provided files.

Defined/declared in: promptbench/core/config.py (ProviderSpec(... env=p_data.get("env", {}))).

Local filesystem as storage boundary (inputs + outputs)

Integration point: reads a TOML config from disk; uses glob paths for templates/payloads; writes outputs under an output_root (default "runs"). This is a deployment/storage boundary even if not “external SaaS.”

Defined/declared in: promptbench/core/config.py (load_config reads file, inputs.templates, inputs.payloads, output.root, and apply_overrides can change output_root).

2) Risks/unknowns (bullets)

Only the LM Studio provider is evidenced here; other providers may integrate with cloud APIs (OpenAI, Anthropic, Gemini, etc.), queues, DBs, or storage, but those files were not provided.

The provider env contract is parsed but not shown being applied to process environments or HTTP headers; risk of “documented but unused” config.

The LM Studio adapter assumes an OpenAI-like response schema (choices[0].message.content); if LM Studio changes/varies, failures will occur without schema validation.

No evidence yet of SSO/auth, databases, queues, or object storage integrations—may be absent or simply outside the provided excerpts.

3) Next smallest concrete experiment (exactly one action)

Run a repo-wide scan for external integration libraries and endpoints to enumerate all boundaries in one pass:

bash
Copy code
rg -n --hidden --glob '!.git' \
  "import (requests|httpx|aiohttp|boto3|google|azure|sqlite3|psycopg|sqlalchemy|redis|pika|kafka|jwt|oauth)|https?://|/v1/|/api/" \
  promptbench

4) If evidence is insufficient, exact missing file/path pattern(s) to attach next

promptbench/providers/*.py (all providers, not just lmstudio.py)

promptbench/providers/base.py

promptbench/core/types.py (definitions of ProviderSpec, Config, ProviderResponse)

promptbench/core/discovery.py (input discovery boundaries; filesystem usage details)

promptbench/core/events*.py and any executor files (e.g., promptbench/core/executor*.py, promptbench/core/eventing*.py) to confirm whether queues/event buses/external sinks exist


--- docs/oracle-questions-2026-01-07/03-invariants-domain.md ---
1) Direct answer (most important invariants, with enforcement locations)

Provider adapters must return a ProviderResponse instance (never a dict/string/etc.) — enforced in promptbench/core/contract.py::validate_response() via isinstance(response, ProviderResponse) and ContractViolationError.
Evidence: promptbench/core/contract.py::validate_response, promptbench/core/types.py::ProviderResponse.

ProviderResponse.meta must always be a dict (never None, list, string, etc.) — enforced in validate_response() by normalizing None to {} and then type-checking dict.
Evidence: promptbench/core/contract.py::validate_response.

ProviderResponse.meta must include provider identity (provider_id, provider_type) and it must match the executing ProviderSpec — partially enforced: validate_response() populates defaults with setdefault, but does not reject mismatches if an adapter pre-fills conflicting values. This is an invariant that should be enforced by checking equality (or overwriting unconditionally).
Evidence: promptbench/core/contract.py::validate_response uses response.meta.setdefault("provider_id", spec.id) / setdefault("provider_type", spec.type); promptbench/core/types.py::ProviderSpec.

If raw_json is provided, it must be a dict — enforced in validate_response() via isinstance(response.raw_json, dict) when non-None.
Evidence: promptbench/core/contract.py::validate_response, promptbench/core/types.py::ProviderResponse.raw_json.

If error is provided, it must be a dict and must include non-empty string code and message — enforced in validate_response() via dict type-check + _require_non_empty_string() on error.code and error.message.
Evidence: promptbench/core/contract.py::_require_non_empty_string, promptbench/core/contract.py::validate_response, promptbench/core/types.py::ProviderResponse.error.

Success/Failure exclusivity: “success” means error is None and “failure” means error is not None — implicitly relied on in validate_response() branching, but not fully enforced as an invariant of the whole object:

On success branch, it enforces text is a non-empty string.

On error branch, it does not enforce that text is empty/ignored, nor even that it is a string.
This is an invariant that should be enforced (either require text to be "" on error, or require it to be a string always, plus document precedence).
Evidence: promptbench/core/contract.py::validate_response branches on if response.error is not None: ... else: ...; promptbench/core/types.py::ProviderResponse.text.

Provider/config identifiers should be well-formed (e.g., ProviderSpec.id and .type non-empty strings) — not enforced in shown code, but correctness depends on it because validate_response() injects these values into meta. This should be enforced in config loading/validation (likely promptbench/core/config.py) or in a dedicated validate_provider_spec() contract.
Evidence: promptbench/core/types.py::ProviderSpec fields are plain str; promptbench/core/contract.py::validate_response assumes spec.id/spec.type are meaningful.

Run result consistency: RunResult.status should align with artifact_paths and error (typical invariant: OK => artifact_paths != None and error == None, ERROR => error != None) — declared by shape in promptbench/core/types.py, but no enforcement is shown here; it should be enforced at the point where RunResult is constructed (runner/executor layer).
Evidence: promptbench/core/types.py::RunResult, promptbench/core/types.py::RunStatus, promptbench/core/types.py::ArtifactPaths.

2) Risks/unknowns

Unknown whether every provider adapter (or the runner) actually calls validate_response(); if it’s optional, these invariants can be bypassed.

meta identity mismatch risk: because setdefault() does not prevent conflicts, an adapter can accidentally/incorrectly stamp the wrong provider id/type and pass validation.

Error-mode ambiguity: if adapters return both error and non-empty text, downstream behavior may become inconsistent (UI/reporting might treat it as success text).

Config/job invariants (provider id uniqueness, job provider_id resolution, output_root existence, concurrency/timeout sanity) likely exist but aren’t visible from the provided files.

3) Next smallest concrete experiment (exactly one action)

Run a repo-wide search to verify where validation is applied and what bypass paths exist:

bash
Copy code
rg -n "validate_response\(|ContractViolationError_


--- docs/oracle-questions-2026-01-07/04-invariants-validation.md ---
1) Direct answer (validation boundaries + gaps)

Workspace/filesystem validation (structural, pre-run) happens in promptbench/core/validation.py::validate_workspace(): it validates that the workspace root exists, that each required subdir in WORKSPACE_SUBDIRS exists, and emits warnings if templates/ or payloads/ are empty ("No templates found…", "No payloads found…"). Evidence: if not paths.root.exists(): … return diagnostics, loop over WORKSPACE_SUBDIRS, and the glob("**/*") emptiness checks in validate_workspace(). (See promptbench/core/validation.py snippet.)

Config/TOML schema validation (syntactic + shallow schema, hard-fail) happens in promptbench/core/config.py::load_config():

Parse errors are wrapped as ConfigError("Failed to parse TOML config…").

Required sections/fields are enforced: inputs, inputs.templates, inputs.payloads, and providers must exist, and providers must be non-empty. Evidence: if "inputs" not in data: …, if "providers" not in data: …, if not data["providers"]: …. (See promptbench/core/config.py snippet.)

Enum/domain validation (narrow, hard-fail) exists for only one field: runner.composition_mode is validated by casting to CompositionMode, failing with a targeted message listing allowed values. Evidence: comp_mode = CompositionMode(raw_mode) inside try/except ValueError. (See promptbench/core/config.py snippet.)

Provider-spec validation is minimal (likely domain gap boundary): in load_config() each provider only enforces presence of type; everything else is passed through without type/shape checks (model, url, args, env). Evidence: if "type" not in p_data: raise … and then ProviderSpec(... model=p_data.get("model"), url=p_data.get("url"), args=p_data.get("args", []), env=p_data.get("env", {})). This is a likely domain-validation gap (e.g., a provider “type” may require url, but nothing enforces it here). (See promptbench/core/config.py snippet.)

CLI override validation is effectively absent (high-likelihood inconsistency source) in apply_overrides():

It mutates fields directly and only casts timeout via int(...) with no bounds/range checks and no exception handling for non-numeric values. Evidence: config.timeout = int(overrides["timeout"]), config.concurrency = overrides["concurrency"], etc. This can produce inconsistent state (negative/zero concurrency, negative timeouts, non-int concurrency, etc.) and can also raise an uncaught ValueError if timeout is not parseable. (See promptbench/core/config.py snippet.)

Validation boundary mismatch: “workspace diagnostics” vs “config exceptions”:

Workspace validation is non-throwing and returns List[Diagnostic] (plus diagnostics_payload() converts to dicts).

Config validation is throwing (ConfigError).
This split makes it easy for callers to accidentally ignore workspace errors/warnings (because they’re just returned data) while config errors crash fast. Evidence: validate_workspace() returns diagnostics; load_config() raises ConfigError. (See both snippets.)

Type validation gaps can leak runtime errors past load_config():

load_config() assumes shapes: data["providers"].items() implies a dict; if TOML yields an unexpected type (or user edits it wrongly), you’ll get an AttributeError (not wrapped) because only TOML parsing is wrapped, not the subsequent structural assumptions. Evidence: only the open()/tomllib.load() block is in try/except Exception, not the rest of the function body. (See promptbench/core/config.py snippet.)

2) Risks/unknowns (most likely inconsistent-state gaps)

Unknown whether validate_workspace() is always invoked before operations that assume .promptbench/... exists (runner, discovery, CLI “run”). The file shows the validator, but not its call sites.

Unknown whether there is deeper validation elsewhere (e.g., per-provider requirements, template/payload file format validation, schema validation for payload contents). The provided code only shows workspace + config-level validation.

apply_overrides() mutates the Config object; without seeing promptbench/core/types.py::Config, it’s unknown whether Config is intended to be immutable, and whether mutation causes subtle shared-state issues if the same config instance is reused across runs.

validate_workspace()’s emptiness check uses glob("**/*"); depending on expected behavior, it may treat “only directories present” as “non-***_


--- docs/oracle-questions-2026-01-07/05-caching-state-layers.md ---
1) Direct answer (stateful components, lifecycle, invalidation/expiry, where implemented)

Persisted discovery cache (on-disk JSON)

What/where: .promptbench/cache/discovery_cache.json (path computed via workspace_paths(base).cache / "discovery_cache.json" in CacheManager.__init__). Implemented in promptbench/core/discovery_cache.py: CacheManager.

Lifecycle: created/used per base_dir; CacheManager.__init__ ensures the cache/ directory exists (mkdir(..., exist_ok=True)).

Invalidation/expiry: no time-based TTL. Freshness is computed by comparing cached mtimes to current mtimes (_entry_is_fresh); any mismatch or missing file invalidates.

Atomic write state via temp file + replace

What/where: CacheManager.save() writes JSON to a NamedTemporaryFile(..., delete=False) in the cache directory and then os.replace()s it into place. Implemented in promptbench/core/discovery_cache.py: CacheManager.save.

Lifecycle: temp file exists briefly; final cache file is replaced atomically (best-effort) on each save.

Invalidation/expiry: not applicable; this is a persistence mechanism. Failure mode is partial/invalid JSON if a crash happens before replace, but load() swallows exceptions and returns {}.

In-memory cache materialization

What/where: CacheManager.load() returns a Python Dict[str, dict] from the JSON file (or {} if missing/invalid). Implemented in promptbench/core/discovery_cache.py: CacheManager.load.

Lifecycle: whatever calls load() holds this dict in memory; this module itself does not keep an in-memory singleton—state is passed around by callers.

Invalidation/expiry: caller-dependent; module-provided invalidation is the _entry_is_fresh() check (if caller uses it).

Filesystem “workspace” state (directory structure as persistent state)

What/where: .promptbench/ root plus subdirectories (templates, payloads, skills, providers, sets, runs, cache). Implemented in promptbench/core/workspace.py (WORKSPACE_DIR, WORKSPACE_SUBDIRS, ensure_workspace_structure, workspace_paths).

Lifecycle: created on demand via ensure_workspace_structure() (mkdir for each subdir). Existence checks via workspace_exists().

Invalidation/expiry: none; this is persistent structural state.

Cache invalidation inputs: root mtimes derived from glob patterns

What/where: _glob_root() computes a “root” directory for a glob by walking path parts until it sees a glob metachar (* ? [), then _root_mtimes() stats those roots and stores mtimes keyed by root path string. Implemented in promptbench/core/discovery_cache.py: _glob_root, _root_mtimes.

Lifecycle: computed at cache-check time; not persisted unless stored inside cache entries as root_mtimes.

Invalidation/expiry: if any root mtime differs from cached root_mtimes, the entry is stale (_entry_is_fresh returns False).

Cache invalidation inputs: per-file mtimes

What/where: _file_mtimes() stats each discovered path and records st_mtime. _entry_is_fresh() re-stats each cached file path and requires exact equality with cached mtime. Implemented in promptbench/core/discovery_cache.py: _file_mtimes, _entry_is_fresh.

Lifecycle: computed at discovery/cache-validation time.

Invalidation/expiry: any missing file or mtime mismatch makes the entry stale. Notably, this is an exact float equality check against filesystem mtimes.

2) Risks/unknowns

No TTL / size bound: cache entries appear to live indefinitely unless mtimes change; there’s no expiry policy visible in these files.

Mtime equality brittleness: _entry_is_fresh() requires current != cached_mtime to invalidate; exact equality can be fragile across filesystems, tooling that preserves timestamps, timestamp resolution differences, or clock skew.

Root-mtime heuristic gaps: _glob_root() stops at the first glob segment; changes in deeper directories/files may not bump the computed root directory mtime on some filesystems, so root_mtimes may fail to detect some changes unless the per-file mtimes list covers them.

Silent corruption handling hides issues: load() returns {} on any exception (including JSON corruption), which can mask real problems and cause unexpected full re-discovery.

Broader system state unknown: from the provided files alone, there’s no visibility into:

any additional caches (e.g., provide


--- docs/oracle-questions-2026-01-07/06-caching-state-consistency.md ---
1) Direct answer (top consistency risks + cache knobs)

Recursive-glob stale reads (missing invalidation for new matches in existing subdirs). Cache freshness is gated by (a) mtimes of the already-cached files and (b) mtimes of the glob roots (root_mtimes) computed from the non-wildcard prefix of each pattern. New files that match ** patterns inside already-existing subdirectories can be missed because the cached file list won’t include them, and the chosen root directory’s st_mtime may not change when only a deep child directory changes. Evidence: cache reuse check in discover_files_cached(...) with _entry_is_fresh(...), plus _root_mtimes(...) using _glob_root(...) and only root.stat().st_mtime (not a recursive walk). [promptbench/core/discovery.py:discover_files_cached] + [promptbench/core/discovery_cache.py:_glob_root,_root_mtimes,_entry_is_fresh]

Policy/cache-key skew: allow_outside_root is not part of the cache key. Results produced with allow_outside_root=True can be returned later when allow_outside_root=False without re-enforcing the base-dir constraint (because the cached entry is returned before calling discover_files(...), and the key does not encode allow_outside_root). Evidence: key is cache_key(globs, base_path, filters) (no allow_outside_root), while enforcement happens only inside discover_files(...) when not cached. [promptbench/core/discovery.py:discover_files,discover_files_cached] + [promptbench/core/discovery_cache.py:cache_key]

CWD vs base_dir mismatch can yield “random” cache behavior for relative globs. discover_files(...) runs glob.glob(expanded_pattern, recursive=True) on the raw pattern (after ~ expansion) without anchoring it to base_dir, so relative patterns are evaluated relative to process CWD. Meanwhile _glob_root(...) does anchor non-absolute patterns to base_dir when computing root_mtimes. This can produce inconsistent invalidation/freshness decisions (and even incorrect discovery) when base_dir != cwd. Evidence: discover_files(...) does not join base_dir into the glob pattern; _glob_root(...) explicitly does. [promptbench/core/discovery.py:discover_files] + [promptbench/core/discovery_cache.py:_glob_root]

Mtime-based freshness can miss changes (stale reads) on coarse-resolution or nonstandard filesystems. Freshness is exact-equality on st_mtime for each cached file path; if a change does not update st_mtime (timestamp granularity, clock skew, some sync/network FS behaviors), _entry_is_fresh(...) can return true and serve stale data. Evidence: _entry_is_fresh(...) compares current != cached_mtime and otherwise treats the entry as fresh; no content hashing/size checks. [promptbench/core/discovery_cache.py:_entry_is_fresh,_file_mtimes]

Write skew / last-writer-wins across processes. The cache is a single JSON file loaded/saved with no locking; concurrent runs can overwrite each other’s updates (not corruption due to atomic replace, but lost updates). Evidence: load() reads whole JSON; save() writes full JSON then os.replace(...) with no merge/lock. [promptbench/core/discovery_cache.py:CacheManager.load,CacheManager.save]

Cache behavior knobs/configs present in-code (no TTL/expiry knobs in this layer). Available knobs are: force_refresh (bypass cache), cache injection (override cache manager/location), base_dir (both cache location and key component), and input dimensions globs/filters (key + results). There is no TTL, max-age, size limit, or selective invalidation beyond mtimes. Evidence: function signature and key construction. [promptbench/core/discovery.py:discover_files_cached] + [promptbench/core/discovery_cache.py:cache_key]

2) Risks / unknowns

workspace_paths(base) determines where discovery_cache.json lives; without promptbench/core/workspace.py, the exact on-disk location and any environment/config knobs for cache placement are unknown. [promptbench/core/discovery_cache.py:CacheManager.__init__]

It’s unclear whether higher layers (CLI/config) expose user-facing switches (e.g., --no-cache, --force-refresh, config.toml fields) beyond the raw discover_files_cached(...) parameters; those call sites aren’t included.

If callers mutate files via operations that preserve mtimes (copying with metadata, certain tooling), freshness may be incorrectly assessed as “fresh,” causing hard-t


--- docs/oracle-questions-2026-01-07/07-background-jobs-discovery.md ---
1) Direct answer (background jobs/workers/scheduled tasks)

In-process “background” execution via thread pool (not a separate worker system).
Jobs are executed concurrently inside the current process using concurrent.futures.ThreadPoolExecutor(max_workers=config.concurrency or 1) and executor.submit(_run_single_job, ...); completion is awaited via as_completed(...). Trigger is a direct call to run_jobs(...) / run_jobs_with_events(...) (no queue/daemon/cron evident in these files). (promptbench/runner/executor.py: run_jobs; promptbench/runner/eventing_executor.py: run_jobs_with_events)

Unit of work/payload = JobSpec executed against a ProviderAdapter.
Each submitted task runs _run_single_job(job, config, adapters) which reads template_path + payload_path + skill_path, composes a system prompt, runs the provider, then writes artifacts. (executor.py: _run_single_job steps 1–5; eventing_executor.py: _run_single_job)

Retry mechanism is synchronous per job attempt-loop (no backoff/jitter shown).
Retries are driven by max_attempts = max(1, config.retries + 1) and a while attempt_count < max_attempts: loop. A response is retryable if response.error["retryable"] is True or response.error["code"] is in RETRYABLE_CODES (HTTP_408/429/5xx, RATE_LIMIT, TIMEOUT, TEMPORARY_UNAVAILABLE). (executor.py: RETRYABLE_CODES, _should_retry_response, retry loop in _run_single_job; same in eventing_executor.py)

Retry logging/payload captured as an “attempts” list in run metadata.
Each attempt appends {"attempt": attempt_count, "error": ...} either from adapter exceptions, contract violations, or provider response.error; attempt_count and attempts are stored in meta persisted by write_run_artifacts(...). (executor.py: attempts, meta["attempt_count"], meta["attempts"]; eventing_executor.py: same)

Contract violations are treated as non-retryable and terminate the job early.
ContractViolationError is caught, recorded as {"code": "CONTRACT_VIOLATION" ...}, then the loop breaks (no further attempts). (executor.py: except ContractViolationError ... break; eventing_executor.py: same)

Idempotency is not guaranteed end-to-end; retries can re-invoke providers and re-write artifacts.
Retries re-call adapter.run(ctx) and will typically incur repeated provider side effects/costs unless the adapter/provider is inherently idempotent. Artifact writing happens once after the retry loop, but repeated runs (or crashes mid-run) can create multiple run directories depending on make_run_dir(...) behavior; these files don’t show a dedupe/idempotency key. (executor.py: retry loop calls adapter.run; later make_run_dir(...) + write_run_artifacts(...); eventing_executor.py: same)

Event “worker” is just synchronous JSONL emission to a provided sink (no scheduler).
run_jobs_with_events(..., event_sink, run_id) emits JOB_STARTED, JOB_COMPLETED, and ARTIFACT_WRITTEN to event_sink via emit_event(...) as jobs run/finish; the “attempt” field in emitted payload is currently hard-coded as 1 in these events (does not reflect retries). (eventing_executor.py: emit_event calls in run_jobs_with_events)

2) Risks / unknowns

No evidence (in the provided files) of a real background worker system (Celery/RQ/Sidekiq-like), a durable queue, or scheduled tasks; however, absence cannot be concluded without scanning the rest of promptbench/ (only two runner files were provided).

Retry policy gaps: no exponential backoff/jitter, no per-code cooldown, and no circuit breaker; concurrent retries across threads can amplify rate-limits/cost.

Observability mismatch: event payload’s "attempt": 1 is emitted for start/completion regardless of actual retry count, so downstream logs/UX may misrepresent retry behavior. (eventing_executor.py: ev_


--- docs/oracle-questions-2026-01-07/08-background-jobs-reliability.md ---
1) Direct answer (main reliability controls + what’s missing/inconsistent)

Concurrency limit exists, but only as a single global knob. run_jobs() uses ThreadPoolExecutor(max_workers=config.concurrency or 1), so total parallelism is capped, but there’s no per-provider/per-host throttling or backpressure beyond that global thread count. (promptbench/runner/executor.py: max_workers = config.concurrency or 1; ThreadPoolExecutor(max_workers=max_workers))

Retry count exists, but it’s attempt-count-based only (no time budget). _run_single_job() sets max_attempts = max(1, config.retries + 1) and loops until attempts are exhausted. (promptbench/runner/executor.py: max_attempts = max(1, config.retries + 1); while attempt_count < max_attempts:)

Retry classification is partially standardized but depends on provider adapters being consistent. _should_retry_response() retries if error.retryable is True or if error.code is in a hardcoded RETRYABLE_CODES set (HTTP 408/429/5xx, TIMEOUT, etc.). If adapters don’t emit those codes (or don’t set retryable=True), retry behavior becomes inconsistent across providers. (promptbench/runner/executor.py: RETRYABLE_CODES = {...}; _should_retry_response() logic)

Adapter exceptions are retried unconditionally (until max attempts), but without a structured “retryable” check. Any thrown exception becomes {"code": "ADAPTER_EXCEPTION" ...} and the code immediately continues to the next attempt if attempts remain—regardless of whether it was a permanent error (e.g., bad API key) vs transient. (promptbench/runner/executor.py: except Exception as e: ... if attempt_count < max_attempts: continue)

Contract violations fail fast (no retry), which is consistent but creates a sharp boundary. ContractViolationError is converted to CONTRACT_VIOLATION and the loop breaks immediately. (promptbench/runner/executor.py: except ContractViolationError ... break)

No backoff/jitter exists anywhere in the retry loop. Retries happen immediately (continue with no sleep), which is especially risky for 429/rate limits and transient outages because parallel jobs will “hammer” the provider in lockstep. (promptbench/runner/executor.py: retry loop has continue paths with no delay)

No dead-lettering mechanism exists (only “error artifacts”). Failures are recorded in meta["attempts"], meta["attempt_count"], and RunStatus.ERROR, and written via write_run_artifacts(...), but there’s no separate DLQ store, quarantine folder/manifest, or “needs-reprocess” queue. (promptbench/runner/executor.py: attempts collection; meta = {..., "status": ..., "attempts": attempts}; write_run_artifacts(...))

No built-in reprocessing flow exists beyond “rerun the whole job.” The system persists detailed attempt metadata, but there’s no first-class command/path to re-enqueue failed jobs from prior runs, no “resume” semantics, and no idempotency keying at the provider-call level—so reprocessing is manual and may duplicate side effects if providers/tools are not read-only. (promptbench/runner/executor.py: artifacts include metadata, but there’s no requeue logic in run_jobs() / _run_single_job())

2) Risks/unknowns

Unknown whether adapters implement their own timeouts/backoff. ProviderAdapter.run() has no timeout or retry contract in the base class, so reliability behavior may be fragmented across provider implementations. (promptbench/providers/base.py: run(self, context) -> ProviderResponse is abstract; no timeout/backoff hooks)

Thread safety is not guaranteed. ThreadPoolExecutor will call adapter.run() concurrently across jobs; if adapters share client/session state, this can introduce race conditions unless each adapter is explicitly thread-safe. (promptbench/runner/executor.py: submits _run_single_job concurrently; promptbench/providers/base.py: no thread-safety guidance)

Failure persistence is asymmetric. Normal paths write artifacts, but if something fails before write_run_artifacts() (e.g., during reading inputs, composing, building skill context), the outer except returns INTERNAL_JOB_ERROR without showing any persistence here; whether partial artifacts exist depends on what make_run_dir() / other layers do (not shown). (promptbench/runner/executor.py: outer except returns RunResult; no artifact write in that block)

No guardrails against rate-limit storms. With immediate retries + concurrent threads, a single 429 condition can fan out into synchronized retry bursts.

3) Next smallest concrete experiment (exactly one action)

Run a repo-wide search to enumerate all retry/backoff/concurrency/dead-letter hooks so you can confirm there isn’t a second reliability layer elsewhere:

bash
Copy code
rg -n "(RETRYABLE_CODES|_should_retry_response|backoff|jitter|dead.?letter|DLQ|reprocess|retry|concurrency|max_workers|ThreadPool_


--- examples/expected-output.md ---
```md
#1
[Spec]
SYSTEM: Conversation-to-Issue-Ticket Converter

Objective
- Convert an exported user↔assistant conversation into ONE cleaned, coherent issue ticket.
- Preserve all key details; remove noise; do NOT add new facts.

Input
- Exported conversation text with alternating “user” and “assistant” messages (any format: plain text/markdown/JSON-like; timestamps optional).

Non-negotiables
- Fidelity: Do not invent requirements, causes, decisions, timelines, metrics, links, or outcomes.
- Completeness: Retain all materially relevant details (goals, constraints, edge cases, decisions, rejected options, action items, dependencies, risks).
- No questions: Never ask for missing info. Use “Unknown” / “Not provided” where needed.
- De-duplication: Merge repeats; keep the clearest formulation.
- Conflicts: If contradictions exist, report both versions with attribution (per user / per assistant). Do not resolve by guessing.
- Terminology: Normalize names/terms to the most consistent wording present in the conversation.
- Traceability: For critical/ambiguous details, add attribution and optionally a short quote fragment (≤25 words).
- Security/privacy: Redact credentials and sensitive personal data; replace with “[REDACTED]” and note the redaction.

Process
1) Segment into: problem statement(s), context, requirements, constraints, environment, attempted fixes, errors, decisions, next steps.
2) Identify: primary issue; secondary issues (if any); stakeholders/owners (if stated); target system/component; impact/urgency signals.
3) Extract artifacts: repro steps; expected vs actual; error messages/logs; links/IDs/filenames/code snippets (lightly cleaned, meaning preserved).
4) Produce one consolidated narrative + structured ticket.

Output (STRICT markdown template)
Title:
- Concise, specific, action-oriented.

Summary:
- 2–5 sentences: what’s wrong/needed, who is affected, why it matters.

Background / Context:
- Relevant history + constraints.

Current Behavior (Actual):
- Bullets: symptoms, observed outputs, error text.

Expected Behavior:
- Bullets: success definition.

Requirements:
- Bullets: explicit requirements + constraints (performance, compatibility, compliance, UX, scope limits).

Out of Scope:
- Bullets: exclusions stated/implied; if none, “Not provided”.

Reproduction Steps:
- Numbered steps; if unavailable, “Not provided”.

Environment:
- OS, app version, browser, device, deployment, flags, configs; use “Unknown” when missing.

Evidence:
- Verbatim logs/errors, screenshot/attachment references, links, file paths, IDs.

Decisions / Agreements:
- Bullets with attribution where needed.

Open Items / Unknowns:
- Bullets of missing info that blocks execution (no questions).

Risks / Dependencies:
- Bullets of dependencies/integrations/approvals/known risks.

Acceptance Criteria:
- Testable checklist derived from requirements + expected behavior.
- If vague, translate to minimal testable criteria without adding scope.

Priority & Severity (if inferable from text):
- Priority: P0–P3
- Severity: S0–S3
- If not clearly supported, “Not provided”.

Labels (optional):
- 3–8 tags only if supported by the conversation.

Style
- Crisp bullets; no filler; self-contained ticket understandable without the conversation.
```

```md
#2
[Directive]
Convert the provided user↔assistant conversation export into ONE consolidated issue ticket.

Rules
- Do NOT add facts. Do NOT guess. Do NOT ask questions.
- Keep all material details; remove chit-chat and duplicates.
- If info is missing, write “Unknown” or “Not provided”.
- If the conversation conflicts, list both versions with attribution (per user / per assistant); don’t resolve.
- Redact credentials/sensitive personal data as “[REDACTED]” and note the redaction.
- For critical/ambiguous points, add attribution and optionally a ≤25-word quote fragment.

Extract
- Primary + secondary issues (if any)
- Target system/component, stakeholders/owners (if stated)
- Requirements + constraints
- Attempted fixes, errors/logs (verbatim), links/IDs/files/code snippets
- Steps to reproduce (if present)
- Impact/urgency cues, risks/dependencies, decisions, next steps

Output only this markdown ticket (headings and order must match)
Title:
Summary:
Background / Context:
Current Behavior (Actual):
Expected Behavior:
Requirements:
Out of Scope:
Reproduction Steps:
Environment:
Evidence:
Decisions / Agreements:
Open Items / Unknowns:
Risks / Dependencies:
Acceptance Criteria:
Priority & Severity (if inferable from text):
Labels (optional):
```

```md
#3
[QA-Ready]
SYSTEM: Conversation-to-Issue-Ticket Converter

Input
- {conversation_export}

Hard validations
- Output ONLY a single issue ticket in markdown.
- Include EVERY section in the specified order, even if content is “Unknown/Not provided”.
- No questions. No speculative language. No new facts.
- Conflicts: explicitly surface contradictions with attribution (User vs Assistant).
- Evidence: keep error/log text verbatim; lightly clean code/paths without changing meaning.
- Privacy: redact any secrets or sensitive personal data as “[REDACTED]”.

Ticket template (must match exactly)
Title:
Summary:
Background / Context:
Current Behavior (Actual):
Expected Behavior:
Requirements:
Out of Scope:
Reproduction Steps:
Environment:
Evidence:
Decisions / Agreements:
Open Items / Unknowns:
Risks / Dependencies:
Acceptance Criteria:
Priority & Severity (if inferable from text):
Labels (optional):

Acceptance Criteria guidance
- Convert requirements into testable checks (Given/When/Then style acceptable).
- If requirements are vague, produce minimal testable criteria without expanding scope.

Style
- Bullets preferred; concise, concrete nouns/verbs; self-contained.
```


--- examples/user-input.md ---

```md primary.md
You are **Hiro — Prompt Optimization Specialist**. Transform any raw user prompt into up to **4 concise, high-leverage variants** that preserve intent while improving clarity, constraints, and outcome specificity.

**Your job**

* Keep the user’s original goal intact. Remove fluff, tighten verbs, and make deliverables and success criteria explicit.
* Resolve ambiguity with **neutral defaults** or **clearly marked placeholders** like `{context}`, `{inputs}`, `{constraints}`, `{acceptance_criteria}`, `{format}`, `{deadline}`.
* Add structure (steps, bullets, numbered requirements) only when it improves execution.
* Match or gently improve the **tone** implied by the user (directive/spec-like, polite, collaborative). Never over-polish into marketing-speak.
* Do **not** introduce tools, external data, or scope changes unless the user asked for them.
* Prefer active voice, testable requirements, and measurable outputs.

**Output rules**

* Return **only** the variants, each in its **own fenced code block**. No commentary, no preamble, no trailing notes.
* Produce **1–4 variants** (default 3). Stop at 4 unless the user explicitly requests more. Number each (#1,#2,#n).
* For each block, begin with a short bracketed style tag (e.g., `[Directive]`, `[Spec]`, `[Polite]`, `[QA-Ready]`) on the first line, then the optimized prompt on subsequent lines.

**Optimization checklist (apply silently)**

* Clarify objective and end artifact.
* Specify audience/user/environment if implied.
* Pin input sources and constraints.
* Define acceptance criteria and non-goals.
* State format/structure and any length limits.
* Include edge cases or examples if the user hinted at them.
* Keep placeholders where the user must decide.

**Now optimize the next input.**
User prompt: {paste user’s raw prompt here}
```

---

```md secondary.md

SYSTEM: Conversation-to-Issue-Ticket Converter

Role
- Convert an exported user↔assistant conversation into a single cleaned, coherent issue ticket.
- Preserve all key details. Remove noise. Do not add new facts.

Input
- An exported conversation containing alternating messages from “user” and “assistant” (any format: plain text, markdown, JSON-like, timestamps optional).

Core rules
- Fidelity: Do not invent requirements, causes, decisions, timelines, metrics, or outcomes.
- Completeness: Keep every materially relevant detail (goals, constraints, edge cases, decisions, rejected options, action items, dependencies, risks).
- No questions: Do not ask the reader for missing info. If information is missing, mark it explicitly as “Unknown” or “Not provided”.
- De-duplication: Merge repeats and restatements. Keep the clearest formulation.
- Conflict handling: If the conversation contradicts itself, report both versions and attribute them (User vs Assistant). Do not resolve by guessing.
- Terminology: Normalize names and terms (features, components, people, systems) using the most consistent wording from the conversation.
- Traceability: When a detail is critical or ambiguous, include a short attributed quote fragment (≤25 words) or “(per user)” / “(per assistant)”.
- Security/privacy: Keep secrets out. If the conversation includes credentials or sensitive personal data, redact and note “[REDACTED]”.
- Output only the issue ticket. No meta-commentary.

Process
1) Parse and segment the conversation into: problem statement(s), context, requirements, constraints, environment, attempted fixes, errors, decisions, next steps.
2) Identify:
   - Primary issue
   - Secondary issues (if present)
   - Stakeholders/owners (if stated)
   - Target system/component
   - Impact and urgency signals
3) Extract concrete artifacts:
   - Steps to reproduce
   - Expected vs actual behavior
   - Error messages/logs
   - Links, IDs, filenames, code snippets (lightly cleaned; preserve meaning)
4) Produce one consolidated, coherent narrative and a structured ticket.

Output format (strict)
Title:
- Concise, specific, action-oriented. Keep short.

Summary:
- 2–5 sentences describing what’s wrong / needed, who is affected, and why it matters.

Background / Context:
- Relevant history and constraints from the conversation.

Current Behavior (Actual):
- Bullet list. Include symptoms, observed outputs, error text.

Expected Behavior:
- Bullet list. Clear success definition.

Requirements:
- Bullet list of explicit requirements extracted from the conversation.
- Include constraints (performance, compatibility, compliance, UX, scope limits).

Out of Scope:
- Bullet list of exclusions stated or implied by the conversation. If none, “Not provided”.

Reproduction Steps:
- Numbered steps. If not available, “Not provided”.

Environment:
- OS, app version, browser, device, deployment, flags, configs. Use “Unknown” when missing.

Evidence:
- Logs/errors (verbatim), screenshots/attachments references, links, file paths, IDs.

Decisions / Agreements:
- Bullet list of decisions made in the conversation, with attribution where needed.

Open Items / Unknowns:
- Bullet list of missing info that blocks execution. No questions; just state unknowns.

Risks / Dependencies:
- Bullet list of dependencies, integrations, approvals, or known risks mentioned.

Acceptance Criteria:
- Testable checklist statements. Derive from requirements and expected behavior.
- If requirements are vague, translate into minimal testable criteria without adding new scope.

Priority & Severity (if inferable from text):
- Priority: P0–P3
- Severity: S0–S3
- Only infer if conversation provides clear cues; otherwise “Not provided”.

Labels (optional):
- 3–8 tags (e.g., bug, enhancement, auth, ui, performance). Only if supported by conversation.

Style constraints
- Use crisp bullet points. No filler.
- Prefer concrete nouns/verbs over abstract phrasing.
- Keep ticket self-contained and understandable without reading the conversation.



```


--- .taskmaster/templates/example_prd.txt ---
<context>
# Overview  
[Provide a high-level overview of your product here. Explain what problem it solves, who it's for, and why it's valuable.]

# Core Features  
[List and describe the main features of your product. For each feature, include:
- What it does
- Why it's important
- How it works at a high level]

# User Experience  
[Describe the user journey and experience. Include:
- User personas
- Key user flows
- UI/UX considerations]
</context>
<PRD>
# Technical Architecture  
[Outline the technical implementation details:
- System components
- Data models
- APIs and integrations
- Infrastructure requirements]

# Development Roadmap  
[Break down the development process into phases:
- MVP requirements
- Future enhancements
- Do not think about timelines whatsoever -- all that matters is scope and detailing exactly what needs to be build in each phase so it can later be cut up into tasks]

# Logical Dependency Chain
[Define the logical order of development:
- Which features need to be built first (foundation)
- Getting as quickly as possible to something usable/visible front end that works
- Properly pacing and scoping each feature so it is atomic but can also be built upon and improved as development approaches]

# Risks and Mitigations  
[Identify potential risks and how they'll be addressed:
- Technical challenges
- Figuring out the MVP that we can build upon
- Resource constraints]

# Appendix  
[Include any additional information:
- Research findings
- Technical specifications]
</PRD>

--- .taskmaster/templates/example_prd_rpg.txt ---
<rpg-method>
# Repository Planning Graph (RPG) Method - PRD Template

This template teaches you (AI or human) how to create structured, dependency-aware PRDs using the RPG methodology from Microsoft Research. The key insight: separate WHAT (functional) from HOW (structural), then connect them with explicit dependencies.

## Core Principles

1. **Dual-Semantics**: Think functional (capabilities) AND structural (code organization) separately, then map them
2. **Explicit Dependencies**: Never assume - always state what depends on what
3. **Topological Order**: Build foundation first, then layers on top
4. **Progressive Refinement**: Start broad, refine iteratively

## How to Use This Template

- Follow the instructions in each `<instruction>` block
- Look at `<example>` blocks to see good vs bad patterns
- Fill in the content sections with your project details
- The AI reading this will learn the RPG method by following along
- Task Master will parse the resulting PRD into dependency-aware tasks

## Recommended Tools for Creating PRDs

When using this template to **create** a PRD (not parse it), use **code-context-aware AI assistants** for best results:

**Why?** The AI needs to understand your existing codebase to make good architectural decisions about modules, dependencies, and integration points.

**Recommended tools:**
- **Claude Code** (claude-code CLI) - Best for structured reasoning and large contexts
- **Cursor/Windsurf** - IDE integration with full codebase context
- **Gemini CLI** (gemini-cli) - Massive context window for large codebases
- **Codex/Grok CLI** - Strong code generation with context awareness

**Note:** Once your PRD is created, `task-master parse-prd` works with any configured AI model - it just needs to read the PRD text itself, not your codebase.
</rpg-method>

---

<overview>
<instruction>
Start with the problem, not the solution. Be specific about:
- What pain point exists?
- Who experiences it?
- Why existing solutions don't work?
- What success looks like (measurable outcomes)?

Keep this section focused - don't jump into implementation details yet.
</instruction>

## Problem Statement
[Describe the core problem. Be concrete about user pain points.]

## Target Users
[Define personas, their workflows, and what they're trying to achieve.]

## Success Metrics
[Quantifiable outcomes. Examples: "80% task completion via autopilot", "< 5% manual intervention rate"]

</overview>

---

<functional-decomposition>
<instruction>
Now think about CAPABILITIES (what the system DOES), not code structure yet.

Step 1: Identify high-level capability domains
- Think: "What major things does this system do?"
- Examples: Data Management, Core Processing, Presentation Layer

Step 2: For each capability, enumerate specific features
- Use explore-exploit strategy:
  * Exploit: What features are REQUIRED for core value?
  * Explore: What features make this domain COMPLETE?

Step 3: For each feature, define:
- Description: What it does in one sentence
- Inputs: What data/context it needs
- Outputs: What it produces/returns
- Behavior: Key logic or transformations

<example type="good">
Capability: Data Validation
  Feature: Schema validation
    - Description: Validate JSON payloads against defined schemas
    - Inputs: JSON object, schema definition
    - Outputs: Validation result (pass/fail) + error details
    - Behavior: Iterate fields, check types, enforce constraints

  Feature: Business rule validation
    - Description: Apply domain-specific validation rules
    - Inputs: Validated data object, rule set
    - Outputs: Boolean + list of violated rules
    - Behavior: Execute rules sequentially, short-circuit on failure
</example>

<example type="bad">
Capability: validation.js
  (Problem: This is a FILE, not a CAPABILITY. Mixing structure into functional thinking.)

Capability: Validation
  Feature: Make sure data is good
  (Problem: Too vague. No inputs/outputs. Not actionable.)
</example>
</instruction>

## Capability Tree

### Capability: [Name]
[Brief description of what this capability domain covers]

#### Feature: [Name]
- **Description**: [One sentence]
- **Inputs**: [What it needs]
- **Outputs**: [What it produces]
- **Behavior**: [Key logic]

#### Feature: [Name]
- **Description**:
- **Inputs**:
- **Outputs**:
- **Behavior**:

### Capability: [Name]
...

</functional-decomposition>

---

<structural-decomposition>
<instruction>
NOW think about code organization. Map capabilities to actual file/folder structure.

Rules:
1. Each capability maps to a module (folder or file)
2. Features within a capability map to functions/classes
3. Use clear module boundaries - each module has ONE responsibility
4. Define what each module exports (public interface)

The goal: Create a clear mapping between "what it does" (functional) and "where it lives" (structural).

<example type="good">
Capability: Data Validation
  → Maps to: src/validation/
    ├── schema-validator.js      (Schema validation feature)
    ├── rule-validator.js         (Business rule validation feature)
    └── index.js                  (Public exports)

Exports:
  - validateSchema(data, schema)
  - validateRules(data, rules)
</example>

<example type="bad">
Capability: Data Validation
  → Maps to: src/utils.js
  (Problem: "utils" is not a clear module boundary. Where do I find validation logic?)

Capability: Data Validation
  → Maps to: src/validation/everything.js
  (Problem: One giant file. Features should map to separate files for maintainability.)
</example>
</instruction>

## Repository Structure

```
project-root/
├── src/
│   ├── [module-name]/       # Maps to: [Capability Name]
│   │   ├── [file].js        # Maps to: [Feature Name]
│   │   └── index.js         # Public exports
│   └── [module-name]/
├── tests/
└── docs/
```

## Module Definitions

### Module: [Name]
- **Maps to capability**: [Capability from functional decomposition]
- **Responsibility**: [Single clear purpose]
- **File structure**:
  ```
  module-name/
  ├── feature1.js
  ├── feature2.js
  └── index.js
  ```
- **Exports**:
  - `functionName()` - [what it does]
  - `ClassName` - [what it does]

</structural-decomposition>

---

<dependency-graph>
<instruction>
This is THE CRITICAL SECTION for Task Master parsing.

Define explicit dependencies between modules. This creates the topological order for task execution.

Rules:
1. List modules in dependency order (foundation first)
2. For each module, state what it depends on
3. Foundation modules should have NO dependencies
4. Every non-foundation module should depend on at least one other module
5. Think: "What must EXIST before I can build this module?"

<example type="good">
Foundation Layer (no dependencies):
  - error-handling: No dependencies
  - config-manager: No dependencies
  - base-types: No dependencies

Data Layer:
  - schema-validator: Depends on [base-types, error-handling]
  - data-ingestion: Depends on [schema-validator, config-manager]

Core Layer:
  - algorithm-engine: Depends on [base-types, error-handling]
  - pipeline-orchestrator: Depends on [algorithm-engine, data-ingestion]
</example>

<example type="bad">
- validation: Depends on API
- API: Depends on validation
(Problem: Circular dependency. This will cause build/runtime issues.)

- user-auth: Depends on everything
(Problem: Too many dependencies. Should be more focused.)
</example>
</instruction>

## Dependency Chain

### Foundation Layer (Phase 0)
No dependencies - these are built first.

- **[Module Name]**: [What it provides]
- **[Module Name]**: [What it provides]

### [Layer Name] (Phase 1)
- **[Module Name]**: Depends on [[module-from-phase-0], [module-from-phase-0]]
- **[Module Name]**: Depends on [[module-from-phase-0]]

### [Layer Name] (Phase 2)
- **[Module Name]**: Depends on [[module-from-phase-1], [module-from-foundation]]

[Continue building up layers...]

</dependency-graph>

---

<implementation-roadmap>
<instruction>
Turn the dependency graph into concrete development phases.

Each phase should:
1. Have clear entry criteria (what must exist before starting)
2. Contain tasks that can be parallelized (no inter-dependencies within phase)
3. Have clear exit criteria (how do we know phase is complete?)
4. Build toward something USABLE (not just infrastructure)

Phase ordering follows topological sort of dependency graph.

<example type="good">
Phase 0: Foundation
  Entry: Clean repository
  Tasks:
    - Implement error handling utilities
    - Create base type definitions
    - Setup configuration system
  Exit: Other modules can import foundation without errors

Phase 1: Data Layer
  Entry: Phase 0 complete
  Tasks:
    - Implement schema validator (uses: base types, error handling)
    - Build data ingestion pipeline (uses: validator, config)
  Exit: End-to-end data flow from input to validated output
</example>

<example type="bad">
Phase 1: Build Everything
  Tasks:
    - API
    - Database
    - UI
    - Tests
  (Problem: No clear focus. Too broad. Dependencies not considered.)
</example>
</instruction>

## Development Phases

### Phase 0: [Foundation Name]
**Goal**: [What foundational capability this establishes]

**Entry Criteria**: [What must be true before starting]

**Tasks**:
- [ ] [Task name] (depends on: [none or list])
  - Acceptance criteria: [How we know it's done]
  - Test strategy: [What tests prove it works]

- [ ] [Task name] (depends on: [none or list])

**Exit Criteria**: [Observable outcome that proves phase complete]

**Delivers**: [What can users/developers do after this phase?]

---

### Phase 1: [Layer Name]
**Goal**:

**Entry Criteria**: Phase 0 complete

**Tasks**:
- [ ] [Task name] (depends on: [[tasks-from-phase-0]])
- [ ] [Task name] (depends on: [[tasks-from-phase-0]])

**Exit Criteria**:

**Delivers**:

---

[Continue with more phases...]

</implementation-roadmap>

---

<test-strategy>
<instruction>
Define how testing will be integrated throughout development (TDD approach).

Specify:
1. Test pyramid ratios (unit vs integration vs e2e)
2. Coverage requirements
3. Critical test scenarios
4. Test generation guidelines for Surgical Test Generator

This section guides the AI when generating tests during the RED phase of TDD.

<example type="good">
Critical Test Scenarios for Data Validation module:
  - Happy path: Valid data passes all checks
  - Edge cases: Empty strings, null values, boundary numbers
  - Error cases: Invalid types, missing required fields
  - Integration: Validator works with ingestion pipeline
</example>
</instruction>

## Test Pyramid

```
        /\
       /E2E\       ← [X]% (End-to-end, slow, comprehensive)
      /------\
     /Integration\ ← [Y]% (Module interactions)
    /------------\
   /  Unit Tests  \ ← [Z]% (Fast, isolated, deterministic)
  /----------------\
```

## Coverage Requirements
- Line coverage: [X]% minimum
- Branch coverage: [X]% minimum
- Function coverage: [X]% minimum
- Statement coverage: [X]% minimum

## Critical Test Scenarios

### [Module/Feature Name]
**Happy path**:
- [Scenario description]
- Expected: [What should happen]

**Edge cases**:
- [Scenario description]
- Expected: [What should happen]

**Error cases**:
- [Scenario description]
- Expected: [How system handles failure]

**Integration points**:
- [What interactions to test]
- Expected: [End-to-end behavior]

## Test Generation Guidelines
[Specific instructions for Surgical Test Generator about what to focus on, what patterns to follow, project-specific test conventions]

</test-strategy>

---

<architecture>
<instruction>
Describe technical architecture, data models, and key design decisions.

Keep this section AFTER functional/structural decomposition - implementation details come after understanding structure.
</instruction>

## System Components
[Major architectural pieces and their responsibilities]

## Data Models
[Core data structures, schemas, database design]

## Technology Stack
[Languages, frameworks, key libraries]

**Decision: [Technology/Pattern]**
- **Rationale**: [Why chosen]
- **Trade-offs**: [What we're giving up]
- **Alternatives considered**: [What else we looked at]

</architecture>

---

<risks>
<instruction>
Identify risks that could derail development and how to mitigate them.

Categories:
- Technical risks (complexity, unknowns)
- Dependency risks (blocking issues)
- Scope risks (creep, underestimation)
</instruction>

## Technical Risks
**Risk**: [Description]
- **Impact**: [High/Medium/Low - effect on project]
- **Likelihood**: [High/Medium/Low]
- **Mitigation**: [How to address]
- **Fallback**: [Plan B if mitigation fails]

## Dependency Risks
[External dependencies, blocking issues]

## Scope Risks
[Scope creep, underestimation, unclear requirements]

</risks>

---

<appendix>
## References
[Papers, documentation, similar systems]

## Glossary
[Domain-specific terms]

## Open Questions
[Things to resolve during development]
</appendix>

---

<task-master-integration>
# How Task Master Uses This PRD

When you run `task-master parse-prd <file>.txt`, the parser:

1. **Extracts capabilities** → Main tasks
   - Each `### Capability:` becomes a top-level task

2. **Extracts features** → Subtasks
   - Each `#### Feature:` becomes a subtask under its capability

3. **Parses dependencies** → Task dependencies
   - `Depends on: [X, Y]` sets task.dependencies = ["X", "Y"]

4. **Orders by phases** → Task priorities
   - Phase 0 tasks = highest priority
   - Phase N tasks = lower priority, properly sequenced

5. **Uses test strategy** → Test generation context
   - Feeds test scenarios to Surgical Test Generator during implementation

**Result**: A dependency-aware task graph that can be executed in topological order.

## Why RPG Structure Matters

Traditional flat PRDs lead to:
- ❌ Unclear task dependencies
- ❌ Arbitrary task ordering
- ❌ Circular dependencies discovered late
- ❌ Poorly scoped tasks

RPG-structured PRDs provide:
- ✅ Explicit dependency chains
- ✅ Topological execution order
- ✅ Clear module boundaries
- ✅ Validated task graph before implementation

## Tips for Best Results

1. **Spend time on dependency graph** - This is the most valuable section for Task Master
2. **Keep features atomic** - Each feature should be independently testable
3. **Progressive refinement** - Start broad, use `task-master expand` to break down complex tasks
4. **Use research mode** - `task-master parse-prd --research` leverages AI for better task generation
</task-master-integration>


--- benchmarks/demo/71d38850b64a/composed_system.txt ---
You are a world-class creative writer.
Your goal is to fulfill the user request with elegance and precision.

{{USER_PROMPT}}

User prompt: Write a haiku about a rusty robot in a garden.



--- benchmarks/demo/71d38850b64a/output.txt ---
--- Assistant Response ---
I have analyzed your request and generated this response based on the provided template.
Input received via stdin:
You are a world-class creative writer.
Your goal is to fulfill the user request with elegance and precision.

{{USER_PROMPT}}

User prompt: Write a haiku about a rusty robot in a garden.



--- tickets/mcp/promptbench_mcp_server_design_implementation.md ---
# promptbench-mcp-server

## tree

```text
promptbench-mcp-server/
  pyproject.toml
  README.md
  promptbench_mcp/
    __init__.py
    server.py
    models.py
    prompt_templates.py
    promptbench_cli.py
    security.py
  tests/
    test_security.py
```

---

## promptbench-mcp-server/pyproject.toml

```toml
# path: promptbench-mcp-server/pyproject.toml
[project]
name = "promptbench-mcp-server"
version = "0.1.0"
description = "MCP server exposing promptbench as tools (matrix, prompt generation, run)"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
  "mcp>=1.8.0",
  "pydantic>=2.6.0",
]

[project.scripts]
promptbench-mcp = "promptbench_mcp.server:main"

[tool.pytest.ini_options]
addopts = "-q"
```

---

## promptbench-mcp-server/README.md

```md
<!-- path: promptbench-mcp-server/README.md -->
# promptbench-mcp-server

Expose `promptbench` as an MCP server so agents/assistants can:
- read workspace health (doctor)
- discover inputs
- build a matrix (templates/payloads/skills/providers)
- generate a prompt-engineer payload (from structured fields)
- run selected jobs via promptbench

## Requirements
- Python 3.11+
- `promptbench` installed/available in the same environment (the server shells out to `python3 -m promptbench ...`).

## Install (uv)
```bash
# shorthand: uv venv && uv pip install -e .
uv venv
uv pip install -e .

# ensure promptbench is available too (either installed or editable from your repo)
# uv pip install -e ../promptbench
```

## Run (stdio)
```bash
promptbench-mcp
```

## Run (streamable HTTP)
```bash
promptbench-mcp --transport streamable_http --host 127.0.0.1 --port 8765
```

## Security model
The server restricts file IO to allowlisted roots.

Configure allowlisted roots (colon-separated):
```bash
export PROMPTBENCH_MCP_ROOTS="/path/to/your/workspaces:/another/root"
```

Default: current working directory.

## Typical agent flow
1) `promptbench_matrix(config_path)` → choose templates/payloads/skills/providers
2) `promptbench_render_prompt_engineer_payload(...)` → get markdown
3) `promptbench_write_payload(...)` → save payload into `.promptbench/payloads/_mcp/`
4) `promptbench_run(...)` → run selected jobs + return summary
```

---

## promptbench-mcp-server/promptbench_mcp/__init__.py

```python
# path: promptbench-mcp-server/promptbench_mcp/__init__.py
__all__ = ["__version__"]

__version__ = "0.1.0"
```

---

## promptbench-mcp-server/promptbench_mcp/security.py

```python
# path: promptbench-mcp-server/promptbench_mcp/security.py
from __future__ import annotations

import os
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable


DEFAULT_MAX_BYTES: int = 2_000_000  # 2MB safety limit for file reads


@dataclass(frozen=True)
class RootPolicy:
    """Defines which filesystem roots are allowed for read/write operations."""

    roots: tuple[Path, ...]

    @staticmethod
    def from_env(env_var: str = "PROMPTBENCH_MCP_ROOTS") -> "RootPolicy":
        raw = os.environ.get(env_var, "").strip()
        if not raw:
            # Default to current working directory.
            return RootPolicy(roots=(Path.cwd().resolve(),))
        parts = [p.strip() for p in raw.split(os.pathsep) if p.strip()]
        roots = tuple(Path(p).expanduser().resolve() for p in parts)
        return RootPolicy(roots=roots)


def _is_relative_to(candidate: Path, root: Path) -> bool:
    try:
        candidate.resolve().is_relative_to(root)
        return True
    except Exception:
        return False


def resolve_under_roots(path: str | Path, policy: RootPolicy) -> Path:
    p = Path(path).expanduser().resolve()
    for r in policy.roots:
        if _is_relative_to(p, r):
            return p
    raise ValueError(
        f"Path not allowed: {p}. Set PROMPTBENCH_MCP_ROOTS to allow it."
    )


def ensure_dir(path: Path) -> None:
    path.mkdir(parents=True, exist_ok=True)


def safe_read_text(path: Path, *, max_bytes: int = DEFAULT_MAX_BYTES, encoding: str = "utf-8") -> str:
    st = path.stat()
    if st.st_size > max_bytes:
        raise ValueError(f"Refusing to read {st.st_size} bytes (limit={max_bytes}).")
    return path.read_text(encoding=encoding)


def safe_write_text(
    path: Path,
    content: str,
    *,
    overwrite: bool,
    encoding: str = "utf-8",
) -> None:
    if path.exists() and not overwrite:
        raise ValueError(
            f"Refusing to overwrite existing file: {path}. Set overwrite=true to replace it."
        )
    ensure_dir(path.parent)
    path.write_text(content, encoding=encoding)


def normalize_selection_list(items: Iterable[str] | None) -> list[str]:
    if not items:
        return []
    out: list[str] = []
    for x in items:
        s = (x or "").strip()
        if s:
            out.append(s)
    return out
```

---

## promptbench-mcp-server/promptbench_mcp/promptbench_cli.py

```python
# path: promptbench-mcp-server/promptbench_mcp/promptbench_cli.py
from __future__ import annotations

import asyncio
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable

from .security import RootPolicy, resolve_under_roots


@dataclass(frozen=True)
class SubprocessResult:
    stdout: str
    stderr: str
    exit_code: int


async def _run_python_module(
    module: str,
    args: list[str],
    *,
    timeout_s: float,
    cwd: Path | None,
    env: dict[str, str] | None,
    max_stdout_bytes: int = 5_000_000,
    max_stderr_bytes: int = 2_000_000,
) -> SubprocessResult:
    """Run `python3 -m <module> ...` safely (no shell)."""

    proc = await asyncio.create_subprocess_exec(
        "python3",
        "-m",
        module,
        *args,
        cwd=str(cwd) if cwd else None,
        env=env,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )

    try:
        stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
    except asyncio.TimeoutError:
        proc.kill()
        await proc.communicate()
        raise TimeoutError(f"Subprocess timed out after {timeout_s}s")

    # Truncate aggressively to avoid context blowups.
    stdout_b = stdout_b[:max_stdout_bytes]
    stderr_b = stderr_b[:max_stderr_bytes]

    return SubprocessResult(
        stdout=stdout_b.decode("utf-8", errors="replace"),
        stderr=stderr_b.decode("utf-8", errors="replace"),
        exit_code=int(proc.returncode or 0),
    )


def _json_loads_or_error(text: str, *, label: str) -> Any:
    try:
        return json.loads(text)
    except json.JSONDecodeError as e:
        preview = text[:5000]
        raise ValueError(f"Failed to parse {label} as JSON: {e}. Output preview: {preview}")


def _csv(items: Iterable[str] | None) -> str | None:
    if not items:
        return None
    cleaned = [x.strip() for x in items if x and x.strip()]
    return ",".join(cleaned) if cleaned else None


async def promptbench_doctor(*, config_path: str | None, workspace_path: str | None, timeout_s: float) -> list[dict[str, Any]]:
    args: list[str] = ["doctor", "--json"]
    if config_path:
        args.extend(["--config", config_path])
    elif workspace_path:
        args.extend(["--path", workspace_path])
    else:
        raise ValueError("Either config_path or workspace_path is required")

    res = await _run_python_module("promptbench", args, timeout_s=timeout_s, cwd=None, env=None)
    if res.exit_code != 0 and not res.stdout.strip():
        raise RuntimeError(res.stderr.strip() or "promptbench doctor failed")

    return _json_loads_or_error(res.stdout, label="doctor output")


async def promptbench_discover(*, config_path: str, force_refresh: bool, timeout_s: float) -> dict[str, Any]:
    args = ["discover", "--config", config_path]
    if force_refresh:
        args.append("--force-refresh")

    res = await _run_python_module("promptbench", args, timeout_s=timeout_s, cwd=None, env=None)
    if res.exit_code != 0:
        raise RuntimeError(res.stderr.strip() or res.stdout.strip() or "promptbench discover failed")

    return _json_loads_or_error(res.stdout, label="discover output")


async def promptbench_matrix(*, config_path: str, force_refresh: bool, timeout_s: float) -> dict[str, Any]:
    args = ["matrix", "--config", config_path]
    if force_refresh:
        args.append("--force-refresh")

    res = await _run_python_module("promptbench", args, timeout_s=timeout_s, cwd=None, env=None)
    if res.exit_code != 0:
        raise RuntimeError(res.stderr.strip() or res.stdout.strip() or "promptbench matrix failed")

    return _json_loads_or_error(res.stdout, label="matrix output")


async def promptbench_run(
    *,
    config_path: str,
    output: str | None,
    limit: int | None,
    concurrency: int | None,
    providers: list[str] | None,
    templates: list[str] | None,
    payloads: list[str] | None,
    skills: list[str] | None,
    force_refresh: bool,
    timeout_s: float,
    event_log: str = "-",
) -> tuple[list[dict[str, Any]], SubprocessResult]:
    """Runs promptbench with JSONL events to stdout.

    Returns: (events, subprocess_result)
    """
    args: list[str] = ["--config", config_path, "--event-log", event_log]

    if output:
        args.extend(["--output", output])
    if limit is not None:
        args.extend(["--limit", str(limit)])
    if concurrency is not None:
        args.extend(["--concurrency", str(concurrency)])

    p_csv = _csv(providers)
    if p_csv:
        args.extend(["--providers", p_csv])

    t_csv = _csv(templates)
    if t_csv:
        args.extend(["--templates", t_csv])

    pl_csv = _csv(payloads)
    if pl_csv:
        args.extend(["--payloads", pl_csv])

    s_csv = _csv(skills)
    if s_csv:
        args.extend(["--skills", s_csv])

    if force_refresh:
        args.append("--force-refresh")

    res = await _run_python_module("promptbench", args, timeout_s=timeout_s, cwd=None, env=None)

    # stdout is expected to be JSONL events when event_log == '-'.
    events: list[dict[str, Any]] = []
    for line in res.stdout.splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            events.append(json.loads(line))
        except json.JSONDecodeError:
            # If promptbench ever prints non-JSON to stdout, keep a sentinel record.
            events.append({"type": "non_json_stdout", "text": line[:500]})

    return events, res


def validate_config_path(config_path: str, policy: RootPolicy) -> Path:
    p = resolve_under_roots(config_path, policy)
    if not p.exists():
        raise ValueError(f"Config not found: {p}")
    return p


def validate_workspace_path(workspace_path: str, policy: RootPolicy) -> Path:
    p = resolve_under_roots(workspace_path, policy)
    if not p.exists():
        raise ValueError(f"Workspace path not found: {p}")
    return p


def validate_any_path(path: str, policy: RootPolicy) -> Path:
    return resolve_under_roots(path, policy)
```

---

## promptbench-mcp-server/promptbench_mcp/prompt_templates.py

```python
# path: promptbench-mcp-server/promptbench_mcp/prompt_templates.py
from __future__ import annotations

from dataclasses import dataclass


PROMPT_ENGINEER_PAYLOAD_TEMPLATE = """We plan to use {user_idea} for our `{project_in_question}` tool. {additional_information}

Not always provided, inference from context or ask additional questions seeking what information you need to answer effective.

Our pain point: {pain_point}

```md {reference_file_name}
{reference_file_content}
```

### What we need from the prompt-engineer
- optimized prompt that will get the {target_agent} to find us a solution for adding capabilities to our existing {project} tool giving it {capability}

Our question to the prompt-engineer: {optimize_prompt}
"""


@dataclass(frozen=True)
class PromptEngineerPayload:
    markdown: str


def render_prompt_engineer_payload(
    *,
    user_idea: str,
    project_in_question: str,
    additional_information: str | None,
    pain_point: str | None,
    reference_file_name: str,
    reference_file_content: str,
    target_agent: str,
    project: str,
    capability: str,
    optimize_prompt: str,
) -> PromptEngineerPayload:
    ai = (additional_information or "").strip()
    pp = (pain_point or "").strip()

    if not ai:
        ai = ""
    if not pp:
        pp = "(not provided)"

    md = PROMPT_ENGINEER_PAYLOAD_TEMPLATE.format(
        user_idea=user_idea.strip(),
        project_in_question=project_in_question.strip(),
        additional_information=ai.strip(),
        pain_point=pp.strip(),
        reference_file_name=reference_file_name.strip(),
        reference_file_content=reference_file_content.rstrip(),
        target_agent=target_agent.strip(),
        project=project.strip(),
        capability=capability.strip(),
        optimize_prompt=optimize_prompt.strip(),
    )

    # Minor normalization to avoid accidental trailing whitespace walls.
    md = "\n".join([line.rstrip() for line in md.splitlines()]).strip() + "\n"

    return PromptEngineerPayload(markdown=md)
```

---

## promptbench-mcp-server/promptbench_mcp/models.py

```python
# path: promptbench-mcp-server/promptbench_mcp/models.py
from __future__ import annotations

from pathlib import Path
from typing import Any, Literal

from pydantic import BaseModel, Field


class DoctorInput(BaseModel):
    config_path: str | None = Field(default=None, description="Path to promptbench TOML config")
    workspace_path: str | None = Field(default=None, description="Workspace base directory")
    timeout_s: float = Field(default=10.0, ge=0.5, le=120.0)


class MatrixInput(BaseModel):
    config_path: str = Field(description="Path to promptbench TOML config")
    force_refresh: bool = Field(default=False)
    timeout_s: float = Field(default=20.0, ge=0.5, le=300.0)


class DiscoverInput(BaseModel):
    config_path: str = Field(description="Path to promptbench TOML config")
    force_refresh: bool = Field(default=False)
    timeout_s: float = Field(default=20.0, ge=0.5, le=300.0)


class ReadFileInput(BaseModel):
    path: str = Field(description="Path to read (must be under allowlisted roots)")
    max_bytes: int = Field(default=200000, ge=1, le=2_000_000)


class RenderPromptEngineerPayloadInput(BaseModel):
    user_idea: str
    project_in_question: str
    additional_information: str | None = None
    pain_point: str | None = None

    reference_file_name: str = Field(default="reference.md")
    reference_file_content: str | None = Field(default=None, description="Raw markdown to embed")
    reference_file_path: str | None = Field(default=None, description="Read markdown from this file")

    target_agent: str
    project: str
    capability: str
    optimize_prompt: str


class WritePayloadInput(BaseModel):
    config_path: str = Field(description="Path to promptbench TOML config (used to locate workspace)")
    filename: str = Field(description="Filename to write (e.g. prompt-engineer.md)")
    content: str = Field(description="Markdown content to write")
    overwrite: bool = Field(default=False)


class RunInput(BaseModel):
    config_path: str = Field(description="Path to promptbench TOML config")

    output: str | None = Field(default=None, description="Override output root directory")
    limit: int | None = Field(default=None, ge=1)
    concurrency: int | None = Field(default=None, ge=1, le=256)

    providers: list[str] | None = None
    templates: list[str] | None = None
    payloads: list[str] | None = None
    skills: list[str] | None = None

    force_refresh: bool = Field(default=False)

    timeout_s: float = Field(default=3600.0, ge=1.0, le=24 * 3600.0)

    max_events: int = Field(default=2000, ge=0, le=10000)
    events_tail: int = Field(default=200, ge=0, le=2000)


class RunResult(BaseModel):
    exit_code: int
    stderr_tail: str
    events_tail: list[dict[str, Any]]
    events_count: int


class WritePayloadResult(BaseModel):
    path: str
    bytes_written: int
```

---

## promptbench-mcp-server/promptbench_mcp/server.py

```python
# path: promptbench-mcp-server/promptbench_mcp/server.py
from __future__ import annotations

import argparse
from pathlib import Path
from typing import Any

from mcp.server.fastmcp import FastMCP, Context

from .models import (
    DoctorInput,
    MatrixInput,
    DiscoverInput,
    ReadFileInput,
    RenderPromptEngineerPayloadInput,
    WritePayloadInput,
    WritePayloadResult,
    RunInput,
    RunResult,
)
from .prompt_templates import render_prompt_engineer_payload
from .promptbench_cli import (
    promptbench_doctor,
    promptbench_discover,
    promptbench_matrix,
    promptbench_run,
    validate_any_path,
    validate_config_path,
    validate_workspace_path,
)
from .security import RootPolicy, safe_read_text, safe_write_text


mcp = FastMCP("promptbench_mcp")


def _policy(ctx: Context | None = None) -> RootPolicy:
    # Context exists for future extension (e.g., per-user policy).
    return RootPolicy.from_env()


def _workspace_from_config(config_path: Path) -> Path:
    # promptbench stores sets under: <workspace>/.promptbench/{templates,payloads,skills}
    return config_path.resolve().parent


def _payload_dir(workspace_root: Path) -> Path:
    return workspace_root / ".promptbench" / "payloads" / "_mcp"


@mcp.tool(
    name="promptbench_doctor",
    annotations={
        "title": "Promptbench: Doctor (workspace validation)",
        "readOnlyHint": True,
        "destructiveHint": False,
        "idempotentHint": True,
        "openWorldHint": False,
    },
)
async def tool_promptbench_doctor(params: DoctorInput, ctx: Context) -> list[dict[str, Any]]:
    """Validate a promptbench workspace and return diagnostics.

    Provide either `config_path` (preferred) or `workspace_path`.
    """
    policy = _policy(ctx)

    config_path = None
    workspace_path = None

    if params.config_path:
        config_path = str(validate_config_path(params.config_path, policy))
    if params.workspace_path:
        workspace_path = str(validate_workspace_path(params.workspace_path, policy))

    return await promptbench_doctor(
        config_path=config_path,
        workspace_path=workspace_path,
        timeout_s=params.timeout_s,
    )


@mcp.tool(
    name="promptbench_discover",
    annotations={
        "title": "Promptbench: Discover (preflight)",
        "readOnlyHint": True,
        "destructiveHint": False,
        "idempotentHint": True,
        "openWorldHint": False,
    },
)
async def tool_promptbench_discover(params: DiscoverInput, ctx: Context) -> dict[str, Any]:
    """Run `promptbench discover` and return the JSON preflight payload."""
    policy = _policy(ctx)
    _ = validate_config_path(params.config_path, policy)
    return await promptbench_discover(
        config_path=params.config_path,
        force_refresh=params.force_refresh,
        timeout_s=params.timeout_s,
    )


@mcp.tool(
    name="promptbench_matrix",
    annotations={
        "title": "Promptbench: Matrix (inputs + sets)",
        "readOnlyHint": True,
        "destructiveHint": False,
        "idempotentHint": True,
        "openWorldHint": False,
    },
)
async def tool_promptbench_matrix(params: MatrixInput, ctx: Context) -> dict[str, Any]:
    """Return the promptbench matrix JSON (templates/payloads/skills/providers + set files).

    This is equivalent to running: `python3 -m promptbench matrix --config <path>`.
    """
    policy = _policy(ctx)
    _ = validate_config_path(params.config_path, policy)
    return await promptbench_matrix(
        config_path=params.config_path,
        force_refresh=params.force_refresh,
        timeout_s=params.timeout_s,
    )


@mcp.tool(
    name="promptbench_read_file",
    annotations={
        "title": "Promptbench: Read file (allowlisted roots)",
        "readOnlyHint": True,
        "destructiveHint": False,
        "idempotentHint": True,
        "openWorldHint": False,
    },
)
async def tool_promptbench_read_file(params: ReadFileInput, ctx: Context) -> str:
    """Read a local file (restricted to allowlisted roots) and return its text."""
    policy = _policy(ctx)
    p = validate_any_path(params.path, policy)
    return safe_read_text(p, max_bytes=params.max_bytes)


@mcp.tool(
    name="promptbench_render_prompt_engineer_payload",
    annotations={
        "title": "Promptbench: Render prompt-engineer payload (markdown)",
        "readOnlyHint": True,
        "destructiveHint": False,
        "idempotentHint": True,
        "openWorldHint": False,
    },
)
async def tool_render_prompt_engineer_payload(params: RenderPromptEngineerPayloadInput, ctx: Context) -> str:
    """Render your prompt-engineer payload template to markdown.

    If `reference_file_path` is provided, the server will read it (must be allowlisted).
    Otherwise, provide `reference_file_content`.
    """
    policy = _policy(ctx)

    reference_content = params.reference_file_content
    if params.reference_file_path:
        p = validate_any_path(params.reference_file_path, policy)
        reference_content = safe_read_text(p)

    if reference_content is None:
        reference_content = ""

    rendered = render_prompt_engineer_payload(
        user_idea=params.user_idea,
        project_in_question=params.project_in_question,
        additional_information=params.additional_information,
        pain_point=params.pain_point,
        reference_file_name=params.reference_file_name,
        reference_file_content=reference_content,
        target_agent=params.target_agent,
        project=params.project,
        capability=params.capability,
        optimize_prompt=params.optimize_prompt,
    )
    return rendered.markdown


@mcp.tool(
    name="promptbench_write_payload",
    annotations={
        "title": "Promptbench: Write payload file into workspace",
        "readOnlyHint": False,
        "destructiveHint": True,
        "idempotentHint": False,
        "openWorldHint": False,
    },
)
async def tool_promptbench_write_payload(params: WritePayloadInput, ctx: Context) -> WritePayloadResult:
    """Write a payload markdown file under `<workspace>/.promptbench/payloads/_mcp/`.

    This is intentionally scoped so agents can create new payloads without arbitrary filesystem access.
    """
    policy = _policy(ctx)
    config_p = validate_config_path(params.config_path, policy)
    workspace_root = _workspace_from_config(config_p)

    # Ensure computed target stays under allowlisted roots.
    _ = validate_any_path(str(workspace_root), policy)

    target_dir = _payload_dir(workspace_root)
    target_path = (target_dir / params.filename).resolve()

    # Enforce that writes stay in the target dir.
    if target_dir.resolve() not in target_path.parents:
        raise ValueError("Invalid filename (path traversal detected)")

    safe_write_text(target_path, params.content, overwrite=params.overwrite)
    return WritePayloadResult(path=str(target_path), bytes_written=len(params.content.encode("utf-8")))


@mcp.tool(
    name="promptbench_run",
    annotations={
        "title": "Promptbench: Run selected jobs",
        "readOnlyHint": False,
        "destructiveHint": False,
        "idempotentHint": False,
        "openWorldHint": True,
    },
)
async def tool_promptbench_run(params: RunInput, ctx: Context) -> RunResult:
    """Run promptbench for the selected templates/payloads/skills/providers.

    Uses `--event-log -` so stdout is JSONL events; stderr is logs.

    Notes:
    - This tool may call external model providers.
    - It writes run artifacts into the configured output root.
    """
    policy = _policy(ctx)
    _ = validate_config_path(params.config_path, policy)

    events, proc = await promptbench_run(
        config_path=params.config_path,
        output=params.output,
        limit=params.limit,
        concurrency=params.concurrency,
        providers=params.providers,
        templates=params.templates,
        payloads=params.payloads,
        skills=params.skills,
        force_refresh=params.force_refresh,
        timeout_s=params.timeout_s,
        event_log="-",
    )

    # Avoid returning huge event streams by default.
    events_count = len(events)
    tail_n = min(params.events_tail, events_count)
    events_tail = events[-tail_n:] if tail_n > 0 else []

    # Keep a short stderr tail for debugging.
    stderr_tail = (proc.stderr or "")[-4000:]

    return RunResult(
        exit_code=proc.exit_code,
        stderr_tail=stderr_tail,
        events_tail=events_tail,
        events_count=events_count,
    )


def main() -> None:
    parser = argparse.ArgumentParser(description="promptbench MCP server")
    parser.add_argument(
        "--transport",
        default="stdio",
        choices=["stdio", "streamable_http"],
        help="Transport to use",
    )
    parser.add_argument("--host", default="127.0.0.1", help="Bind host (streamable_http)")
    parser.add_argument("--port", type=int, default=8765, help="Bind port (streamable_http)")

    args = parser.parse_args()

    if args.transport == "stdio":
        mcp.run()
    else:
        mcp.run(transport="streamable_http", host=args.host, port=args.port)


if __name__ == "__main__":
    main()
```

---

## promptbench-mcp-server/tests/test_security.py

```python
# path: promptbench-mcp-server/tests/test_security.py
from __future__ import annotations

import os
from pathlib import Path

import pytest

from promptbench_mcp.security import RootPolicy, resolve_under_roots


def test_resolve_under_roots_allows_within_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    root = tmp_path / "root"
    root.mkdir()
    child = root / "child" / "file.txt"
    child.parent.mkdir()
    child.write_text("x")

    monkeypatch.setenv("PROMPTBENCH_MCP_ROOTS", str(root))
    policy = RootPolicy.from_env()

    resolved = resolve_under_roots(child, policy)
    assert resolved == child.resolve()


def test_resolve_under_roots_denies_outside_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    root = tmp_path / "root"
    root.mkdir()

    outside = tmp_path / "outside.txt"
    outside.write_text("x")

    monkeypatch.setenv("PROMPTBENCH_MCP_ROOTS", str(root))
    policy = RootPolicy.from_env()

    with pytest.raises(ValueError):
        _ = resolve_under_roots(outside, policy)
```



--- .taskmaster/AGENTS.md ---
# Task Master AI - Agent Integration Guide

## Essential Commands

### Core Workflow Commands

```bash
# Project Setup
task-master init                                    # Initialize Task Master in current project
task-master parse-prd .taskmaster/docs/prd.md       # Generate tasks from PRD document
task-master models --setup                        # Configure AI models interactively

# Daily Development Workflow
task-master list                                   # Show all tasks with status
task-master next                                   # Get next available task to work on
task-master show <id>                             # View detailed task information (e.g., task-master show 1.2)
task-master set-status --id=<id> --status=done    # Mark task complete

# Task Management
task-master add-task --prompt="description" --research        # Add new task with AI assistance
task-master expand --id=<id> --research --force              # Break task into subtasks
task-master update-task --id=<id> --prompt="changes"         # Update specific task
task-master update --from=<id> --prompt="changes"            # Update multiple tasks from ID onwards
task-master update-subtask --id=<id> --prompt="notes"        # Add implementation notes to subtask

# Analysis & Planning
task-master analyze-complexity --research          # Analyze task complexity
task-master complexity-report                      # View complexity analysis
task-master expand --all --research               # Expand all eligible tasks

# Dependencies & Organization
task-master add-dependency --id=<id> --depends-on=<id>       # Add task dependency
task-master move --from=<id> --to=<id>                       # Reorganize task hierarchy
task-master validate-dependencies                            # Check for dependency issues
task-master generate                                         # Update task markdown files (usually auto-called)
```

## Key Files & Project Structure

### Core Files

- `.taskmaster/tasks/tasks.json` - Main task data file (auto-managed)
- `.taskmaster/config.json` - AI model configuration (use `task-master models` to modify)
- `.taskmaster/docs/prd.md` - Product Requirements Document for parsing (`.md` extension recommended for better editor support)
- `.taskmaster/tasks/*.txt` - Individual task files (auto-generated from tasks.json)
- `.env` - API keys for CLI usage

**PRD File Format:** While both `.txt` and `.md` extensions work, **`.md` is recommended** because:
- Markdown syntax highlighting in editors improves readability
- Proper rendering when previewing in VS Code, GitHub, or other tools
- Better collaboration through formatted documentation

### Claude Code Integration Files

- `CLAUDE.md` - Auto-loaded context for Claude Code (this file)
- `.claude/settings.json` - Claude Code tool allowlist and preferences
- `.claude/commands/` - Custom slash commands for repeated workflows
- `.mcp.json` - MCP server configuration (project-specific)

### Directory Structure

```
project/
├── .taskmaster/
│   ├── tasks/              # Task files directory
│   │   ├── tasks.json      # Main task database
│   │   ├── task-1.md      # Individual task files
│   │   └── task-2.md
│   ├── docs/              # Documentation directory
│   │   ├── prd.md         # Product requirements (.md recommended)
│   ├── reports/           # Analysis reports directory
│   │   └── task-complexity-report.json
│   ├── templates/         # Template files
│   │   └── example_prd.md  # Example PRD template (.md recommended)
│   └── config.json        # AI models & settings
├── .claude/
│   ├── settings.json      # Claude Code configuration
│   └── commands/         # Custom slash commands
├── .env                  # API keys
├── .mcp.json            # MCP configuration
└── CLAUDE.md            # This file - auto-loaded by Claude Code
```

## MCP Integration

Task Master provides an MCP server that Claude Code can connect to. Configure in `.mcp.json`:

```json
{
  "mcpServers": {
    "task-master-ai": {
      "command": "npx",
      "args": ["-y", "task-master-ai"],
      "env": {
        "TASK_MASTER_TOOLS": "core",
        "ANTHROPIC_API_KEY": "your_key_here",
        "PERPLEXITY_API_KEY": "your_key_here",
        "OPENAI_API_KEY": "OPENAI_API_KEY_HERE",
        "GOOGLE_API_KEY": "GOOGLE_API_KEY_HERE",
        "XAI_API_KEY": "XAI_API_KEY_HERE",
        "OPENROUTER_API_KEY": "OPENROUTER_API_KEY_HERE",
        "MISTRAL_API_KEY": "MISTRAL_API_KEY_HERE",
        "AZURE_OPENAI_API_KEY": "AZURE_OPENAI_API_KEY_HERE",
        "OLLAMA_API_KEY": "OLLAMA_API_KEY_HERE"
      }
    }
  }
}
```

### MCP Tool Tiers

Default: `core` (7 tools). Set via `TASK_MASTER_TOOLS` env var.

| Tier | Count | Tools |
|------|-------|-------|
| `core` | 7 | `get_tasks`, `next_task`, `get_task`, `set_task_status`, `update_subtask`, `parse_prd`, `expand_task` |
| `standard` | 14 | core + `initialize_project`, `analyze_project_complexity`, `expand_all`, `add_subtask`, `remove_task`, `add_task`, `complexity_report` |
| `all` | 44+ | standard + dependencies, tags, research, autopilot, scoping, models, rules |

**Upgrade when tool unavailable:** Edit MCP config, change `TASK_MASTER_TOOLS` from `"core"` to `"standard"` or `"all"`, restart MCP.

### Essential MCP Tools

```javascript
help; // = shows available taskmaster commands
// Project setup
initialize_project; // = task-master init
parse_prd; // = task-master parse-prd

// Daily workflow
get_tasks; // = task-master list
next_task; // = task-master next
get_task; // = task-master show <id>
set_task_status; // = task-master set-status

// Task management
add_task; // = task-master add-task
expand_task; // = task-master expand
update_task; // = task-master update-task
update_subtask; // = task-master update-subtask
update; // = task-master update

// Analysis
analyze_project_complexity; // = task-master analyze-complexity
complexity_report; // = task-master complexity-report
```

## Claude Code Workflow Integration

### Standard Development Workflow

#### 1. Project Initialization

```bash
# Initialize Task Master
task-master init

# Create or obtain PRD, then parse it (use .md extension for better editor support)
task-master parse-prd .taskmaster/docs/prd.md

# Analyze complexity and expand tasks
task-master analyze-complexity --research
task-master expand --all --research
```

If tasks already exist, another PRD can be parsed (with new information only!) using parse-prd with --append flag. This will add the generated tasks to the existing list of tasks..

#### 2. Daily Development Loop

```bash
# Start each session
task-master next                           # Find next available task
task-master show <id>                     # Review task details

# During implementation, check in code context into the tasks and subtasks
task-master update-subtask --id=<id> --prompt="implementation notes..."

# Complete tasks
task-master set-status --id=<id> --status=done
```

#### 3. Multi-Claude Workflows

For complex projects, use multiple Claude Code sessions:

```bash
# Terminal 1: Main implementation
cd project && claude

# Terminal 2: Testing and validation
cd project-test-worktree && claude

# Terminal 3: Documentation updates
cd project-docs-worktree && claude
```

### Custom Slash Commands

Create `.claude/commands/taskmaster-next.md`:

```markdown
Find the next available Task Master task and show its details.

Steps:

1. Run `task-master next` to get the next task
2. If a task is available, run `task-master show <id>` for full details
3. Provide a summary of what needs to be implemented
4. Suggest the first implementation step
```

Create `.claude/commands/taskmaster-complete.md`:

```markdown
Complete a Task Master task: $ARGUMENTS

Steps:

1. Review the current task with `task-master show $ARGUMENTS`
2. Verify all implementation is complete
3. Run any tests related to this task
4. Mark as complete: `task-master set-status --id=$ARGUMENTS --status=done`
5. Show the next available task with `task-master next`
```

## Tool Allowlist Recommendations

Add to `.claude/settings.json`:

```json
{
  "allowedTools": [
    "Edit",
    "Bash(task-master *)",
    "Bash(git commit:*)",
    "Bash(git add:*)",
    "Bash(npm run *)",
    "mcp__task_master_ai__*"
  ]
}
```

## Configuration & Setup

### API Keys Required

At least **one** of these API keys must be configured:

- `ANTHROPIC_API_KEY` (Claude models) - **Recommended**
- `PERPLEXITY_API_KEY` (Research features) - **Highly recommended**
- `OPENAI_API_KEY` (GPT models)
- `GOOGLE_API_KEY` (Gemini models)
- `MISTRAL_API_KEY` (Mistral models)
- `OPENROUTER_API_KEY` (Multiple models)
- `XAI_API_KEY` (Grok models)

An API key is required for any provider used across any of the 3 roles defined in the `models` command.

### Model Configuration

```bash
# Interactive setup (recommended)
task-master models --setup

# Set specific models
task-master models --set-main claude-3-5-sonnet-20241022
task-master models --set-research perplexity-llama-3.1-sonar-large-128k-online
task-master models --set-fallback gpt-4o-mini
```

## Task Structure & IDs

### Task ID Format

- Main tasks: `1`, `2`, `3`, etc.
- Subtasks: `1.1`, `1.2`, `2.1`, etc.
- Sub-subtasks: `1.1.1`, `1.1.2`, etc.

### Task Status Values

- `pending` - Ready to work on
- `in-progress` - Currently being worked on
- `done` - Completed and verified
- `deferred` - Postponed
- `cancelled` - No longer needed
- `blocked` - Waiting on external factors

### Task Fields

```json
{
  "id": "1.2",
  "title": "Implement user authentication",
  "description": "Set up JWT-based auth system",
  "status": "pending",
  "priority": "high",
  "dependencies": ["1.1"],
  "details": "Use bcrypt for hashing, JWT for tokens...",
  "testStrategy": "Unit tests for auth functions, integration tests for login flow",
  "subtasks": []
}
```

## Claude Code Best Practices with Task Master

### Context Management

- Use `/clear` between different tasks to maintain focus
- This CLAUDE.md file is automatically loaded for context
- Use `task-master show <id>` to pull specific task context when needed

### Iterative Implementation

1. `task-master show <subtask-id>` - Understand requirements
2. Explore codebase and plan implementation
3. `task-master update-subtask --id=<id> --prompt="detailed plan"` - Log plan
4. `task-master set-status --id=<id> --status=in-progress` - Start work
5. Implement code following logged plan
6. `task-master update-subtask --id=<id> --prompt="what worked/didn't work"` - Log progress
7. `task-master set-status --id=<id> --status=done` - Complete task

### Complex Workflows with Checklists

For large migrations or multi-step processes:

1. Create a markdown PRD file describing the new changes: `touch task-migration-checklist.md` (prds can be .txt or .md)
2. Use Taskmaster to parse the new prd with `task-master parse-prd --append` (also available in MCP)
3. Use Taskmaster to expand the newly generated tasks into subtasks. Consdier using `analyze-complexity` with the correct --to and --from IDs (the new ids) to identify the ideal subtask amounts for each task. Then expand them.
4. Work through items systematically, checking them off as completed
5. Use `task-master update-subtask` to log progress on each task/subtask and/or updating/researching them before/during implementation if getting stuck

### Git Integration

Task Master works well with `gh` CLI:

```bash
# Create PR for completed task
gh pr create --title "Complete task 1.2: User authentication" --body "Implements JWT auth system as specified in task 1.2"

# Reference task in commits
git commit -m "feat: implement JWT auth (task 1.2)"
```

### Parallel Development with Git Worktrees

```bash
# Create worktrees for parallel task development
git worktree add ../project-auth feature/auth-system
git worktree add ../project-api feature/api-refactor

# Run Claude Code in each worktree
cd ../project-auth && claude    # Terminal 1: Auth work
cd ../project-api && claude     # Terminal 2: API work
```

## Troubleshooting

### AI Commands Failing

```bash
# Check API keys are configured
cat .env                           # For CLI usage

# Verify model configuration
task-master models

# Test with different model
task-master models --set-fallback gpt-4o-mini
```

### MCP Connection Issues

- Check `.mcp.json` configuration
- Verify Node.js installation
- Use `--mcp-debug` flag when starting Claude Code
- Use CLI as fallback if MCP unavailable

### Task File Sync Issues

```bash
# Regenerate task files from tasks.json
task-master generate

# Fix dependency issues
task-master fix-dependencies
```

DO NOT RE-INITIALIZE. That will not do anything beyond re-adding the same Taskmaster core files.

## Important Notes

### AI-Powered Operations

These commands make AI calls and may take up to a minute:

- `parse_prd` / `task-master parse-prd`
- `analyze_project_complexity` / `task-master analyze-complexity`
- `expand_task` / `task-master expand`
- `expand_all` / `task-master expand --all`
- `add_task` / `task-master add-task`
- `update` / `task-master update`
- `update_task` / `task-master update-task`
- `update_subtask` / `task-master update-subtask`

### File Management

- Never manually edit `tasks.json` - use commands instead
- Never manually edit `.taskmaster/config.json` - use `task-master models`
- Task markdown files in `tasks/` are auto-generated
- Run `task-master generate` after manual changes to tasks.json

### Claude Code Session Management

- Use `/clear` frequently to maintain focused context
- Create custom slash commands for repeated Task Master workflows
- Configure tool allowlist to streamline permissions
- Use headless mode for automation: `claude -p "task-master next"`

### Multi-Task Updates

- Use `update --from=<id>` to update multiple future tasks
- Use `update-task --id=<id>` for single task updates
- Use `update-subtask --id=<id>` for implementation logging

### Research Mode

- Add `--research` flag for research-based AI enhancement
- Requires a research model API key like Perplexity (`PERPLEXITY_API_KEY`) in environment
- Provides more informed task creation and updates
- Recommended for complex technical tasks

---

_This guide ensures Claude Code has immediate access to Task Master's essential functionality for agentic development workflows._


--- README.md ---
# prompt-bench

A systematic prompt benchmark runner for comparing system templates, user payloads, and optional "skills" across multiple local and CLI model providers.

## Overview

`prompt-bench` provides a repeatable way to run a Cartesian product of **System Templates × User Payloads × Optional Skills × Providers**. It acts as a "Fused Prompt Generator," taking a skeleton house, an influence prompt, and a target prompt to forge a single, unique, optimized output.

## Key Features

- **Triple-Layer Recursive Fusion**:
  - **The House (Skeleton)**: The outer meta-instruction (Template).
  - **The Influence (Primary)**: The style/optimization driver (Skill).
  - **The Target (Secondary)**: The core content to be transformed (Payload).
  - In `engine` mode, these are fused recursively by replacing instruction lines verbatim.
- **Multi-Mode Composition Engine**:
  - **Engine**: Strict spec-compliant recursive last-line replacement.
  - **Marker**: Standard `{{USER_PROMPT}}` variable replacement.
  - **Append**: Simple end-of-file concatenation.
- **Multi-Provider Support**:
  - **LM Studio**: OpenAI-compatible HTTP adapter with system-only fallback and auto-model unloading.
  - **Codex CLI**: Standardized execution of the `codex` tool.
  - **Gemini CLI**: Headless execution with optional JSON output parsing.
- **Atomic Artifact Capture**: Writes a dedicated directory per job containing the composed prompt, raw response, and a structured `output.json`.
- **Inputs Manifest**: Each run includes a lightweight `inputs.json` with template/payload/skill metadata.
- **Markdown Output Option**: Save assistant output as `output.md` when preferred.
- **Event Stream (JSONL)**: Optional lifecycle events for jobs and artifacts, used by the TUI.

## Installation & Setup

Ensure you have [uv](https://github.com/astral-sh/uv) installed. If you plan to build the TUI, install Go 1.24+.

### Automated Setup (Recommended)

```bash
git clone <repo-url>
cd prompt-bench
./scripts/setup.sh
```

This will install dependencies, create necessary directories, and generate sample input files.

### Run a Demo

```bash
./scripts/run_demo.sh
```

Demonstrates the artifact generation and recursive fusion logic using a local mock provider.

---

## Usage

### 1. Configure `config.toml` ([config example](demo_config.toml))

```toml
[inputs]
# Outer Skeleton
templates = "templates/prompt-composition-engine.md"
# Influence/Primary Snippets
skills = "skills/*.md"
# Target/Secondary Content
payloads = "payloads/*.md"

[runner]
composition_mode = "engine"
concurrency = 4
# Optional: use eventing executor without --event-log (JSONL emitted to stdout)
use_eventing_executor = true
# Optional: write assistant output to output.md instead of output.txt
write_markdown_output = true

[skill_router]
# Auto-map skills based on template/payload text.
enabled = true
# Override where skills are discovered (default: ~/.codex/skills)
skills_path = "~/.codex/skills"
# Force specific skills to always attach
force_skills = ["python"]
```

### 2. Run the Benchmark

```bash
uv run promptbench --config config.toml
```

### 3. Optional: emit an event stream

```bash
uv run promptbench --config config.toml --event-log -
```

You can also enable the eventing executor via config with `runner.use_eventing_executor = true`.

### 4. Optional: preflight discovery

```bash
uv run promptbench discover --config config.toml
```

---

## Composition Modes

### `engine` (Recommended for Prompt Generation)

Strictly follows the `templates/prompt-composition-engine.md` spec with a recursive twist:

1. **Inner Fusion**: The **Skill** (Influence) last line is replaced with `User prompt: [Payload Content]`.
2. **Outer Fusion**: The **Template** (House) last line is replaced with `User prompt: [Fused Skill Content]`.
3. **Preservation**: All trailing blank lines and verbatim characters are preserved.

### `marker`

Replaces the string `{{USER_PROMPT}}` in the template with the skill-injected payload.

### `append`

Adds a newline, the string `User prompt:`, and then the payload to the very end of the template.

---

## Artifact Structure

For every job, a directory is created in the output root:

```md
runs/
├── YYYY-MM-DD_HH-MM-SS-ffffff_<job_id>/
│   ├── composed_system.txt   # The final recursive prompt sent to the model
│   ├── output.txt            # The verbatim assistant response text (or output.md)
│   ├── output.json           # The raw JSON response (if available)
│   ├── inputs.json           # Template/payload/skill inputs for quick inspection
│   └── run.json              # Metadata, duration, and job-specific logs
└── summary.json              # Batch summary of all executed jobs
```

## TUI

The TUI lives under `tui/` as its own Go module.

Build and install (Linux/WSL + Windows exe) using the helper script:

```bash
scripts/build_install_promptbench-tui.sh
```

Then launch:

```bash
promptbench tui
```

### TUI Advanced Options

From the setup screen press `a` to open advanced options. You can override:

- output directory
- job limit
- concurrency
- provider list

Press `s` to save the overrides to a sidecar file: `<config>.tui.json`. This file is reloaded automatically when you select the same config.

## Development

### Running Tests

```bash
uv run pytest tests/
```

## License

MIT


## Links discovered
- [uv](https://github.com/astral-sh/uv)
- [config example](https://github.com/AcidicSoil/promptbench/blob/main/demo_config.toml)

--- scripts/skill_context_router.py ---
#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import os
from pathlib import Path

from promptbench.utils.skill_context import resolve_context


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Resolve nested skill context and scenario attachments.")
    parser.add_argument("--skills", default="", help="Comma-separated skill names or paths")
    parser.add_argument("--skills-dir", default="", help="Override skills root (default: ~/.codex/skills or $CODEX_SKILLS_DIR)")
    parser.add_argument("--context-map", default="", help="Optional JSON/TOML context map file")
    parser.add_argument("--category", default="", help="Category selector for context map")
    parser.add_argument("--reference", default="", help="Reference selector for context map")
    parser.add_argument("--horizon", default="", help="Horizon selector for context map")
    parser.add_argument("--tags", default="", help="Comma-separated tags for context map")
    return parser.parse_args()


def main() -> int:
    args = parse_args()
    skills_root = args.skills_dir or os.environ.get("CODEX_SKILLS_DIR", "~/.codex/skills")
    skills_root_path = Path(skills_root).expanduser()
    selectors = [s.strip() for s in args.skills.split(",") if s.strip()]
    context_map = Path(args.context_map).expanduser() if args.context_map else None
    tags = [t.strip() for t in args.tags.split(",") if t.strip()]

    resolution = resolve_context(
        selectors,
        skills_root_path,
        context_map=context_map,
        category=args.category or None,
        reference=args.reference or None,
        horizon=args.horizon or None,
        tags=tags,
    )

    payload = {
        "skills": [{"name": skill.name, "path": str(skill.path)} for skill in resolution.skills],
        "context_files": [str(path) for path in resolution.context_files],
        "nested_files": [str(path) for path in resolution.nested_files],
        "scenario_files": [str(path) for path in resolution.scenario_files],
        "missing": resolution.missing,
    }
    print(json.dumps(payload, indent=2))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())


--- skills/when-to-use-this-skill.md ---

Produce “when to use this skill” documentation + examples for developers, aligned to the Agent Skills concept.

References (must be cited/derived from these pages only):

- <https://developers.openai.com/codex/skills/>
- <https://agentskills.io/home>

Output structure (markdown):

1) Summary (2–4 bullets): what this skill is for and what it improves
2) Optimal usage moments: map to lifecycle phases
3) Example library (minimum 12 examples), presented as a table with columns:
   - Phase
   - Goal
   - Inputs required (files, constraints, audience, format)
   - Raw prompt (before)
   - Optimized prompt (after) — include placeholders like {context}, {inputs}, {constraints}, {acceptance_criteria}, {format}, {deadline}
   - Acceptance criteria (how to judge the optimized prompt is “good”)
4) Non-goals / anti-patterns: 5 cases where the skill should NOT be used

Rules:

- Keep tone developer-neutral and execution-oriented.
- Use realistic SDLC phases and include at least 2 examples per major phase.


--- payloads/haiku.md ---
Write a haiku about a rusty robot in a garden.


--- payloads/transcript-cleaner_v2.md ---
SYSTEM: Conversation-to-Issue-Ticket Converter

Role

- Convert an exported user↔assistant conversation into a single cleaned, coherent issue ticket.
- Preserve all key details. Remove noise. Do not add new facts.

Input

- An exported conversation containing alternating messages from “user” and “assistant” (any format: plain text, markdown, JSON-like, timestamps optional).

Core rules

- Fidelity: Do not invent requirements, causes, decisions, timelines, metrics, or outcomes.
- Completeness: Keep every materially relevant detail (goals, constraints, edge cases, decisions, rejected options, action items, dependencies, risks).
- No questions: Do not ask the reader for missing info. If information is missing, mark it explicitly as “Unknown” or “Not provided”.
- De-duplication: Merge repeats and restatements. Keep the clearest formulation.
- Conflict handling: If the conversation contradicts itself, report both versions and attribute them (User vs Assistant). Do not resolve by guessing.
- Terminology: Normalize names and terms (features, components, people, systems) using the most consistent wording from the conversation.
- Traceability: When a detail is critical or ambiguous, include a short attributed quote fragment (≤25 words) or “(per user)” / “(per assistant)”.
- Security/privacy: Keep secrets out. If the conversation includes credentials or sensitive personal data, redact and note “[REDACTED]”.
- Output only the issue ticket. No meta-commentary.

Process

1) Parse and segment the conversation into: problem statement(s), context, requirements, constraints, environment, attempted fixes, errors, decisions, next steps.
2) Identify:
   - Primary issue
   - Secondary issues (if present)
   - Stakeholders/owners (if stated)
   - Target system/component
   - Impact and urgency signals
3) Extract concrete artifacts:
   - Steps to reproduce
   - Expected vs actual behavior
   - Error messages/logs
   - Links, IDs, filenames, code snippets (lightly cleaned; preserve meaning)
4) Produce one consolidated, coherent narrative and a structured ticket.

Output format (strict)
Title:

- Concise, specific, action-oriented. Keep short.

Summary:

- 2–5 sentences describing what’s wrong / needed, who is affected, and why it matters.

Background / Context:

- Relevant history and constraints from the conversation.

Current Behavior (Actual):

- Bullet list. Include symptoms, observed outputs, error text.

Expected Behavior:

- Bullet list. Clear success definition.

Requirements:

- Bullet list of explicit requirements extracted from the conversation.
- Include constraints (performance, compatibility, compliance, UX, scope limits).

Out of Scope:

- Bullet list of exclusions stated or implied by the conversation. If none, “Not provided”.

Reproduction Steps:

- Numbered steps. If not available, “Not provided”.

Environment:

- OS, app version, browser, device, deployment, flags, configs. Use “Unknown” when missing.

Evidence:

- Logs/errors (verbatim), screenshots/attachments references, links, file paths, IDs.

Decisions / Agreements:

- Bullet list of decisions made in the conversation, with attribution where needed.

Open Items / Unknowns:

- Bullet list of missing info that blocks execution. No questions; just state unknowns.

Risks / Dependencies:

- Bullet list of dependencies, integrations, approvals, or known risks mentioned.

Acceptance Criteria:

- Testable checklist statements. Derive from requirements and expected behavior.
- If requirements are vague, translate into minimal testable criteria without adding new scope.

Priority & Severity (if inferable from text):

- Priority: P0–P3
- Severity: S0–S3
- Only infer if conversation provides clear cues; otherwise “Not provided”.

Labels (optional):

- 3–8 tags (e.g., bug, enhancement, auth, ui, performance). Only if supported by conversation.

Style constraints

- Use crisp bullet points. No filler.
- Prefer concrete nouns/verbs over abstract phrasing.
- Keep ticket self-contained and understandable without reading the conversation.


--- promptbench/cli.py ---
import argparse
import sys
import os
import json
import shutil
import signal
import subprocess
import time
from datetime import datetime, timezone
from pathlib import Path

from promptbench.core.config import load_config, apply_overrides
from promptbench.core.discovery import discover_files_cached
from promptbench.core.events import EventType, RunEvent, emit_event
from promptbench.core.sets import discover_sets, resolve_selections
from promptbench.core.validation import diagnostics_payload, validate_workspace
from promptbench.core.workspace import ensure_workspace_structure, ensure_set_dirs, workspace_paths
from promptbench.runner.matrix import build_jobs
from promptbench.runner.executor import run_jobs
from promptbench.runner.eventing_executor import run_jobs_with_events
from promptbench.runner.summary import build_summary, write_summary
from promptbench.providers.lmstudio import LMStudioAdapter
from promptbench.providers.codex_cli import CodexCLIAdapter
from promptbench.providers.gemini_cli import GeminiCLIAdapter


def _should_allow_outside_root(globs, base_dir: Path) -> bool:
    for pattern in globs:
        if not pattern:
            continue
        expanded = Path(pattern).expanduser()
        candidate = expanded if expanded.is_absolute() else (base_dir / expanded)
        try:
            if not candidate.resolve().is_relative_to(base_dir):
                return True
        except ValueError:
            return True
    return False


def _resolve_workspace_glob(value: str | None, base_dir: Path, category: str) -> str | None:
    if not value:
        return value
    raw = value.strip()
    if raw in ("@workspace", "workspace"):
        paths = workspace_paths(base_dir)
        target = getattr(paths, category, None)
        if target is None:
            return value
        return str(target / "**/*")
    if raw.startswith("workspace:"):
        _, _, name = raw.partition(":")
        if name.strip() == category:
            paths = workspace_paths(base_dir)
            target = getattr(paths, category, None)
            if target is None:
                return value
            return str(target / "**/*")
    return value


def _build_run_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="promptbench - Systematic prompt benchmark runner")
    parser.add_argument("--config", "-c", required=True, help="Path to TOML config file")
    parser.add_argument("--output", "-o", help="Override output root directory")
    parser.add_argument("--limit", "-l", type=int, help="Limit number of jobs to run")
    parser.add_argument("--concurrency", type=int, help="Number of concurrent jobs")
    parser.add_argument("--providers", help="Comma-separated list of provider IDs to run")
    parser.add_argument("--templates", help="Comma-separated list of template files to run")
    parser.add_argument("--payloads", help="Comma-separated list of payload files to run")
    parser.add_argument("--skills", help="Comma-separated list of skill files to run")
    parser.add_argument("--event-log", help="Write JSONL event stream to file, or '-' for stdout")
    parser.add_argument("--force-refresh", action="store_true", help="Force discovery refresh (ignore cache)")
    return parser


def _emit_run_event(event_sink, run_id, event_type, job_id="", provider_id="", payload=None):
    emit_event(
        event_sink,
        RunEvent(
            ts=datetime.now(timezone.utc),
            type=event_type,
            run_id=run_id,
            job_id=job_id,
            provider_id=provider_id,
            payload=payload or {},
        ),
    )


def _run_benchmark(argv) -> int:
    parser = _build_run_parser()
    args = parser.parse_args(argv)

    event_sink = None
    run_id = None
    log_stream = sys.stdout

    try:
        # 1. Load config
        if not os.path.exists(args.config):
            print(f"Error: Config file not found: {args.config}", file=sys.stderr)
            return 1

        config = load_config(args.config)

        # 2. Apply overrides
        overrides = {}
        if args.output:
            overrides["output_root"] = args.output
        if args.concurrency:
            overrides["concurrency"] = args.concurrency

        config = apply_overrides(config, overrides)

        use_eventing = config.use_eventing_executor or bool(args.event_log)
        if use_eventing:
            if args.event_log:
                if args.event_log == "-":
                    event_sink = sys.stdout
                else:
                    event_sink = open(args.event_log, "w", encoding="utf-8")
            else:
                event_sink = sys.stdout
            run_id = f"run-{int(time.time())}"

        log_stream = sys.stderr if event_sink is sys.stdout else sys.stdout

        if event_sink:
            _emit_run_event(
                event_sink,
                run_id,
                EventType.CONFIG_LOADED,
                payload={"config": args.config},
            )

        # 3. Discover files
        config_dir = Path(args.config).resolve().parent
        config.templates_glob = _resolve_workspace_glob(config.templates_glob, config_dir, "templates")
        config.payloads_glob = _resolve_workspace_glob(config.payloads_glob, config_dir, "payloads")
        if config.skills_glob:
            config.skills_glob = _resolve_workspace_glob(config.skills_glob, config_dir, "skills")

        templates = discover_files_cached(
            [config.templates_glob],
            base_dir=config_dir,
            allow_outside_root=_should_allow_outside_root([config.templates_glob], config_dir),
            force_refresh=args.force_refresh,
        )
        payloads = discover_files_cached(
            [config.payloads_glob],
            base_dir=config_dir,
            allow_outside_root=_should_allow_outside_root([config.payloads_glob], config_dir),
            force_refresh=args.force_refresh,
        )
        skills = []
        if config.skills_glob:
            skills = discover_files_cached(
                [config.skills_glob],
                base_dir=config_dir,
                allow_outside_root=_should_allow_outside_root([config.skills_glob], config_dir),
                force_refresh=args.force_refresh,
            )

        if args.templates:
            templates = _parse_input_list(args.templates, config_dir, category="templates")
        if args.payloads:
            payloads = _parse_input_list(args.payloads, config_dir, category="payloads")
        if args.skills:
            skills = _parse_input_list(args.skills, config_dir, category="skills")

        if event_sink:
            _emit_run_event(
                event_sink,
                run_id,
                EventType.DISCOVERY_COMPLETE,
                payload={
                    "templates": len(templates),
                    "payloads": len(payloads),
                    "skills": len(skills),
                    "providers": len(config.providers),
                },
            )

        # 4. Build jobs
        provider_filter = args.providers.split(",") if args.providers else None
        jobs = build_jobs(
            templates,
            payloads,
            skills,
            config.providers,
            provider_filter=provider_filter,
            limit=args.limit,
            base_dir=str(config_dir),
        )

        if not jobs:
            print("No jobs to run.", file=log_stream)
            return 0

        print(f"Executing {len(jobs)} jobs...", file=log_stream)

        # 5. Initialize adapters
        adapters = {}
        for p in config.providers:
            if p.type == "lmstudio":
                adapters[p.id] = LMStudioAdapter(p)
            elif p.type == "codex_cli":
                adapters[p.id] = CodexCLIAdapter(p)
            elif p.type == "gemini_cli":
                adapters[p.id] = GeminiCLIAdapter(p)
            else:
                print(f"Warning: Unknown provider type '{p.type}' for provider '{p.id}'")

        # 6. Run jobs
        if event_sink:
            results = run_jobs_with_events(jobs, config, adapters, event_sink, run_id)
        else:
            results = run_jobs(jobs, config, adapters)

        # 7. Cleanup / Unload models
        print("Cleaning up resources...", file=log_stream)
        for adapter_id, adapter in adapters.items():
            try:
                adapter.unload()
            except Exception as e:
                print(f"Warning: Failed to unload provider '{adapter_id}': {e}", file=log_stream)

        # 8. Generate summary
        summary = build_summary(config, results)
        summary_path = os.path.join(config.output_root, "summary.json")
        write_summary(summary, summary_path)

        print(f"Done! Summary written to {summary_path}", file=log_stream)
        print(f"Stats: {summary['stats']['ok']} ok, {summary['stats']['error']} error", file=log_stream)

        return 0

    except Exception as e:
        print(f"Unexpected error: {e}", file=sys.stderr)
        return 1
    finally:
        if event_sink and event_sink is not sys.stdout:
            event_sink.close()


def _discover(argv) -> int:
    parser = argparse.ArgumentParser(description="promptbench discover - preflight input discovery")
    parser.add_argument("--config", "-c", required=True, help="Path to TOML config file")
    parser.add_argument("--force-refresh", action="store_true", help="Force discovery refresh (ignore cache)")
    args = parser.parse_args(argv)

    if not os.path.exists(args.config):
        print(f"Error: Config file not found: {args.config}", file=sys.stderr)
        return 1

    config = load_config(args.config)

    config_dir = Path(args.config).resolve().parent
    config.templates_glob = _resolve_workspace_glob(config.templates_glob, config_dir, "templates")
    config.payloads_glob = _resolve_workspace_glob(config.payloads_glob, config_dir, "payloads")
    if config.skills_glob:
        config.skills_glob = _resolve_workspace_glob(config.skills_glob, config_dir, "skills")

    templates = discover_files_cached(
        [config.templates_glob],
        base_dir=config_dir,
        allow_outside_root=_should_allow_outside_root([config.templates_glob], config_dir),
        force_refresh=args.force_refresh,
    )
    payloads = discover_files_cached(
        [config.payloads_glob],
        base_dir=config_dir,
        allow_outside_root=_should_allow_outside_root([config.payloads_glob], config_dir),
        force_refresh=args.force_refresh,
    )
    skills = []
    if config.skills_glob:
        skills = discover_files_cached(
            [config.skills_glob],
            base_dir=config_dir,
            allow_outside_root=_should_allow_outside_root([config.skills_glob], config_dir),
            force_refresh=args.force_refresh,
        )

    skills_count = len(skills) if skills else 0
    effective_skills = skills_count or 1
    jobs_count = len(templates) * len(payloads) * effective_skills * len(config.providers)

    payload = {
        "templates": len(templates),
        "payloads": len(payloads),
        "skills": skills_count,
        "providers": len(config.providers),
        "jobs": jobs_count,
        "output_root": config.output_root,
        "workspace": diagnostics_payload(config_dir),
    }

    print(json.dumps(payload))
    return 0


def _matrix(argv) -> int:
    parser = argparse.ArgumentParser(description="promptbench matrix - list inputs for TUI selection")
    parser.add_argument("--config", "-c", required=True, help="Path to TOML config file")
    parser.add_argument("--force-refresh", action="store_true", help="Force discovery refresh (ignore cache)")
    args = parser.parse_args(argv)

    if not os.path.exists(args.config):
        print(f"Error: Config file not found: {args.config}", file=sys.stderr)
        return 1

    config = load_config(args.config)
    config_dir = Path(args.config).resolve().parent

    config.templates_glob = _resolve_workspace_glob(config.templates_glob, config_dir, "templates")
    config.payloads_glob = _resolve_workspace_glob(config.payloads_glob, config_dir, "payloads")
    if config.skills_glob:
        config.skills_glob = _resolve_workspace_glob(config.skills_glob, config_dir, "skills")

    templates = discover_files_cached(
        [config.templates_glob],
        base_dir=config_dir,
        allow_outside_root=_should_allow_outside_root([config.templates_glob], config_dir),
        force_refresh=args.force_refresh,
    )
    payloads = discover_files_cached(
        [config.payloads_glob],
        base_dir=config_dir,
        allow_outside_root=_should_allow_outside_root([config.payloads_glob], config_dir),
        force_refresh=args.force_refresh,
    )
    skills = []
    if config.skills_glob:
        skills = discover_files_cached(
            [config.skills_glob],
            base_dir=config_dir,
            allow_outside_root=_should_allow_outside_root([config.skills_glob], config_dir),
            force_refresh=args.force_refresh,
        )

    providers = [
        {"id": p.id, "type": p.type, "model": p.model, "url": p.url}
        for p in config.providers
    ]

    template_sets = [s.__dict__ for s in discover_sets(config_dir, "templates")]
    payload_sets = [s.__dict__ for s in discover_sets(config_dir, "payloads")]
    skill_sets = [s.__dict__ for s in discover_sets(config_dir, "skills")]

    skills_count = len(skills) if skills else 0
    effective_skills = skills_count or 1
    jobs_count = len(templates) * len(payloads) * effective_skills * len(config.providers)

    payload = {
        "templates": templates,
        "payloads": payloads,
        "skills": skills,
        "template_sets": template_sets,
        "payload_sets": payload_sets,
        "skill_sets": skill_sets,
        "providers": providers,
        "output_root": config.output_root,
        "jobs": jobs_count,
    }

    print(json.dumps(payload))
    return 0


def _init_workspace(argv) -> int:
    parser = argparse.ArgumentParser(description="promptbench init - scaffold workspace")
    parser.add_argument("--path", help="Workspace base directory (default: cwd)")
    args = parser.parse_args(argv)

    base_dir = Path(args.path).expanduser().resolve() if args.path else Path.cwd().resolve()
    paths = ensure_workspace_structure(base_dir)
    ensure_set_dirs(base_dir, ["templates", "payloads", "skills"])

    sample_template = paths.templates / "example.md"
    if not sample_template.exists():
        sample_template.write_text("You are a helpful assistant.\n\n{{payload}}\n\n{{skill}}\n", encoding="utf-8")

    sample_payload = paths.payloads / "example.md"
    if not sample_payload.exists():
        sample_payload.write_text("Write a short greeting for the user.\n", encoding="utf-8")

    sample_skill = paths.skills / "example.md"
    if not sample_skill.exists():
        sample_skill.write_text("Keep responses under 50 words.\n", encoding="utf-8")

    template_set = paths.sets / "templates" / "default.txt"
    if not template_set.exists():
        template_set.write_text("templates/example.md\n", encoding="utf-8")

    payload_set = paths.sets / "payloads" / "default.txt"
    if not payload_set.exists():
        payload_set.write_text("payloads/example.md\n", encoding="utf-8")

    skill_set = paths.sets / "skills" / "default.txt"
    if not skill_set.exists():
        skill_set.write_text("skills/example.md\n", encoding="utf-8")

    print(f"Workspace ready at {paths.root}")
    return 0


def _doctor(argv) -> int:
    parser = argparse.ArgumentParser(description="promptbench doctor - validate workspace")
    parser.add_argument("--path", help="Workspace base directory (default: cwd)")
    parser.add_argument("--config", help="Path to TOML config file (uses its directory)")
    parser.add_argument("--json", action="store_true", help="Emit JSON diagnostics")
    args = parser.parse_args(argv)

    if args.config:
        base_dir = Path(args.config).expanduser().resolve().parent
    elif args.path:
        base_dir = Path(args.path).expanduser().resolve()
    else:
        base_dir = Path.cwd().resolve()

    diagnostics = validate_workspace(base_dir)
    has_errors = any(d.level == "error" for d in diagnostics)

    if args.json:
        print(json.dumps([d.__dict__ for d in diagnostics]))
    else:
        if not diagnostics:
            print("Workspace OK.")
        for diag in diagnostics:
            level = diag.level.upper()
            hint = f" (hint: {diag.hint})" if diag.hint else ""
            print(f"{level}: {diag.message}{hint}")

    return 1 if has_errors else 0


def _find_tui_binary() -> str | None:
    path = shutil.which("promptbench-tui")
    if path:
        return path

    local_candidate = Path(__file__).resolve().parent / "promptbench-tui"
    if local_candidate.exists() and local_candidate.is_file():
        return str(local_candidate)

    repo_candidate = Path(__file__).resolve().parents[1] / "tui" / "bin" / "promptbench-tui"
    if repo_candidate.exists() and repo_candidate.is_file():
        return str(repo_candidate)

    return None


def _run_tui(argv) -> int:
    binary = _find_tui_binary()
    if not binary:
        print("Error: promptbench-tui binary not found in PATH or repo.", file=sys.stderr)
        return 1

    proc = subprocess.Popen([binary] + argv)
    try:
        proc.wait()
    except KeyboardInterrupt:
        try:
            proc.send_signal(signal.SIGINT)
        except Exception:
            pass
        proc.wait()

    return proc.returncode or 0


def main():
    if len(sys.argv) > 1:
        cmd = sys.argv[1]
        if cmd == "discover":
            sys.exit(_discover(sys.argv[2:]))
        if cmd == "matrix":
            sys.exit(_matrix(sys.argv[2:]))
        if cmd == "init":
            sys.exit(_init_workspace(sys.argv[2:]))
        if cmd == "doctor":
            sys.exit(_doctor(sys.argv[2:]))
        if cmd == "tui":
            sys.exit(_run_tui(sys.argv[2:]))

    sys.exit(_run_benchmark(sys.argv[1:]))


if __name__ == "__main__":
    main()


def _parse_input_list(raw: str, base_dir: Path, category: str | None = None) -> list[str]:
    items = [item.strip() for item in raw.split(",") if item.strip()]
    if category:
        return resolve_selections(items, category, base_dir)
    resolved = []
    for item in items:
        path = Path(item).expanduser()
        if not path.is_absolute():
            path = (base_dir / path).resolve()
        resolved.append(str(path))
    return resolved


--- promptbench/__init__.py ---


--- promptbench/__main__.py ---
from .cli import main

if __name__ == '__main__':
    main()

--- promptbench/providers/base.py ---
from abc import ABC, abstractmethod
from dataclasses import dataclass
from promptbench.core.types import ProviderSpec, JobSpec, Config, ProviderResponse

@dataclass
class ProviderRunContext:
    composed_system: str
    config: Config
    job: JobSpec

class ProviderAdapter(ABC):
    def __init__(self, spec: ProviderSpec):
        self.spec = spec

    @abstractmethod
    def run(self, context: ProviderRunContext) -> ProviderResponse:
        """
        Execute the provider with the given context and return a standardized response.
        """
        pass

    def unload(self) -> bool:
        """
        (Optional) Cleanup or unload the model from memory.
        Returns True if successful or not needed.
        """
        return True

--- promptbench/providers/codex_cli.py ---
import subprocess
import os
from promptbench.providers.base import ProviderAdapter, ProviderRunContext
from promptbench.core.types import ProviderResponse

class CodexCLIAdapter(ProviderAdapter):
    def run(self, context: ProviderRunContext) -> ProviderResponse:
        cli_path = self.spec.args[0] if self.spec.args else "codex"
        # Codex exec semantics: PROMPT="-" means read from stdin
        env = os.environ.copy()
        env.update(self.spec.env)
        env["PROMPT"] = "-"
        
        try:
            cmd = [cli_path, "exec"]
            if len(self.spec.args) > 1:
                cmd.extend(self.spec.args[1:])
                
            process = subprocess.Popen(
                cmd,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
                env=env
            )
            
            stdout, stderr = process.communicate(input=context.composed_system)
            
            if process.returncode != 0:
                message = stderr.strip() or "Codex CLI returned non-zero exit code"
                return ProviderResponse(
                    stdout=stdout,
                    stderr=stderr,
                    error={
                        "code": "CODEX_CLI_ERROR",
                        "exit_code": process.returncode,
                        "message": message,
                    },
                )

            if not stdout.strip():
                return ProviderResponse(
                    stdout=stdout,
                    stderr=stderr,
                    error={
                        "code": "CODEX_CLI_EMPTY_OUTPUT",
                        "message": "Codex CLI returned empty output",
                    },
                )
            
            return ProviderResponse(
                text=stdout,
                stdout=stdout,
                stderr=stderr,
                meta={"exit_code": process.returncode}
            )
            
        except Exception as e:
            return ProviderResponse(
                error={"code": "CODEX_CLI_EXCEPTION", "message": str(e)}
            )


--- promptbench/core/compose.py ---
from enum import Enum
from typing import Optional
from promptbench.core.types import CompositionMode

class SkillInjectionMode(str, Enum):
    PREFIX = "prefix"
    SUFFIX = "suffix"
    NONE = "none"

def compose_system(
    template_text: str, 
    payload_text: str, 
    skill_text: Optional[str] = None,
    mode: CompositionMode = CompositionMode.MARKER
) -> str:
    """
    Build the composed system prompt. 
    In 'engine' mode, this performs recursive replacement:
    1. If skill exists, replace Skill's last line with 'User prompt: {payload}'.
    2. Replace Template's last line with 'User prompt: {Result of 1}'.
    """
    if mode == CompositionMode.REPLACE_LAST_LINE:
        # Step 1: Fuse Skill and Payload if Skill exists
        if skill_text and skill_text.lower() != "none":
            # The Skill is now the 'Primary' and the Payload is the 'Secondary'
            fused_inner = compose_engine_style(skill_text, payload_text)
            # Step 2: Fuse the House (Template) with the fused inner content
            return compose_engine_style(template_text, fused_inner)
        else:
            # No skill, just fuse House and Payload
            return compose_engine_style(template_text, payload_text)

    # Fallback to older logic for other modes
    effective_payload = payload_text
    if skill_text and skill_text.lower() != "none":
        effective_payload = inject_skill(payload_text, skill_text)

    if mode == CompositionMode.MARKER and "{{USER_PROMPT}}" in template_text:
        return template_text.replace("{{USER_PROMPT}}", effective_payload, 1)
    
    return f"{template_text}\nUser prompt:\n{effective_payload}"

def compose_engine_style(primary_text: str, secondary_text: str) -> str:
    """
    Implements the 'prompt-composition-engine' logic:
    1. Locates the last non-empty line in primary.
    2. Replaces it with 'User prompt: {secondary}'.
    3. Preserves trailing blank lines.
    """
    lines = primary_text.splitlines(keepends=True)
    if not lines:
        return f"User prompt: {secondary_text}"

    # Find the index of the last non-empty/non-whitespace line
    target_idx = -1
    for i in range(len(lines) - 1, -1, -1):
        if lines[i].strip():
            target_idx = i
            break
    
    if target_idx == -1: # All lines are empty or whitespace
        return f"User prompt: {secondary_text}\n" + "".join(lines)

    # Preserve the line ending if it had one
    original_line = lines[target_idx]
    newline = ""
    if original_line.endswith("\r\n"):
        newline = "\r\n"
    elif original_line.endswith("\n"):
        newline = "\n"

    # Replace the target line verbatim as per spec: "User prompt: {S}"
    lines[target_idx] = f"User prompt: {secondary_text}{newline}" 
    
    return "".join(lines)

def inject_skill(payload_text: str, skill_text: str, mode: SkillInjectionMode = SkillInjectionMode.PREFIX) -> str:
    """Standard non-engine skill injection."""
    if not skill_text or skill_text.lower() == "none":
        return payload_text
    delimiter = "\n---\n"
    if mode == SkillInjectionMode.PREFIX:
        return f"{skill_text}{delimiter}{payload_text}"
    elif mode == SkillInjectionMode.SUFFIX:
        return f"{payload_text}{delimiter}{skill_text}"
    else:
        return f"{skill_text}{payload_text}"

--- promptbench/core/config.py ---
import tomllib
import pathlib
from typing import Dict, Any, Optional
from promptbench.core.types import Config, ProviderSpec, CompositionMode, SkillRouterConfig
from promptbench.core.errors import ConfigError

def load_config(path: str) -> Config:
    """Load and validate TOML configuration."""
    try:
        with open(path, "rb") as f:
            data = tomllib.load(f)
    except Exception as e:
        raise ConfigError(f"Failed to parse TOML config: {path}", details={"error": str(e)})

    # Validation
    if "inputs" not in data:
        raise ConfigError("Missing 'inputs' section in config")
    
    inputs = data["inputs"]
    if "templates" not in inputs:
        raise ConfigError("Missing 'inputs.templates' in config")
    if "payloads" not in inputs:
        raise ConfigError("Missing 'inputs.payloads' in config")
    
    if "providers" not in data:
        raise ConfigError("Missing 'providers' section in config")
    if not data["providers"]:
        raise ConfigError("At least one provider must be defined")

    providers = []
    for p_id, p_data in data["providers"].items():
        if "type" not in p_data:
            raise ConfigError(f"Provider '{p_id}' missing 'type'")
        
        providers.append(ProviderSpec(
            id=p_id,
            type=p_data["type"],
            model=p_data.get("model"),
            url=p_data.get("url"),
            args=p_data.get("args", []),
            env=p_data.get("env", {})
        ))

    runner_cfg = data.get("runner", {}) or {}

    # Parse composition mode
    raw_mode = runner_cfg.get("composition_mode", "marker")
    try:
        comp_mode = CompositionMode(raw_mode)
    except ValueError:
        raise ConfigError(f"Invalid composition_mode: {raw_mode}. Expected one of: {[m.value for m in CompositionMode]}")

    router_cfg = data.get("skill_router", {}) or {}
    skill_router = SkillRouterConfig(
        enabled=bool(router_cfg.get("enabled", False)),
        skills_path=router_cfg.get("skills_path"),
        force_skills=list(router_cfg.get("force_skills", [])),
    )

    return Config(
        templates_glob=inputs["templates"],
        payloads_glob=inputs["payloads"],
        skills_glob=inputs.get("skills"),
        output_root=data.get("output", {}).get("root", "runs"),
        providers=providers,
        concurrency=runner_cfg.get("concurrency", 1),
        retries=runner_cfg.get("retries", 0),
        timeout=runner_cfg.get("timeout", 60),
        use_eventing_executor=runner_cfg.get("use_eventing_executor", False),
        composition_mode=comp_mode,
        write_markdown_output=bool(runner_cfg.get("write_markdown_output", False)),
        skill_router=skill_router,
    )

def apply_overrides(config: Config, overrides: Dict[str, Any]) -> Config:
    """Apply CLI overrides to the configuration object."""
    if "output_root" in overrides:
        config.output_root = overrides["output_root"]
    if "concurrency" in overrides:
        config.concurrency = overrides["concurrency"]
    if "retries" in overrides:
        config.retries = overrides["retries"]
    if "timeout" in overrides:
        config.timeout = int(overrides["timeout"])
    return config


--- promptbench/core/constants.py ---
RUN_JSON_SCHEMA_VERSION = 1


--- promptbench/core/contract.py ---
from __future__ import annotations

from typing import Any

from promptbench.core.types import ProviderResponse, ProviderSpec


class ContractViolationError(ValueError):
    pass


def _require_non_empty_string(value: Any, field: str) -> str:
    if not isinstance(value, str) or not value.strip():
        raise ContractViolationError(f"{field} must be a non-empty string")
    return value


def validate_response(response: ProviderResponse, spec: ProviderSpec) -> ProviderResponse:
    if not isinstance(response, ProviderResponse):
        raise ContractViolationError("Provider adapters must return ProviderResponse")

    if response.meta is None:
        response.meta = {}
    if not isinstance(response.meta, dict):
        raise ContractViolationError("ProviderResponse.meta must be a dict")

    response.meta.setdefault("provider_id", spec.id)
    response.meta.setdefault("provider_type", spec.type)

    if response.raw_json is not None and not isinstance(response.raw_json, dict):
        raise ContractViolationError("ProviderResponse.raw_json must be a dict when provided")

    if response.error is not None:
        if not isinstance(response.error, dict):
            raise ContractViolationError("ProviderResponse.error must be a dict when provided")
        _require_non_empty_string(response.error.get("code"), "error.code")
        _require_non_empty_string(response.error.get("message"), "error.message")
    else:
        if not isinstance(response.text, str):
            raise ContractViolationError("ProviderResponse.text must be a string")
        if not response.text.strip():
            raise ContractViolationError("ProviderResponse.text must be non-empty when error is None")

    return response


--- promptbench/core/discovery.py ---
import glob
import pathlib
import os
from typing import List, Optional, Union

from promptbench.core.discovery_cache import CacheManager, cache_key, _entry_is_fresh, _file_mtimes, _root_mtimes


def _is_within_base(path: pathlib.Path, base_dir: pathlib.Path) -> bool:
    try:
        path.resolve().relative_to(base_dir.resolve())
        return True
    except ValueError:
        return False

def discover_files(
    globs: List[str],
    filters: Optional[List[str]] = None,
    base_dir: Optional[Union[str, pathlib.Path]] = None,
    allow_outside_root: bool = False
) -> List[str]:
    """
    Discover files using one or more glob patterns.
    Returns a stable, alphabetically sorted list of file paths.
    Supports tilde (~) expansion for home directories.
    """
    base_path = pathlib.Path(base_dir).resolve() if base_dir else None
    paths = set()
    for pattern in globs:
        # Expand ~ to user home directory
        expanded_pattern = os.path.expanduser(pattern)
        
        # Expand globs recursively if ** is present
        for path in glob.glob(expanded_pattern, recursive=True):
            p = pathlib.Path(path)
            if p.is_file():
                if base_path and not allow_outside_root:
                    if not _is_within_base(p, base_path):
                        raise ValueError(f"Disallowed path outside base_dir: {p} (base_dir={base_path})")
                # Apply filters if provided (e.g., skip hidden files)
                if filters:
                    if any(f in p.name for f in filters):
                        continue
                paths.add(str(p))
    
    return sorted(list(paths))


def discover_files_cached(
    globs: List[str],
    filters: Optional[List[str]] = None,
    base_dir: Optional[Union[str, pathlib.Path]] = None,
    allow_outside_root: bool = False,
    cache: Optional[CacheManager] = None,
    force_refresh: bool = False,
) -> List[str]:
    base_path = pathlib.Path(base_dir).resolve() if base_dir else pathlib.Path.cwd().resolve()
    if cache is None:
        cache = CacheManager(base_path)
    key = cache_key(globs, base_path, filters)
    cache_data = cache.load()
    entry = cache_data.get(key)

    if entry and not force_refresh and _entry_is_fresh(entry, globs, base_path):
        return entry.get("paths", [])

    paths = discover_files(
        globs=globs,
        filters=filters,
        base_dir=base_path,
        allow_outside_root=allow_outside_root,
    )
    cache_data[key] = {
        "paths": paths,
        "mtimes": _file_mtimes(paths),
        "root_mtimes": _root_mtimes(globs, base_path),
    }
    cache.save(cache_data)
    return paths


--- templates/creative_writer.md ---
You are a world-class creative writer.
Your goal is to fulfill the user request with elegance and precision.

{{USER_PROMPT}}

End of response.


--- templates/prompt-composition-engine.md ---
```md
You are an AI prompt-composition engine.

GOAL
Compose a single SYSTEM message by combining a PRIMARY system template with a SECONDARY user request, then return the assistant reply produced when the composed SYSTEM message is used as the only message in a chat completion call.

INPUTS
- PRIMARY.md (string P): a system-prompt template that ends with an instruction line telling the model to optimize the next input.
- SECONDARY.md (string S): the full user request to be optimized.

TASK
1) Read PRIMARY.md as string P (preserve all characters and line breaks).
2) Read SECONDARY.md as string S (preserve all characters and line breaks).
3) In P, locate the final line (the last line of the file). Replace ONLY that final line with exactly:
   User prompt: {S}
   where {S} is the entire contents of SECONDARY.md inserted verbatim (no trimming, no escaping, no wrapping, no summarizing).
   - Keep every prior line of P unchanged.
   - Ensure there is a newline before "User prompt:" if P does not already end with one.
4) Construct SYSTEM = modified P from step 3.
5) Call a chat completion API using ONLY this single message:
   messages = [{ "role": "system", "content": SYSTEM }]
   - Do NOT add user/assistant messages.
   - Do NOT add tool messages.
   - Do NOT add extra instructions.
6) Return the resulting assistant reply EXACTLY as received:
   - No preface, no commentary, no markdown fences, no metadata, no citations.
   - Preserve the reply text verbatim.

VALIDATION / EDGE CASES
- If PRIMARY.md ends with trailing blank lines, treat the last non-empty line as the “final line” to be replaced, and preserve the trailing blank lines after replacement.
- If PRIMARY.md contains multiple occurrences of “User prompt:” or similar, ignore them; only replace the final line as defined above.
- If SECONDARY.md contains triple backticks, XML/JSON, or other delimiters, include them verbatim; do not escape.
- If PRIMARY.md has no newline at EOF, handle cleanly and still produce a valid combined string.

MINIMAL EXAMPLE (SCHEMATIC)
messages: [
  {
    "role": "system",
    "content": "<PRIMARY.md content, unchanged except final line replaced>\nUser prompt: <SECONDARY.md content verbatim>"
  }
]


```

--- templates/prompt-composition-engine_v3.md ---
You are a prompt-composition engine.

GOAL
Given a TEMPLATE (system template) and a PAYLOAD (user request), compose a single SYSTEM message by inserting PAYLOAD into TEMPLATE, then produce the assistant reply that would result if that composed SYSTEM were the only message in the chat.

INPUTS

- TEMPLATE.md (string T): system-prompt template text.
- PAYLOAD.md (string U): the full user request text.

COMPOSITION RULE

- TEMPLATE should contain exactly one insertion marker: {{USER_PROMPT}}
- If the marker exists: replace the first occurrence of {{USER_PROMPT}} with U verbatim.
- If the marker does not exist: append the following to the end of T (ensuring exactly one newline before it):
  User prompt:
  {U}
  where {U} is U verbatim.

TASK

1) Read T exactly (preserve all characters and line breaks).
2) Read U exactly (preserve all characters and line breaks).
3) Construct COMPOSED_SYSTEM using the COMPOSITION RULE.
4) Now act as the assistant with COMPOSED_SYSTEM as your only system message, and produce the response to the embedded user prompt.

OUTPUT
Return ONLY the assistant response text.

- No preface, no commentary, no metadata.
- Preserve the response exactly as generated.

VALIDATION / EDGE CASES

- Do not trim or escape any input.
- If TEMPLATE has trailing blank lines, preserve them.
- If U contains any delimiters (backticks, XML/JSON, etc.), include verbatim.


--- tests/test_artifacts.py ---
import pytest
import pathlib
import json
from promptbench.core.types import ProviderResponse
from promptbench.artifacts.layout import make_run_dir, get_artifact_paths
from promptbench.artifacts.writer import write_run_artifacts

def test_artifact_layout_and_writing(tmp_path):
    root = tmp_path / "runs"
    job_id = "test-job-123"
    
    run_dir = make_run_dir(str(root), job_id)
    paths = get_artifact_paths(run_dir)
    
    assert paths.root == run_dir
    assert job_id in paths.composed_system
    
    resp = ProviderResponse(
        text="Assistant reply",
        meta={"usage": 100},
        raw_json={"full": "data"}
    )
    
    write_run_artifacts(
        paths,
        composed_system="System prompt content",
        provider_response=resp,
        run_meta={"job_id": job_id, "duration": 1.5},
        inputs_manifest={
            "template": {"path": "t.md", "content": "T"},
            "payload": {"path": "p.md", "content": "P"},
            "skill": {"path": None, "selected": []},
        },
        created_at="2026-01-05T00:00:00+00:00",
    )
    
    # Verify files
    assert pathlib.Path(paths.composed_system).read_text() == "System prompt content"
    assert pathlib.Path(paths.output).read_text() == "Assistant reply"
    
    run_json_data = json.loads(pathlib.Path(paths.run_json).read_text())
    assert run_json_data["schema_version"] == 1
    assert run_json_data["meta"]["job_id"] == job_id
    assert run_json_data["provider"]["meta"]["usage"] == 100
    assert run_json_data["created_at"] == "2026-01-05T00:00:00+00:00"
    
    # We now call it output.json
    assert pathlib.Path(paths.output_json).exists()
    output_json_data = json.loads(pathlib.Path(paths.output_json).read_text())
    assert output_json_data == {"full": "data"}

    inputs_json_data = json.loads(pathlib.Path(paths.inputs_json).read_text())
    assert inputs_json_data["template"]["content"] == "T"
    assert inputs_json_data["created_at"] == "2026-01-05T00:00:00+00:00"


--- tests/test_artifacts_concurrency.py ---
import json
from concurrent.futures import ThreadPoolExecutor

from promptbench.core.types import ProviderResponse
from promptbench.artifacts.layout import make_run_dir, get_artifact_paths
from promptbench.artifacts.writer import write_run_artifacts


def _write_artifacts(paths, job_id):
    resp = ProviderResponse(text="ok", meta={"job": job_id})
    write_run_artifacts(
        paths,
        composed_system=f"system-{job_id}",
        provider_response=resp,
        run_meta={"job_id": job_id},
        inputs_manifest={"template": {"path": "t", "content": ""}, "payload": {"path": "p", "content": ""}, "skill": {}},
    )


def test_write_run_artifacts_concurrent_distinct_dirs(tmp_path):
    root = tmp_path / "runs"
    job_ids = [f"job-{i}" for i in range(10)]
    paths_list = []
    for job_id in job_ids:
        run_dir = make_run_dir(str(root), job_id)
        paths_list.append((get_artifact_paths(run_dir), job_id))

    with ThreadPoolExecutor(max_workers=10) as executor:
        futures = [executor.submit(_write_artifacts, paths, job_id) for paths, job_id in paths_list]
        for f in futures:
            f.result()

    for paths, _ in paths_list:
        run_json = json.loads(open(paths.run_json, "r", encoding="utf-8").read())
        output_json = json.loads(open(paths.output_json, "r", encoding="utf-8").read())
        inputs_json = json.loads(open(paths.inputs_json, "r", encoding="utf-8").read())
        assert "meta" in run_json
        assert isinstance(output_json, dict)
        assert "template" in inputs_json


def test_write_run_artifacts_concurrent_same_dir(tmp_path):
    root = tmp_path / "runs"
    run_dir = make_run_dir(str(root), "shared-job")
    paths = get_artifact_paths(run_dir)

    with ThreadPoolExecutor(max_workers=20) as executor:
        futures = [executor.submit(_write_artifacts, paths, f"job-{i}") for i in range(20)]
        for f in futures:
            f.result()

    run_json = json.loads(open(paths.run_json, "r", encoding="utf-8").read())
    output_json = json.loads(open(paths.output_json, "r", encoding="utf-8").read())
    inputs_json = json.loads(open(paths.inputs_json, "r", encoding="utf-8").read())
    assert "meta" in run_json
    assert isinstance(output_json, dict)
    assert "template" in inputs_json


--- tests/test_cli.py ---
import subprocess
import os
import json
from pathlib import Path
from unittest.mock import patch, MagicMock
import sys

def test_cli_smoke_run(tmp_path):
    # Setup directories
    (tmp_path / "templates").mkdir()
    (tmp_path / "payloads").mkdir()
    (tmp_path / "runs").mkdir()
    
    (tmp_path / "templates" / "t1.md").write_text("T: {{USER_PROMPT}}")
    (tmp_path / "payloads" / "p1.md").write_text("P")
    
    config_content = f"""
[inputs]
templates = "{tmp_path}/templates/*.md"
payloads = "{tmp_path}/payloads/*.md"

[output]
root = "{tmp_path}/runs"

[providers.mock]
type = "lmstudio"
url = "http://non-existent"
"""
    config_path = tmp_path / "config.toml"
    config_path.write_text(config_content)
    
    # Run CLI using module mode
    result = subprocess.run(
        ["python3", "-m", "promptbench", "--config", str(config_path)],
        capture_output=True,
        text=True,
        env={**os.environ, "PYTHONPATH": "."}
    )
    
    # The summary should be written even if providers fail
    summary_path = tmp_path / "runs" / "summary.json"
    assert summary_path.exists(), f"Summary file not found. stdout: {result.stdout}, stderr: {result.stderr}"
    
    summary = json.loads(summary_path.read_text())
    assert summary["stats"]["total_jobs"] == 1
    # It should have 1 error because LM Studio non-existent
    assert summary["stats"]["error"] == 1
    assert "Cleaning up resources..." in result.stdout

def test_cli_cleanup_is_called():
    # This is harder to test with subprocess.run without mocking the inner main.
    # We'll just verify the print statement in the smoke test.
    pass

def test_cli_uses_eventing_executor_from_config(tmp_path, monkeypatch):
    from promptbench.core.types import JobSpec, RunResult, RunStatus
    import promptbench.cli as cli

    (tmp_path / "templates").mkdir()
    (tmp_path / "payloads").mkdir()
    (tmp_path / "runs").mkdir()
    (tmp_path / "templates" / "t1.md").write_text("T: {{USER_PROMPT}}")
    (tmp_path / "payloads" / "p1.md").write_text("P")

    config_content = f"""
[inputs]
templates = "{tmp_path}/templates/*.md"
payloads = "{tmp_path}/payloads/*.md"

[output]
root = "{tmp_path}/runs"

[runner]
use_eventing_executor = true

[providers.lms]
type = "lmstudio"
url = "http://localhost:1234"
"""
    config_path = tmp_path / "config.toml"
    config_path.write_text(config_content)

    dummy_job = JobSpec(
        id="job1",
        template_path=str(tmp_path / "templates" / "t1.md"),
        payload_path=str(tmp_path / "payloads" / "p1.md"),
        provider_id="lms",
    )

    called = {}

    def fake_build_jobs(*args, **kwargs):
        return [dummy_job]

    def fake_run_jobs(*args, **kwargs):
        raise AssertionError("run_jobs should not be called when use_eventing_executor is true")

    def fake_run_jobs_with_events(jobs, config, adapters, event_sink, run_id):
        called["event_sink"] = event_sink
        called["run_id"] = run_id
        return [RunResult(status=RunStatus.OK, duration=0.0)]

    class DummyAdapter:
        def __init__(self, spec):
            self.spec = spec
        def unload(self):
            return None

    monkeypatch.setattr(cli, "build_jobs", fake_build_jobs)
    monkeypatch.setattr(cli, "run_jobs", fake_run_jobs)
    monkeypatch.setattr(cli, "run_jobs_with_events", fake_run_jobs_with_events)
    monkeypatch.setattr(cli, "LMStudioAdapter", DummyAdapter)

    exit_code = cli._run_benchmark(["--config", str(config_path)])
    assert exit_code == 0
    assert called.get("event_sink") is sys.stdout
    assert called.get("run_id")


def test_discover_allows_explicit_outside_paths(tmp_path, capsys):
    import promptbench.cli as cli

    external = tmp_path.parent / "external_payloads"
    external.mkdir(parents=True, exist_ok=True)
    (external / "p1.md").write_text("P")

    (tmp_path / "templates").mkdir()
    (tmp_path / "templates" / "t1.md").write_text("T: {{USER_PROMPT}}")

    config_content = f"""
[inputs]
templates = "{tmp_path}/templates/*.md"
payloads = "{external}/*.md"

[providers.mock]
type = "lmstudio"
url = "http://localhost:1234"
"""
    config_path = tmp_path / "config.toml"
    config_path.write_text(config_content)

    exit_code = cli._discover(["--config", str(config_path)])
    assert exit_code == 0
    out = capsys.readouterr().out
    assert "\"payloads\": 1" in out


def test_matrix_lists_inputs(tmp_path, capsys):
    import promptbench.cli as cli

    (tmp_path / "templates").mkdir()
    (tmp_path / "payloads").mkdir()
    (tmp_path / "runs").mkdir()
    (tmp_path / "skills").mkdir()
    (tmp_path / "templates" / "t1.md").write_text("T")
    (tmp_path / "payloads" / "p1.md").write_text("P")
    (tmp_path / "skills" / "s1.md").write_text("S")

    config_content = f"""
[inputs]
templates = "{tmp_path}/templates/*.md"
payloads = "{tmp_path}/payloads/*.md"
skills = "{tmp_path}/skills/*.md"

[providers.mock]
type = "lmstudio"
url = "http://localhost:1234"
"""
    config_path = tmp_path / "config.toml"
    config_path.write_text(config_content)

    exit_code = cli._matrix(["--config", str(config_path)])
    assert exit_code == 0
    out = capsys.readouterr().out
    payload = json.loads(out)
    assert len(payload["templates"]) == 1
    assert len(payload["payloads"]) == 1
    assert len(payload["skills"]) == 1
    assert len(payload["providers"]) == 1


def test_run_uses_explicit_template_payload_list(tmp_path, monkeypatch):
    import promptbench.cli as cli
    from promptbench.core.types import JobSpec, RunResult, RunStatus

    (tmp_path / "templates").mkdir()
    (tmp_path / "payloads").mkdir()
    (tmp_path / "runs").mkdir()
    (tmp_path / "templates" / "t1.md").write_text("T1")
    (tmp_path / "templates" / "t2.md").write_text("T2")
    (tmp_path / "payloads" / "p1.md").write_text("P1")
    (tmp_path / "payloads" / "p2.md").write_text("P2")

    config_content = f"""
[inputs]
templates = "{tmp_path}/templates/*.md"
payloads = "{tmp_path}/payloads/*.md"

[output]
root = "{tmp_path}/runs"

[providers.lms]
type = "lmstudio"
url = "http://localhost:1234"
"""
    config_path = tmp_path / "config.toml"
    config_path.write_text(config_content)

    observed = {}

    def fake_build_jobs(templates, payloads, skills, providers, **kwargs):
        observed["templates"] = list(templates)
        observed["payloads"] = list(payloads)
        return [JobSpec(id="job1", template_path=templates[0], payload_path=payloads[0], provider_id="lms")]

    def fake_run_jobs(*args, **kwargs):
        return [RunResult(status=RunStatus.OK, duration=0.0)]

    class DummyAdapter:
        def __init__(self, spec):
            self.spec = spec
        def unload(self):
            return None

    monkeypatch.setattr(cli, "build_jobs", fake_build_jobs)
    monkeypatch.setattr(cli, "run_jobs", fake_run_jobs)
    monkeypatch.setattr(cli, "LMStudioAdapter", DummyAdapter)

    exit_code = cli._run_benchmark([
        "--config", str(config_path),
        "--templates", str(tmp_path / "templates" / "t2.md"),
        "--payloads", str(tmp_path / "payloads" / "p2.md"),
    ])
    assert exit_code == 0
    assert observed["templates"] == [str(tmp_path / "templates" / "t2.md")]
    assert observed["payloads"] == [str(tmp_path / "payloads" / "p2.md")]


--- tests/test_cli_providers.py ---
import pytest
from unittest.mock import patch, MagicMock
from promptbench.core.types import ProviderSpec, JobSpec, Config
from promptbench.providers.base import ProviderRunContext
from promptbench.providers.codex_cli import CodexCLIAdapter
from promptbench.providers.gemini_cli import GeminiCLIAdapter

def test_codex_cli_success():
    spec = ProviderSpec(id="codex", type="codex_cli", args=["/usr/local/bin/codex"])
    adapter = CodexCLIAdapter(spec)
    
    ctx = ProviderRunContext(
        composed_system="Composed prompt",
        config=Config(templates_glob="", payloads_glob="", output_root="", providers=[]),
        job=JobSpec(id="j1", template_path="", payload_path="", provider_id="codex")
    )
    
    with patch("subprocess.Popen") as mock_popen:
        mock_proc = MagicMock()
        mock_proc.communicate.return_value = ("Success output", "")
        mock_proc.returncode = 0
        mock_popen.return_value = mock_proc
        
        resp = adapter.run(ctx)
        assert resp.text == "Success output"
        assert resp.error is None
        # Verify it called 'exec'
        assert mock_popen.call_args[0][0] == ["/usr/local/bin/codex", "exec"]

def test_gemini_cli_json_success():
    spec = ProviderSpec(id="gemini", type="gemini_cli", args=["gemini", "--output-format", "json"])
    adapter = GeminiCLIAdapter(spec)
    
    ctx = ProviderRunContext(
        composed_system="Prompt",
        config=Config(templates_glob="", payloads_glob="", output_root="", providers=[]),
        job=JobSpec(id="j1", template_path="", payload_path="", provider_id="gemini")
    )
    
    with patch("subprocess.Popen") as mock_popen:
        mock_proc = MagicMock()
        mock_proc.communicate.return_value = ('{"text": "Parsed output"}', "")
        mock_proc.returncode = 0
        mock_popen.return_value = mock_proc
        
        resp = adapter.run(ctx)
        assert resp.text == "Parsed output"
        assert resp.raw_json["text"] == "Parsed output"

def test_cli_failure_capture():
    spec = ProviderSpec(id="codex", type="codex_cli")
    adapter = CodexCLIAdapter(spec)
    
    ctx = ProviderRunContext(
        composed_system="Prompt",
        config=Config(templates_glob="", payloads_glob="", output_root="", providers=[]),
        job=JobSpec(id="j1", template_path="", payload_path="", provider_id="codex")
    )
    
    with patch("subprocess.Popen") as mock_popen:
        mock_proc = MagicMock()
        mock_proc.communicate.return_value = ("Partial output", "Fatal error")
        mock_proc.returncode = 1
        mock_popen.return_value = mock_proc
        
        resp = adapter.run(ctx)
        assert resp.error is not None
        assert resp.error["exit_code"] == 1
        assert resp.error["message"] == "Fatal error"


--- tests/test_compose.py ---
import pytest
from promptbench.core.compose import compose_system, inject_skill, SkillInjectionMode

def test_compose_marker_replacement():
    template = "System instructions.\n{{USER_PROMPT}}\nEnd instructions."
    payload = "Do this."
    expected = "System instructions.\nDo this.\nEnd instructions."
    assert compose_system(template, payload) == expected

def test_compose_marker_replacement_first_only():
    template = "{{USER_PROMPT}} again {{USER_PROMPT}}"
    payload = "FIRST"
    expected = "FIRST again {{USER_PROMPT}}"
    assert compose_system(template, payload) == expected

def test_compose_append_logic():
    template = "System instructions."
    payload = "Do this."
    result = compose_system(template, payload)
    assert result == "System instructions.\nUser prompt:\nDo this."

def test_compose_preserves_trailing_blank_lines():
    template = "Instructions.\n\n"
    payload = "Payload."
    result = compose_system(template, payload)
    # The \n in the f-string adds one newline
    assert result == "Instructions.\n\n\nUser prompt:\nPayload."

def test_inject_skill_prefix():
    payload = "User query."
    skill = "Search skill."
    result = inject_skill(payload, skill, mode=SkillInjectionMode.PREFIX)
    assert result == "Search skill.\n---\nUser query."

def test_inject_skill_none():
    payload = "User query."
    assert inject_skill(payload, "none") == payload
    assert inject_skill(payload, "") == payload


--- tests/test_config.py ---
import pytest
import pathlib
from promptbench.core.config import load_config, apply_overrides
from promptbench.core.errors import ConfigError

def test_load_valid_config(tmp_path):
    config_content = """
    [inputs]
    templates = "templates/*.md"
    payloads = "payloads/*.md"
    skills = "skills/*.md"

    [output]
    root = "custom_runs"

    [runner]
    concurrency = 5
    retries = 2
    use_eventing_executor = true
    write_markdown_output = true

    [skill_router]
    enabled = true
    skills_path = "~/.codex/skills"
    force_skills = ["python"]

    [providers.lms]
    type = "lmstudio"
    model = "gpt-4"
    url = "http://localhost:1234"
    """
    config_file = tmp_path / "config.toml"
    config_file.write_text(config_content)
    
    config = load_config(str(config_file))
    
    assert config.templates_glob == "templates/*.md"
    assert config.output_root == "custom_runs"
    assert config.concurrency == 5
    assert config.use_eventing_executor is True
    assert config.write_markdown_output is True
    assert config.skill_router.enabled is True
    assert config.skill_router.skills_path == "~/.codex/skills"
    assert config.skill_router.force_skills == ["python"]
    assert len(config.providers) == 1
    assert config.providers[0].id == "lms"

def test_load_invalid_config(tmp_path):
    # Missing required inputs
    config_content = """
    [providers.lms]
    type = "lmstudio"
    """
    config_file = tmp_path / "config.toml"
    config_file.write_text(config_content)
    
    with pytest.raises(ConfigError) as excinfo:
        load_config(str(config_file))
    assert "Missing 'inputs' section" in str(excinfo.value)

def test_apply_overrides():
    from promptbench.core.types import Config
    config = Config(templates_glob="t", payloads_glob="p", output_root="old", providers=[])
    
    apply_overrides(config, {"output_root": "new", "concurrency": 10})
    
    assert config.output_root == "new"
    assert config.concurrency == 10


--- tests/test_contract.py ---
import json
import pathlib

import pytest

from promptbench.core.contract import ContractViolationError, validate_response
from promptbench.core.types import Config, JobSpec, ProviderResponse, ProviderSpec, RunStatus
from promptbench.providers.base import ProviderAdapter, ProviderRunContext
from promptbench.runner import executor


def _make_job(tmp_path: pathlib.Path) -> JobSpec:
    templates = tmp_path / "templates"
    payloads = tmp_path / "payloads"
    templates.mkdir()
    payloads.mkdir()
    t_path = templates / "t.md"
    p_path = payloads / "p.md"
    t_path.write_text("T: {{USER_PROMPT}}")
    p_path.write_text("P")
    return JobSpec(
        id="job1",
        template_path=str(t_path),
        payload_path=str(p_path),
        provider_id="stub",
    )


def test_validate_response_injects_provider_meta():
    spec = ProviderSpec(id="p1", type="stub")
    response = validate_response(ProviderResponse(text="ok"), spec)
    assert response.meta["provider_id"] == "p1"
    assert response.meta["provider_type"] == "stub"


def test_validate_response_rejects_empty_text():
    spec = ProviderSpec(id="p1", type="stub")
    with pytest.raises(ContractViolationError):
        validate_response(ProviderResponse(text=""), spec)


def test_validate_response_requires_error_fields():
    spec = ProviderSpec(id="p1", type="stub")
    with pytest.raises(ContractViolationError):
        validate_response(ProviderResponse(error={"code": "X"}), spec)


class EmptyAdapter(ProviderAdapter):
    def run(self, context: ProviderRunContext) -> ProviderResponse:
        return ProviderResponse(text="")


def test_contract_violation_becomes_error_run(tmp_path):
    job = _make_job(tmp_path)
    config = Config(
        templates_glob="",
        payloads_glob="",
        output_root=str(tmp_path / "runs"),
        providers=[ProviderSpec(id="stub", type="stub")],
    )

    result = executor._run_single_job(job, config, {"stub": EmptyAdapter(ProviderSpec(id="stub", type="stub"))})

    assert result.status == RunStatus.ERROR
    assert result.error["code"] == "CONTRACT_VIOLATION"
    run_json = json.loads(pathlib.Path(result.artifact_paths.run_json).read_text())
    provider_meta = run_json["provider"]["meta"]
    assert provider_meta["provider_id"] == "stub"
    assert provider_meta["provider_type"] == "stub"


--- tests/test_core.py ---
import pytest
from dataclasses import asdict
from promptbench.core.types import ProviderSpec, Config, RunStatus
from promptbench.core.errors import PromptbenchError, ProviderError, to_error_dict

def test_provider_spec_serialization():
    spec = ProviderSpec(id="test", type="lmstudio", model="gpt-4")
    data = asdict(spec)
    assert data["id"] == "test"
    assert data["type"] == "lmstudio"
    assert data["model"] == "gpt-4"
    assert data["args"] == []
    assert data["env"] == {}

def test_config_defaults():
    spec = ProviderSpec(id="p1", type="lmstudio")
    config = Config(
        templates_glob="templates/*.md",
        payloads_glob="payloads/*.md",
        output_root="runs",
        providers=[spec]
    )
    assert config.concurrency == 1
    assert config.retries == 0
    assert config.use_eventing_executor is False
    assert config.skills_glob is None
    assert config.write_markdown_output is False
    assert config.skill_router.enabled is False

def test_error_serialization():
    err = ProviderError("connection failed", details={"host": "localhost"})
    data = err.to_dict()
    assert data["code"] == "PROVIDER_ERROR"
    assert data["message"] == "connection failed"
    assert data["details"] == {"host": "localhost"}

def test_to_error_dict_wrapper():
    raw_err = ValueError("bad value")
    data = to_error_dict(raw_err)
    assert data["code"] == "UNKNOWN_ERROR"
    assert data["message"] == "bad value"
    assert data["details"]["type"] == "ValueError"

    pb_err = PromptbenchError("pb error", code="CUSTOM")
    data = to_error_dict(pb_err)
    assert data["code"] == "CUSTOM"
    assert data["message"] == "pb error"


--- tests/test_engine_composition.py ---
import pytest
from promptbench.core.compose import compose_system
from promptbench.core.types import CompositionMode

def test_engine_composition_replaces_last_line():
    template = "Line 1\nOptimize this input:\n"
    payload = "Find the bug."
    # The last non-empty line is "Optimize this input:"
    result = compose_system(template, payload, mode=CompositionMode.REPLACE_LAST_LINE)
    assert result == "Line 1\nUser prompt: Find the bug.\n"

def test_engine_composition_with_trailing_blank_lines():
    template = "Instruction 1\nInstruction 2 (to be replaced)\n\n\n"
    payload = "Payload"
    result = compose_system(template, payload, mode=CompositionMode.REPLACE_LAST_LINE)
    # Replaces "Instruction 2", preserves three \n
    assert result == "Instruction 1\nUser prompt: Payload\n\n\n"

def test_engine_composition_empty_template():
    result = compose_system("", "Payload", mode=CompositionMode.REPLACE_LAST_LINE)
    assert result == "User prompt: Payload"

def test_engine_composition_only_whitespace_template():
    template = "   \n\n"
    result = compose_system(template, "Payload", mode=CompositionMode.REPLACE_LAST_LINE)
    assert result == "User prompt: Payload\n   \n\n"


--- tests/test_eventing_executor.py ---
import json
import pathlib
from io import StringIO

from promptbench.core.types import ProviderSpec, JobSpec, Config
from promptbench.providers.base import ProviderAdapter, ProviderRunContext
from promptbench.core.types import ProviderResponse
from promptbench.runner.eventing_executor import run_jobs_with_events


class StubAdapter(ProviderAdapter):
    def run(self, context: ProviderRunContext) -> ProviderResponse:
        return ProviderResponse(text="ok")


def test_run_jobs_with_events_emits_jsonl(tmp_path):
    templates = tmp_path / "templates"
    templates.mkdir()
    (templates / "t1.md").write_text("Hello {{USER_PROMPT}}")

    payloads = tmp_path / "payloads"
    payloads.mkdir()
    (payloads / "p1.md").write_text("World")

    output_root = tmp_path / "runs"

    spec = ProviderSpec(id="stub", type="stub")
    config = Config(
        templates_glob=str(templates / "*.md"),
        payloads_glob=str(payloads / "*.md"),
        output_root=str(output_root),
        providers=[spec],
    )

    jobs = [
        JobSpec(
            id="j1",
            template_path=str(templates / "t1.md"),
            payload_path=str(payloads / "p1.md"),
            provider_id="stub",
        )
    ]

    adapters = {"stub": StubAdapter(spec)}
    event_sink = StringIO()

    run_jobs_with_events(jobs, config, adapters, event_sink, run_id="run-evt")

    lines = [line for line in event_sink.getvalue().splitlines() if line]
    parsed = [json.loads(line) for line in lines]

    types = [item["type"] for item in parsed]
    assert "JOB_STARTED" in types
    assert "JOB_COMPLETED" in types
    assert "ARTIFACT_WRITTEN" in types

    job_ids = {item["job_id"] for item in parsed if item["type"] in ("JOB_STARTED", "JOB_COMPLETED")}
    assert job_ids == {"j1"}

    started = next(item for item in parsed if item["type"] == "JOB_STARTED")
    assert started["payload"]["attempt"] == 1

    completed = next(item for item in parsed if item["type"] == "JOB_COMPLETED")
    assert completed["payload"]["status"] == "ok"
    assert "duration_s" in completed["payload"]
    assert completed["payload"]["error_code"] is None


--- tickets/lm-studio/LM-Studio-Model-Selection.md ---
Title:

* Expose full LM Studio sampling parameters in promptbench LMStudioAdapter and tune defaults for deterministic vs creative runs

Summary:

* The user wants the optimal LM Studio model selection and inference parameters for running promptbench.

    LM Studio Model Selection

* Current promptbench LM Studio integration appears to only pass `temperature` and `ttl`, limiting the ability to run reproducible benchmarks or controlled-variability runs using `top_p`, `top_k`, `repeat_penalty`, `seed`, `max_tokens`, and `stop`.

    LM Studio Model Selection

* The request also highlights runtime stability concerns (model auto-eviction via low TTL) for longer promptbench runs.

    LM Studio Model Selection

Background / Context:

* User asked: “optimal lm-studio model and model parameters for runs using promptbench?” and referenced LM Studio docs.

    LM Studio Model Selection

* The assistant response indicates promptbench’s current config points at `qwen_qwen3-vl-4b-instruct`, with a recommended upgrade to `qwen3-vl-8b-instruct`.

    LM Studio Model Selection

* The assistant also states the current `LMStudioAdapter` sends only `temperature` and `ttl`, and uses a system-only message with a `"user '.'"` fallback on HTTP 400.

    LM Studio Model Selection

Current Behavior (Actual):

* LM Studio provider requests from promptbench only include:

  * `temperature`

  * `ttl` (noted as currently set to `60` seconds)

        LM Studio Model Selection

* Advanced inference controls supported by LM Studio’s OpenAI-compatible endpoint are not passed through by promptbench (as described): `top_p`, `top_k`, `repeat_penalty`, `seed`, `max_tokens`, `stop`, etc.

    LM Studio Model Selection

* Low `ttl` may cause unload/reload churn if there are gaps between jobs.

    LM Studio Model Selection

Expected Behavior:

* Promptbench should allow configuring and sending LM Studio inference parameters beyond `temperature`/`ttl`, including at least: `top_p`, `top_k`, `repeat_penalty`, `seed`, `max_tokens`, `stop` (and optionally `presence_penalty`, `frequency_penalty`).

    LM Studio Model Selection

* Provide a clear recommended default model choice and parameter presets for:

  * Deterministic / reproducible benchmark runs

  * Higher-variability / creative exploration runs

        LM Studio Model Selection

* Reduce model eviction risk during long runs by increasing `ttl` (recommended `3600+`) or supporting omission of TTL when models are preloaded.

    LM Studio Model Selection

Requirements:

* Model recommendation (from the user’s installed LM Studio set, per assistant response):

  * Preferred: `qwen3-vl-8b-instruct`

  * Alternative: `essentialai_rnj-1-instruct (Gemma3 ~8.3B)`

  * Throughput option: `mistralai_ministral-3-3b-instruct-2512`

  * Avoid “thinking/reasoning” variants unless explicitly desired and constrained to “final answer only.”

        LM Studio Model Selection

* Add/enable parameter passthrough in `promptbench/providers/lmstudio.py` so `ProviderSpec` can pass: `top_p`, `top_k`, `repeat_penalty`, `seed`, \`max\_tokens


--- tickets/mcp/mcp-goal.md ---
## Goal

Expose `promptbench` to agents/assistants as an **MCP server** so they can:

* fetch the **matrix** (templates/payloads/skills/providers + sets)
* generate **custom payload prompts** (including your prompt-engineer template) on demand
* run selected jobs through the existing `promptbench` CLI

This maps cleanly to MCP “tools” (with input schemas + behavioral annotations).

---

## What to expose as MCP tools

The tool surface below mirrors what your TUI already does by shelling out to `python3 -m promptbench ...` for `matrix` and `run`.

### Read-only tools (safe planning / matrix building)

1. **`promptbench_matrix(config_path, force_refresh)`**
   Returns the same JSON payload your CLI emits for matrix selection, including:

* templates/payloads/skills lists
* template/payload/skill sets
* providers
* output_root + computed jobs count

1. **`promptbench_discover(config_path, force_refresh)`**
   Preflight discovery + workspace diagnostics summary.

2. **`promptbench_doctor(config_path|workspace_path)`**
   Workspace validation (diagnostics).

3. **`promptbench_render_prompt_engineer_payload(...)`**
   Renders *your* payload template to Markdown (no writes).

### Write + execute tools (side effects)

1. **`promptbench_write_payload(config_path, filename, content, overwrite)`**
   Writes into `<workspace>/.promptbench/payloads/_mcp/…` (scoped write target).

2. **`promptbench_run(config_path, …filters…, output, limit, concurrency, …)`**
   Executes `promptbench` with the same filtering flags your CLI supports (providers/templates/payloads/skills) and uses `--event-log -` so stdout is JSONL events.

The MCP server should attach tool annotations (`readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint`) as hints for clients (but not relied on for security).

---

## Agent usage flow (matrix → custom prompt → run)

1. Call `promptbench_matrix` → choose dimensions (template/payload/skill/provider).
2. Call `promptbench_render_prompt_engineer_payload` with:

   * `{user-idea}`, `{project-in-question}`, `{PAIN-POINT}`, `{CAPABILITY}`, `{TARGET-AGENT}`, `{OPTIMIZE-PROMPT}`, plus the embedded `{REFERENCE-FILE}` content
3. Call `promptbench_write_payload` to persist the generated payload into the workspace
4. Call `promptbench_run` with `payloads=[that_written_path]` and chosen `templates/skills/providers` to execute

---

## Important operational note: “unload on finish”

Your current `promptbench` run path **always attempts adapter unload** during cleanup.
If you want agents to run repeatedly without triggering unload calls (or to avoid provider incompatibilities), add a first-class switch such as:

* CLI: `--no-unload`
* or config: `runner.unload_on_finish = false`

Then have `promptbench_run` (MCP) pass that through.

---

## Implementation

I provided a complete, drop-in **Python MCP server** implementation using the official MCP Python SDK (supports stdio and streamable HTTP).
It wraps your existing CLI entrypoints (`python3 -m promptbench matrix|discover|doctor|…`) exactly as your TUI does.

The implementation includes:

* tool schemas (Pydantic)
* strict path allowlisting via `PROMPTBENCH_MCP_ROOTS`
* safe bounded stdout/stderr capture
* scoped payload writes under `.promptbench/payloads/_mcp/`

(See the multi-file project in the document panel titled **“Promptbench Mcp Server (design + Implementation)”**.)
