{% extends "base.html" %}
{% from "_macros.html" import rate, score %}
{% block title %}{{ run_id }} — bpmn-evalkit{% endblock %}
{% block body %}
<h1 class="mono">{{ run_id }}</h1>
{% if summary %}
<p class="sub">{{ summary.sut_name }} {{ summary.sut_version }} ·
{{ summary.dataset }} ({{ summary.dataset_version }}, {{ summary.cases }} cases) ·
k={{ summary.k }} · started {{ summary.started }}</p>
{% endif %}

<div class="card" id="status">
  <strong>Status:</strong> <span id="state">{{ status.state }}</span>
  {% if status.get('total') %}
    <progress id="bar" value="{{ status.get('done', 0) }}" max="{{ status.total }}"></progress>
    <span class="muted mono" id="counts">{{ status.get('done', 0) }}/{{ status.total }}</span>
  {% endif %}
  {% if status.get('error') %}<p class="bad mono">{{ status.error }}</p>{% endif %}
  {% if metrics %}<p style="margin-bottom:0"><a href="/runs/{{ run_id }}/report">Markdown report</a></p>{% endif %}
</div>

{% if aggregates %}
<h2>Stage 1 — validity gates</h2>
<div class="scroll"><table>
  <thead><tr><th>Metric</th><th>Rate</th><th>Passed</th><th>Decided</th><th>Skipped</th></tr></thead>
  <tbody>
  {% for gate in ['gate.parse','gate.xsd','gate.lint','gate.wfnet','gate.soundness','gate.reachability','gate.di'] %}
    {% set e = aggregates.get(gate) or {} %}
    <tr>
      <td class="mono">{{ gate }}</td>
      <td>{{ rate(e.get('rate')) }}</td>
      <td>{{ e.get('passed', 0) }}</td>
      <td>{{ e.get('decided', 0) }}</td>
      <td>{% if e.get('skipped') %}<span class="warn">{{ e.skipped }}</span>{% else %}0{% endif %}</td>
    </tr>
  {% endfor %}
  </tbody>
</table></div>
<p class="note"><strong>Reading soundness:</strong>
{% for k, v in (aggregates.get('soundness.verdicts') or {}).items() %}<code>{{ k }}</code>: {{ v }}{% if not loop.last %}, {% endif %}{% endfor %}.
Models that are not workflow nets — more than one start or end event, which is legitimate
BPMN — are reported by <code>gate.wfnet</code> and are never counted as unsound.</p>

{% set s2 = aggregates.get('struct.status') or {} %}
<h2>Stage 2 — element similarity</h2>
{% if s2.get('status') == 'available' %}
{% set match = aggregates.get('struct.match') or {} %}
<div class="scroll"><table>
  {% set has_ceiling = 'struct.ceiling.all' in aggregates %}
  <thead><tr><th>Type</th><th>F1</th><th>F1 median</th><th>Precision</th><th>Recall</th><th>F1 (parsed only)</th><th>Attainable</th>{% if has_ceiling %}<th>Human ceiling</th>{% endif %}<th>n/a</th></tr></thead>
  <tbody>
  {% for bucket, label in [('task','Activities'),('event','Events'),('gateway','Gateways'),('flow','Sequence flows'),('lane','Lanes'),('all','All elements')] %}
    {% set e = aggregates.get('struct.f1.' + bucket) or {} %}
    <tr>
      <td class="mono">{{ label }}</td>
      <td>{{ score(e.get('f1_mean')) }}</td>
      <td>{{ score((e.get('f1') or {}).get('median')) }}</td>
      <td>{{ score(e.get('precision_mean')) }}</td>
      <td>{{ score(e.get('recall_mean')) }}</td>
      <td>{{ score((e.get('valid_only') or {}).get('f1_mean')) }}</td>
      <td>{{ score(e.get('attainable_f1_mean')) }}</td>
      {% if has_ceiling %}<td>{{ score((aggregates.get('struct.ceiling.' + bucket) or {}).get('f1_mean')) }}</td>{% endif %}
      <td>{% if e.get('not_applicable') %}<span class="muted">{{ e.not_applicable }}</span>{% else %}0{% endif %}</td>
    </tr>
  {% endfor %}
  </tbody>
</table></div>
{% if has_ceiling %}
{% set refs = aggregates.get('struct.references') or {} %}
<p class="note"><strong>Human ceiling</strong> — what two gold models of the same description
score against each other, over {{ aggregates.get('struct.ceiling.pairs', 0) }} reference pair(s),
measured with the same matcher and threshold.
{% if refs.get('min_grade') is not none %}Only models graded &ge; {{ refs.min_grade }} counted as
gold; a lower threshold would be a strictly easier benchmark.{% endif %}
It is a second number to read beside the first — not a target and not a divisor. A system
<em>above</em> the ceiling has not beaten the humans; more likely it resembles one reference
closely while the references disagree among themselves.</p>
{% endif %}
<p class="note"><strong>Matcher <code>{{ match.get('matcher') }}</code>, threshold {{ match.get('threshold') }}.</strong>
Both are part of the number — a score computed at another threshold is a different score and must
not be compared with this one. The F1 column is the mean over <em>all</em> outputs: one that produced
nothing scores 0, not "not measured". Where it diverges from the median, the gap is the failure rate.
Subtype agreement among matched pairs:
{{ rate((aggregates.get('struct.subtype_agreement') or {}).get('rate')) }} — an exclusive gateway
matched to a parallel one counts as a match and as a disagreement.</p>
{% else %}
<p class="note"><strong>Skipped.</strong> {{ s2.get('detail') or 'no gold models available at scoring time' }}.
Skipped is not zero and not a pass: nothing was measured.</p>
{% endif %}

{% set behav = aggregates.get('behav.status') %}
{% if behav %}
<h2>Stage 3 — behaviour</h2>
<div class="scroll"><table>
  {% set has_bceiling = 'behav.ceiling.f1' in aggregates %}
  <thead><tr><th>Question</th><th>Metric</th><th>Mean</th><th>Median</th>{% if has_bceiling %}<th>Human ceiling</th>{% endif %}</tr></thead>
  <tbody>
  {% for label, metric in [('Reference behaviour the output admits','behav.footprint.fitness'),('Output behaviour the reference admits','behav.footprint.precision'),('Harmonic mean of the two','behav.footprint.f1')] %}
    {% set e = aggregates.get(metric) or {} %}
    <tr>
      <td>{{ label }}</td>
      <td class="mono">{{ metric }}</td>
      <td>{{ score(e.get('mean')) }}</td>
      <td>{{ score(e.get('median')) }}</td>
      {% if has_bceiling %}<td>{{ score((aggregates.get(metric.replace('footprint','ceiling')) or {}).get('mean')) }}</td>{% endif %}
    </tr>
  {% endfor %}
  </tbody>
</table></div>
{% if has_bceiling %}
<p class="note"><strong>The ceiling is the caveat here, and it is severe.</strong> Two gold models
of the same description score {{ score((aggregates.get('behav.ceiling.f1') or {}).get('mean')) }}
against each other over {{ aggregates.get('behav.ceiling.pairs', 0) }} pair(s). A relation needs
<em>both</em> endpoints matched, so the label-matching bottleneck compounds quadratically — where
humans use different vocabularies this metric is dominated by activity matching rather than by
behaviour.</p>
{% endif %}
<p class="note">Compared model to model — the reference's Petri net is the behavioural
specification, so no event log and no play-out sampling is involved. Scored on
{{ behav.scored }} of {{ behav.total }} outputs
{% set skipped = [] %}{% for k, v in (behav.by_status or {}).items() %}{% if k != 'scored' %}{% set _ = skipped.append(k ~ ': ' ~ v) %}{% endif %}{% endfor %}
{% if skipped %}({{ skipped | join(', ') }}){% endif %}.
Both directions are reported and never read alone: fitness by itself rewards a model that permits
everything, precision by itself one that permits almost nothing. An unconvertible model is counted,
never scored 0 — that is about pm4py's BPMN coverage, not the system. This stage inherits the
stage-2 matching and therefore its threshold.</p>
{% endif %}

{% set ns = namespace(any=false) %}
{% for bucket, label in [('task','Activities'),('event','Events'),('gateway','Gateways'),('lane','Lanes')] %}
  {% if aggregates.get('struct.subtype.confusion.' + bucket) %}{% set ns.any = true %}{% endif %}
{% endfor %}
{% if ns.any %}
<h3>Subtype confusion</h3>
<div class="scroll"><table>
  <thead><tr><th>Type</th><th>Agreement</th><th>Substitution (reference → output)</th><th>Count</th></tr></thead>
  <tbody>
  {% for bucket, label in [('task','Activities'),('event','Events'),('gateway','Gateways'),('lane','Lanes')] %}
    {% set counts = aggregates.get('struct.subtype.confusion.' + bucket) or {} %}
    {% if counts %}
    {% set agreement = aggregates.get('struct.subtype.agreement.' + bucket) or {} %}
    {% set wrong = counts.items() | rejectattr('0', 'in', counts | select('equalto', none) | list) | list %}
    {% set off = [] %}
    {% for key, count in counts.items() %}{% if key.split('→')[0] != key.split('→')[1] %}{% set _ = off.append((key, count)) %}{% endif %}{% endfor %}
    {% if off %}
      {% for key, count in (off | sort(attribute='1', reverse=true))[:6] %}
      <tr>
        <td class="mono">{{ label if loop.first else '' }}</td>
        <td>{{ rate(agreement.get('rate')) if loop.first else '' }}</td>
        <td class="mono">{{ key.split('→')[0] }} → {{ key.split('→')[1] }}</td>
        <td>{{ count }}</td>
      </tr>
      {% endfor %}
    {% else %}
      <tr><td class="mono">{{ label }}</td><td>{{ rate(agreement.get('rate')) }}</td><td class="muted">none</td><td></td></tr>
    {% endif %}
    {% endif %}
  {% endfor %}
  </tbody>
</table></div>
<p class="note">Read <strong>reference → output</strong>: what gold had, then what the system
produced. These are not interchangeable errors — an exclusive gateway modelled as a parallel one
turns a decision into a fork; an inclusive modelled as exclusive narrows it. Sequence flows are
excluded (they match structurally and always agree). Subtype is the XML tag, so a message event
substituted for a timer is invisible here.</p>
{% endif %}

{% set band = aggregates.get('struct.sensitivity.thresholds') %}
{% if band %}
<h3>Threshold sensitivity</h3>
<div class="scroll"><table>
  <thead><tr><th>Type</th>{% for t in band %}<th>{{ '%g'|format(t) }}</th>{% endfor %}<th>Spread</th></tr></thead>
  <tbody>
  {% for bucket, label in [('task','Activities'),('event','Events'),('gateway','Gateways'),('flow','Sequence flows'),('lane','Lanes'),('all','All elements')] %}
    {% set curve = aggregates.get('struct.sensitivity.f1.' + bucket) or {} %}
    {% set ceil = aggregates.get('struct.sensitivity.ceiling.' + bucket) or {} %}
    <tr>
      <td class="mono">{{ label }}</td>
      {% for t in band %}<td>{{ score(curve.get('%g'|format(t))) }}</td>{% endfor %}
      <td>{{ '%.3f'|format(aggregates.get('struct.sensitivity.spread.' + bucket)) if aggregates.get('struct.sensitivity.spread.' + bucket) is not none else 'n/a' }}</td>
    </tr>
    {% if ceil %}
    <tr>
      <td class="muted" style="padding-left:1.5em">↳ human ceiling</td>
      {% for t in band %}<td class="muted">{{ score(ceil.get('%g'|format(t))) }}</td>{% endfor %}
      <td></td>
    </tr>
    {% endif %}
  {% endfor %}
  </tbody>
</table></div>
<p class="note">How much of a number is the threshold. <strong>Not</strong> an invitation to pick a
better one — moving the threshold to improve a score is the failure this table exists to expose.
A conclusion that holds across the whole band is a finding; one that appears only at the run's own
threshold is an artefact of that number.</p>
{% endif %}

<h2>Lint</h2>
<table>
  <tr><td>Error-free outputs</td><td>{{ rate((aggregates.get('gate.lint.error_free') or {}).get('rate')) }}</td></tr>
  <tr><td>Clean (warnings included)</td><td>{{ rate((aggregates.get('gate.lint.clean') or {}).get('rate')) }}</td></tr>
  <tr><td>Errors per output (median)</td><td>{{ (aggregates.get('gate.lint.errors') or {}).get('median') }}</td></tr>
</table>
{% set by_rule = aggregates.get('gate.lint.by_rule') or {} %}
{% if by_rule %}
<h2>Findings by rule</h2>
<table>{% for r, c in by_rule.items() %}<tr><td class="mono">{{ r }}</td><td>{{ c }}</td></tr>{% endfor %}</table>
<p class="note">A headline count is routinely dominated by one rule that encodes a design
decision in the system under test rather than a defect. Always read this table first.</p>
{% endif %}

<h2>Normalisation</h2>
<p class="sub">{{ aggregates.get('norm.rebuilt_flow_links', 0) }} <code>&lt;incoming&gt;</code>/<code>&lt;outgoing&gt;</code>
links rebuilt across {{ aggregates.get('norm.outputs_needing_relink', 0) }} output(s) before any gate ran.
Not a defect score — a signal about interoperability with tooling outside this harness.</p>

<h2>Cases</h2>
<div class="scroll"><table>
  <thead><tr><th>Case</th><th>Rep</th><th>Gates</th><th>Output</th></tr></thead>
  <tbody>
  {% for case in cases %}{% for rep in case.reps %}
    <tr>
      <td class="mono">{{ case.case_id }}</td>
      <td>{{ rep.rep }}</td>
      <td>
        {% for gid, g in rep.gates.items() %}
          {% if g.outcome == 'fail' %}<span class="pill bad" title="{{ g.message }}">{{ gid.replace('gate.','') }}</span>
          {% elif g.outcome == 'skipped' %}<span class="pill" title="{{ g.message }}">{{ gid.replace('gate.','') }}: skipped</span>
          {% endif %}
        {% endfor %}
        {% if rep.error %}<span class="pill bad">no output</span>{% endif %}
      </td>
      <td>
        {% if not rep.error %}
          <a href="/runs/{{ run_id }}/output/{{ case.case_id }}__rep{{ rep.rep }}.bpmn">raw</a> ·
          <a href="/runs/{{ run_id }}/output/{{ case.case_id }}__rep{{ rep.rep }}.bpmn?normalized=true">normalised</a>
        {% else %}<span class="muted">{{ rep.error }}</span>{% endif %}
      </td>
    </tr>
  {% endfor %}{% endfor %}
  </tbody>
</table></div>
{% endif %}

<script>
// Poll only while something is actually happening; a finished run needs no traffic.
const live = {{ 'true' if status.state in ('starting','generating','scoring') else 'false' }};
if (live) {
  const tick = async () => {
    const r = await fetch('/runs/{{ run_id }}/status');
    const s = await r.json();
    document.getElementById('state').textContent = s.current ? s.state + ' — ' + s.current : s.state;
    const bar = document.getElementById('bar'), counts = document.getElementById('counts');
    if (bar && s.total) { bar.value = s.done || 0; bar.max = s.total; }
    if (counts && s.total) counts.textContent = (s.done || 0) + '/' + s.total;
    if (['scored', 'failed'].includes(s.state)) { location.reload(); return; }
    setTimeout(tick, 1500);
  };
  setTimeout(tick, 1000);
}
</script>
{% endblock %}