{{ ui_lang.iaa_title|default('Inter-Annotator Agreement') }}

← {{ ui_lang.iaa_admin_dashboard|default('Admin Dashboard') }}

{{ ui_lang.iaa_overlap_sample|default('Overlap sample') }} — {{ report.n_overlap_items }} item{{ '' if report.n_overlap_items == 1 else 's' }} {{ ui_lang.iaa_at_cap|default('at cap.') }} {% if report.n_items_below_cap %} {{ report.n_items_below_cap }} more {{ 'has' if report.n_items_below_cap == 1 else 'have' }} more than one annotator but are still short of the cap, so {{ 'it is' if report.n_items_below_cap == 1 else 'they are' }} not scored yet. {% endif %} {{ ui_lang.iaa_meta_scored|default('Each schema is scored with the metrics appropriate to its annotation type.') }}

{% if not report.schemas %}

{{ ui_lang.iaa_empty_state|default('No overlap-sample items have reached their cap yet. Once the sample items have been annotated by the configured number of annotators, IAA will appear here.') }}

{% endif %} {% for schema_name, schema in report.schemas.items() %}

{{ schema_name }}{{ schema.kind }} · {{ schema.annotation_type }}

{# Scales, and which metrics belong to them, are classified in potato/server_utils/iaa/presentation.py. Doing it here meant the page inferred what a metric was from its value, which is how a count of skipped items came to be labelled "weak agreement". #} {% for scale in schema.scales %} {% if scale == 'kappa' %}

{{ ui_lang.iaa_scale_kappa_label|default('κ-family scale (κ, α, γ, σ):') }} <0.2 {{ ui_lang.iaa_band_poor|default('poor') }} · 0.2–0.4 {{ ui_lang.iaa_band_fair|default('fair') }} · 0.4–0.6 {{ ui_lang.iaa_band_moderate|default('moderate') }} · 0.6–0.8 {{ ui_lang.iaa_band_substantial|default('substantial') }} · >0.8 {{ ui_lang.iaa_band_strong|default('strong') }}.

{% elif scale == 'correlation' %}

{{ ui_lang.iaa_scale_correlation_label|default('Correlation / ICC:') }} {{ ui_lang.iaa_scale_correlation_note|default('ranges −1 to 1; closer to 1 is stronger agreement.') }}

{% elif scale == 'raw' %}

{{ ui_lang.iaa_scale_raw_label|default('Raw agreement (% agreement, Jaccard, F1):') }} {{ ui_lang.iaa_scale_raw_note|default('0 to 1; no chance correction.') }}

{% elif scale == 'span' %}

{{ ui_lang.iaa_scale_span_label|default('Span F1 / token κ:') }} {{ ui_lang.iaa_scale_span_note_pre|default('0 to 1 (κ can be negative);') }} span_f1_partial {{ ui_lang.iaa_scale_span_note_post|default('is the conventional reporting metric.') }}

{% elif scale == 'lower' %}

{{ ui_lang.iaa_scale_lower_label|default('Error and offset metrics (MAE, RMSE, mean offset):') }} {{ ui_lang.iaa_scale_lower_note|default('lower is better; 0 means perfect agreement. Not colour-banded, because the strong/weak bands would be inverted.') }}

{% elif scale == 'distribution' %}

{{ ui_lang.iaa_scale_distribution_label|default('Kolmogorov–Smirnov statistic:') }} {{ ui_lang.iaa_scale_distribution_note|default('0 to 1; how far the within-item offset distribution sits from the between-item one. A separation measure, not an agreement coefficient.') }}

{% elif scale == 'coverage' %}

{{ ui_lang.iaa_scale_coverage_label|default('Coverage:') }} {{ ui_lang.iaa_scale_coverage_note|default('0 to 1; the share of the task that was answered at all. Not agreement — full coverage says everyone answered, not that they answered the same. Read it as the denominator behind the scores above.') }}

{% endif %} {% endfor %}
{% for row in schema.rows %} {% if row.name not in ('n_items', 'n_annotators', 'n_aligned_items') %} {% endif %} {% endfor %}
{{ ui_lang.iaa_th_metric|default('Metric') }}{{ ui_lang.iaa_th_value|default('Value') }}
{{ row.name }} {% if row.is_text %}{{ row.display }}{% elif row.display %}{{ row.display }}{% endif %} {% if row.band %}{% set band_label = ui_lang.iaa_band_strong|default('strong') if row.band == 'strong' else ui_lang.iaa_band_weak|default('weak') %} — {{ band_label }} {{ ui_lang.iaa_agreement|default('agreement') }}{% endif %} {% if row.context %}{{ row.context }}{% endif %} {% if row.note %}{{ row.note }}{% endif %}
{{ schema.n_items }} {{ ui_lang.iaa_items|default('items') }} {% if schema.n_annotators is not none %} · {{ schema.n_annotators }} {{ ui_lang.iaa_annotators|default('annotators') }}{% endif %} {% if schema.n_aligned_items is not none %} · {{ schema.n_aligned_items }} {{ ui_lang.iaa_fully_aligned_items|default('fully-aligned items') }}{% endif %}
{% if schema.sweep %} {# The sweep is the finding, not an appendix: agreement flat across the parameter means annotators identify the same instant, and agreement that appears only at the loosest setting means the most anyone can claim is that something is wrong somewhere. Neither is visible from a single headline number. #}

{{ ui_lang.iaa_sweep_title|default('Sensitivity to the matching window') }}

{{ ui_lang.iaa_sweep_note|default('Every measure below depends on how far apart two marks may be and still count as the same event. The highlighted row is the headline setting; read down a column to see whether agreement survives a tighter window.') }}

{% for column in schema.sweep.columns %}{% endfor %} {% for row in schema.sweep.rows %} {% for cell in row.cells %} {% endfor %} {% endfor %}
{{ schema.sweep.parameter_label }}{{ column }}
{% if row.parameter is number %}{{ "%g"|format(row.parameter) }}{% else %}{{ row.parameter }}{% endif %} {% if row.is_headline %}{{ ui_lang.iaa_sweep_headline|default('headline') }}{% endif %} {{ cell.display }}{% if cell.footnote %}{{ cell.footnote }} — {{ cell.note }}{% endif %}
{% if schema.sweep.footnotes %}
    {% for note in schema.sweep.footnotes %}
  1. {{ note }}
    2. {% endfor %}
{% endif %} {% endif %}
{% endfor %}
{{ ui_lang.iaa_per_item_breakdown|default('Per-item breakdown') }} ({{ report['items']|length }} {{ ui_lang.iaa_items|default('items') }})
{% for iid, item in report['items'].items() %} {% endfor %}
{{ ui_lang.iaa_th_instance|default('Instance') }}{{ ui_lang.iaa_th_cap|default('Cap') }}{{ ui_lang.iaa_th_annotators|default('Annotators') }}
{{ iid }} {{ item.cap }} {{ item.annotators|join(', ') }}
{# Agreement over time. A single whole-project number averages early and late work together, so a team whose recent agreement has collapsed can still show an acceptable figure. Guidelines drift as soon as new edge cases appear, which is why the advice that produces large agreement gains is to re-calibrate PERIODICALLY, not once. #} {% if drift %}

{{ ui_lang.iaa_drift_title|default('Agreement over time') }}

{% if drift.reason %}

{{ drift.reason }}

{% else %} {% for trigger in drift.triggers %}

{{ ui_lang.iaa_drift_recalibrate|default('Re-calibration suggested') }} — {{ trigger.schema }} {{ '%.0f'|format(trigger.relative_drop * 100) }}% {{ ui_lang.iaa_drift_below_baseline|default('below the project baseline') }} ({{ trigger.metric }} {{ '%.3f'|format(trigger.latest) }} vs {{ '%.3f'|format(trigger.baseline) }}).

{% endfor %}

{{ ui_lang.iaa_drift_note|default('Windows are ordered batches of items, placed by when their last annotator finished. Each window is scored with the same metric as the project as a whole.') }}

{% for schema_name, series in drift.schemas.items() %} {% if series.metric %}
{% for point in series.points %}{% endfor %} {# scope="row" so a screen reader reading "0.520" also reads which schema and baseline it belongs to. This table is two-dimensional; the metric tables above are not. #} {% for point in series.points %} {% endfor %}
{{ ui_lang.iaa_drift_title|default('Agreement over time') }} — {{ schema_name }}
{{ schema_name }} · {{ series.metric }}{{ ui_lang.iaa_drift_th_window|default('W') }}{{ point.window + 1 }}
{{ ui_lang.iaa_drift_baseline|default('Baseline') }} {{ '%.3f'|format(series.baseline) if series.baseline is not none else '—' }} {% if point.value is not none %}{{ '%.3f'|format(point.value) }}{% else %}—{% endif %} {{ point.n_items }} {{ ui_lang.iaa_items|default('items') }}{% if point.sparse %} · {{ ui_lang.iaa_drift_sparse|default('too few to judge') }}{% endif %} {# A chance-corrected coefficient is undefined when there is nothing to be right about by chance, so a window where everyone agreed on everything has no number. Without a marker it plots as a bare dash, which reads as missing data and is the exact opposite of what happened. The marker is two words; the sentence goes in the footnote under the table, because repeating it in every cell forced the column wide enough to make the row unscannable. #} {% if point.value is none and point.note %}{{ point.note }}{% endif %}
{% if series.footnotes %}
    {% for footnote in series.footnotes %}
  • {{ footnote.note }} — {{ footnote.explanation }}
    • {% endfor %}
{% endif %} {% endif %} {% endfor %} {% if drift.markers %}

{{ ui_lang.iaa_drift_codebook_changes|default('Codebook revisions:') }} {% for marker in drift.markers %}r{{ marker.revision }}{% if marker.inferred %} ({{ ui_lang.iaa_drift_approx|default('approximate') }}){% endif %}{% if not loop.last %} · {% endif %}{% endfor %}

{% endif %} {% if drift.n_untimed_items %}

{{ drift.n_untimed_items }} {{ ui_lang.iaa_drift_untimed|default('item(s) carry no timestamp and are not on the timeline. This is expected for annotations imported from another tool.') }}

{% endif %} {% endif %}
{% endif %}