{% macro nf(v) -%}{{ 'NaN' if v is none else '%.4f' | format(v) }}{%- endmacro %} {% macro ef(v) -%}{{ 'NaN' if v is none else '%.4e' | format(v) }}{%- endmacro %} {% macro badge(ok, yes='Yes', no='No') -%}{{ yes if ok else no }}{%- endmacro %} {% macro omnibus(stat, p_value, ok, ok_label, no_label) -%}
Statistic{{ nf(stat) }}
p-value{{ ef(p_value) }}
Result{{ badge(ok, ok_label, no_label) }}
{%- endmacro %}

Experiment Report

Statistical comparison of model performance across contexts.

{% if created_at %}Generated at {{ created_at[:19].replace('T', ' ') }}{% endif %}
{% if metadata %}
Run Parameters

Experiment Setup

Scores Target{{ metadata.scores_target | join(', ') }}
n_splits{{ metadata.n_splits }}
random_state{{ metadata.random_state }}
paired{{ metadata.paired }}
Statistical Flow{{ metadata.resolved_flow }}
Contexts{{ metadata.n_contexts }}
Test Datasets{{ metadata.n_test_datasets }}
Samples / Context{{ metadata.n_metric_samples_per_context }}
{% if metadata.execution_time_seconds is not none %}
Execution Time{{ '%.2f' | format(metadata.execution_time_seconds) }} s
{% endif %}
{% endif %} {% if contexts_summary %}
Experiment

Contexts

{% for ctx in contexts_summary %} {% endfor %}
ContextModelTypeTechnologyTest Dataset
{{ ctx.context_name }} {{ ctx.model_name }} {{ ctx.model_type }} {{ ctx.model_technology }} {{ ctx.test_data_name }}
{% endif %} {% if datasets_summary %}
Data

Datasets

{% for ds in datasets_summary %}

{{ ds.test_data_name }}

Records{{ ds.n_records }}
Features{{ ds.n_features }}
Target{{ ds.target_kind }}
{% if ds.target_kind == 'regression' %}
Target Mean{{ nf(ds.target_mean) }}
Target Std{{ nf(ds.target_std) }}
Target Min{{ nf(ds.target_min) }}
Target Median{{ nf(ds.target_median) }}
Target Max{{ nf(ds.target_max) }}
{% endif %}
{% if ds.target_kind == 'classification' and ds.target_class_counts %}
{% for cls in ds.target_class_counts %} {% endfor %}
ClassCountProportion
{{ cls.label }} {{ cls.count }} {{ '%.2f%%' | format(cls.proportion * 100) }}
{% endif %} {% if ds.feature_stats %}
{% for fs in ds.feature_stats %} {% endfor %}
FeatureCountMeanStdMin25%Median75%Max
{{ fs.feature }} {{ '%.0f' | format(fs.count) }} {{ nf(fs.mean) }} {{ nf(fs.std) }} {{ nf(fs.minimum) }} {{ nf(fs.q25) }} {{ nf(fs.median) }} {{ nf(fs.q75) }} {{ nf(fs.maximum) }}
{% endif %} {% endfor %}
{% endif %} {% for report in reports_by_score %}
Score Target

{{ report.score_target }}

Context Result Statistics

{% for model in report.score_described %} {% endfor %}
Context Name Mean Std Median Min Max Mode
{{ model.context_name }} {{ '%.4f' | format(model.mean) }} {{ '%.4f' | format(model.std) }} {{ '%.4f' | format(model.median) }} {{ '%.4f' | format(model.minimum) }} {{ '%.4f' | format(model.maximum) }} {{ '%.4f' | format(model.mode) }}

Statistical Test Pipeline

{% for etapa in report.ab_tests.pipeline_track %} {{ etapa.replace('_', ' ') }} {% endfor %}
{% if "check_normality_with_shapiro" in report.ab_tests.pipeline_track %}

Shapiro-Wilk

{% for test in report.ab_tests.shapirowilk %} {% endfor %}
ContextStatp-valueNormal?
{{ test.context }} {{ nf(test.stat) }} {{ ef(test.p_value) }} {{ badge(test.is_normal) }}
{% endif %} {% if "check_homocedasticity_with_levene" in report.ab_tests.pipeline_track %}

Levene

{{ omnibus(report.ab_tests.levene.stat, report.ab_tests.levene.p_value, report.ab_tests.levene.is_homoscedastic, 'Homoscedastic', 'Heteroscedastic') }} {% endif %} {% if "perform_bartlett" in report.ab_tests.pipeline_track %}

Bartlett

{{ omnibus(report.ab_tests.bartlett.stat, report.ab_tests.bartlett.p_value, report.ab_tests.bartlett.is_homoscedastic, 'Homoscedastic', 'Heteroscedastic') }} {% endif %} {% if "perform_t_student" in report.ab_tests.pipeline_track %}

T-Student Test

{{ omnibus(report.ab_tests.tstudent.stat, report.ab_tests.tstudent.p_value, report.ab_tests.tstudent.is_significant, 'Significant', 'Not significant') }} {% endif %} {% if "perform_anova" in report.ab_tests.pipeline_track %}

ANOVA Test

{{ omnibus(report.ab_tests.anova.stat, report.ab_tests.anova.p_value, report.ab_tests.anova.is_significant, 'Significant', 'Not significant') }} {% endif %} {% if "perform_welch" in report.ab_tests.pipeline_track %}

Welch Test

{{ omnibus(report.ab_tests.welch.stat, report.ab_tests.welch.p_value, report.ab_tests.welch.is_significant, 'Significant', 'Not significant') }} {% endif %} {% if "perform_kurskalwallis" in report.ab_tests.pipeline_track %}

Kruskal-Wallis

{{ omnibus(report.ab_tests.kurskalwallis.stat, report.ab_tests.kurskalwallis.p_value, report.ab_tests.kurskalwallis.is_significant, 'Significant', 'Not significant') }} {% endif %} {% if "perform_mannwhitney" in report.ab_tests.pipeline_track %}

Mann-Whitney

{% for test in report.ab_tests.mannwhitney %} {% endfor %}
ComparisonStatp-valueSignificant?
{{ test.context }} {{ nf(test.stat) }} {{ ef(test.p_value) }} {{ badge(test.is_significant) }}
{% endif %} {% if "perform_dunn_with_bh_correction" in report.ab_tests.pipeline_track %}

Dunn's Test (BH Correction)

{% for test in report.ab_tests.dunn %} {% endfor %}
ComparisonStatp-valueCorrected p-valueSignificant?
{{ test.context_name_1 }} vs {{ test.context_name_2 }} {{ nf(test.stat) }} {{ ef(test.p_value) }} {{ ef(test.corrected_p_value) }} {{ badge(test.is_significant) }}
{% endif %} {% if "perform_paired_t_student" in report.ab_tests.pipeline_track %}

Paired T-Student Test

{{ omnibus(report.ab_tests.paired_tstudent.stat, report.ab_tests.paired_tstudent.p_value, report.ab_tests.paired_tstudent.is_significant, 'Significant', 'Not significant') }} {% endif %} {% if "perform_wilcoxon" in report.ab_tests.pipeline_track %}

Wilcoxon Signed-Rank Test

{% for test in report.ab_tests.wilcoxon %} {% endfor %}
ComparisonStatp-valueSignificant?
{{ test.context }} {{ nf(test.stat) }} {{ ef(test.p_value) }} {{ badge(test.is_significant) }}
{% endif %} {% if "perform_repeated_measures_anova" in report.ab_tests.pipeline_track %}

Repeated-Measures ANOVA

{{ omnibus(report.ab_tests.repeated_measures_anova.stat, report.ab_tests.repeated_measures_anova.p_value, report.ab_tests.repeated_measures_anova.is_significant, 'Significant', 'Not significant') }} {% endif %} {% if "perform_paired_t_posthoc_with_bh_correction" in report.ab_tests.pipeline_track %}

Paired T-Student Post-hoc (BH Correction)

{% for test in report.ab_tests.paired_tstudent_posthoc %} {% endfor %}
ComparisonStatp-valueCorrected p-valueSignificant?
{{ test.context }} {{ nf(test.stat) }} {{ ef(test.p_value) }} {{ ef(test.corrected_p_value) }} {{ badge(test.is_significant) }}
{% endif %} {% if "perform_friedman" in report.ab_tests.pipeline_track %}

Friedman Test

{{ omnibus(report.ab_tests.friedman.stat, report.ab_tests.friedman.p_value, report.ab_tests.friedman.is_significant, 'Significant', 'Not significant') }} {% endif %} {% if "perform_wilcoxon_posthoc_with_bh_correction" in report.ab_tests.pipeline_track %}

Wilcoxon Signed-Rank Post-hoc (BH Correction)

{% for test in report.ab_tests.wilcoxon %} {% endfor %}
ComparisonStatp-valueCorrected p-valueSignificant?
{{ test.context }} {{ nf(test.stat) }} {{ ef(test.p_value) }} {{ ef(test.corrected_p_value) }} {{ badge(test.is_significant) }}
{% endif %}
{% endfor %}

Significance Conclusions

Best Context