LICENSE
README.md
pyproject.toml
src/it_eval_framework/__init__.py
src/it_eval_framework/api.py
src/it_eval_framework/cli.py
src/it_eval_framework/config.py
src/it_eval_framework/presets.py
src/it_eval_framework/run_layout.py
src/it_eval_framework/task_registry.py
src/it_eval_framework.egg-info/PKG-INFO
src/it_eval_framework.egg-info/SOURCES.txt
src/it_eval_framework.egg-info/dependency_links.txt
src/it_eval_framework.egg-info/entry_points.txt
src/it_eval_framework.egg-info/requires.txt
src/it_eval_framework.egg-info/top_level.txt
src/it_eval_framework/metrics/__init__.py
src/it_eval_framework/metrics/generation_diagnostics.py
src/it_eval_framework/metrics/minimal_pairs.py
src/it_eval_framework/metrics/perplexity.py
src/it_eval_framework/reporting/__init__.py
src/it_eval_framework/reporting/aggregate_results.py
src/it_eval_framework/reporting/compare_checkpoints.py
src/it_eval_framework/reporting/metric_semantics.py
src/it_eval_framework/reporting/normalize_results.py
src/it_eval_framework/resources/generation_prompts.yaml
src/it_eval_framework/runners/__init__.py
src/it_eval_framework/runners/common.py
src/it_eval_framework/runners/launch.py
src/it_eval_framework/runners/lighteval_entry.py
src/it_eval_framework/runners/plan.py
src/it_eval_framework/runners/probe_lighteval_tasks.py
src/it_eval_framework/runners/run_all.py
src/it_eval_framework/runners/run_blimp_it.py
src/it_eval_framework/runners/run_generation.py
src/it_eval_framework/runners/run_lighteval.py
src/it_eval_framework/runners/run_perplexity.py
src/it_eval_framework/utils/__init__.py
src/it_eval_framework/utils/distributed.py
src/it_eval_framework/utils/env.py
src/it_eval_framework/utils/hashing.py
src/it_eval_framework/utils/io.py
src/it_eval_framework/utils/lighteval_runtime.py
src/it_eval_framework/utils/modeling.py
src/it_eval_framework/utils/revisions.py
src/it_eval_framework/utils/run_state.py
tests/test_api.py
tests/test_cli.py
tests/test_compare_checkpoints.py
tests/test_config.py
tests/test_distributed.py
tests/test_environment.py
tests/test_generation_diagnostics.py
tests/test_lighteval_entry.py
tests/test_lighteval_runtime.py
tests/test_metric_semantics.py
tests/test_minimal_pairs.py
tests/test_modeling.py
tests/test_normalize_results.py
tests/test_notebook_walkthrough.py
tests/test_perplexity.py
tests/test_presets.py
tests/test_reproducibility.py
tests/test_revisions.py
tests/test_run_plan.py