LICENSE
MANIFEST.in
README.md
pyproject.toml
setup.cfg
src/crfm_helm.egg-info/PKG-INFO
src/crfm_helm.egg-info/SOURCES.txt
src/crfm_helm.egg-info/dependency_links.txt
src/crfm_helm.egg-info/entry_points.txt
src/crfm_helm.egg-info/not-zip-safe
src/crfm_helm.egg-info/requires.txt
src/crfm_helm.egg-info/top_level.txt
src/helm/__init__.py
src/helm/benchmark/__init__.py
src/helm/benchmark/data_preprocessor.py
src/helm/benchmark/executor.py
src/helm/benchmark/run.py
src/helm/benchmark/run_expander.py
src/helm/benchmark/run_specs.py
src/helm/benchmark/runner.py
src/helm/benchmark/server.py
src/helm/benchmark/slurm_jobs.py
src/helm/benchmark/slurm_runner.py
src/helm/benchmark/test_data_preprocessor.py
src/helm/benchmark/test_run_expander.py
src/helm/benchmark/adaptation/__init__.py
src/helm/benchmark/adaptation/adapter_spec.py
src/helm/benchmark/adaptation/prompt.py
src/helm/benchmark/adaptation/request_state.py
src/helm/benchmark/adaptation/scenario_state.py
src/helm/benchmark/adaptation/adapters/__init__.py
src/helm/benchmark/adaptation/adapters/adapter.py
src/helm/benchmark/adaptation/adapters/adapter_factory.py
src/helm/benchmark/adaptation/adapters/binary_ranking_adapter.py
src/helm/benchmark/adaptation/adapters/generation_adapter.py
src/helm/benchmark/adaptation/adapters/in_context_learning_adapter.py
src/helm/benchmark/adaptation/adapters/language_modeling_adapter.py
src/helm/benchmark/adaptation/adapters/multiple_choice_calibrated_adapter.py
src/helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py
src/helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py
src/helm/benchmark/adaptation/adapters/test_adapter.py
src/helm/benchmark/adaptation/adapters/test_generation_adapter.py
src/helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py
src/helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py
src/helm/benchmark/augmentations/__init__.py
src/helm/benchmark/augmentations/contraction_expansion_perturbation.py
src/helm/benchmark/augmentations/contrast_sets_perturbation.py
src/helm/benchmark/augmentations/correct_to_misspelling.json
src/helm/benchmark/augmentations/data_augmenter.py
src/helm/benchmark/augmentations/dialect_perturbation.py
src/helm/benchmark/augmentations/extra_space_perturbation.py
src/helm/benchmark/augmentations/filler_words_perturbation.py
src/helm/benchmark/augmentations/gender_perturbation.py
src/helm/benchmark/augmentations/lowercase_perturbation.py
src/helm/benchmark/augmentations/mild_mix_perturbation.py
src/helm/benchmark/augmentations/misspelling_perturbation.py
src/helm/benchmark/augmentations/person_name_perturbation.py
src/helm/benchmark/augmentations/perturbation.py
src/helm/benchmark/augmentations/perturbation_description.py
src/helm/benchmark/augmentations/space_perturbation.py
src/helm/benchmark/augmentations/synonym_perturbation.py
src/helm/benchmark/augmentations/test_perturbation.py
src/helm/benchmark/augmentations/typos_perturbation.py
src/helm/benchmark/data_overlap/__init__.py
src/helm/benchmark/data_overlap/export_scenario_text.py
src/helm/benchmark/data_overlap/light_scenario.py
src/helm/benchmark/efficiency_data/inference_denoised_runtimes.json
src/helm/benchmark/efficiency_data/inference_idealized_runtimes.json
src/helm/benchmark/efficiency_data/training_efficiency.json
src/helm/benchmark/metrics/__init__.py
src/helm/benchmark/metrics/basic_metrics.py
src/helm/benchmark/metrics/bbq_metrics.py
src/helm/benchmark/metrics/bias_metrics.py
src/helm/benchmark/metrics/bias_word_lists.py
src/helm/benchmark/metrics/classification_metrics.py
src/helm/benchmark/metrics/code_metrics.py
src/helm/benchmark/metrics/code_metrics_helper.py
src/helm/benchmark/metrics/copyright_metrics.py
src/helm/benchmark/metrics/disinformation_metrics.py
src/helm/benchmark/metrics/dry_run_metrics.py
src/helm/benchmark/metrics/instruction_following_critique_metrics.py
src/helm/benchmark/metrics/machine_translation_metrics.py
src/helm/benchmark/metrics/metric.py
src/helm/benchmark/metrics/metric_name.py
src/helm/benchmark/metrics/metric_service.py
src/helm/benchmark/metrics/numeracy_metrics.py
src/helm/benchmark/metrics/ranking_metrics.py
src/helm/benchmark/metrics/statistic.py
src/helm/benchmark/metrics/summarization_critique_metrics.py
src/helm/benchmark/metrics/summarization_metrics.py
src/helm/benchmark/metrics/test_bias_metrics.py
src/helm/benchmark/metrics/test_classification_metrics.py
src/helm/benchmark/metrics/test_metric.py
src/helm/benchmark/metrics/test_numeracy_metrics.py
src/helm/benchmark/metrics/test_statistic.py
src/helm/benchmark/metrics/toxicity_metrics.py
src/helm/benchmark/metrics/summac/__init__.py
src/helm/benchmark/metrics/summac/model_summac.py
src/helm/benchmark/metrics/summac/utils_misc.py
src/helm/benchmark/metrics/tokens/__init__.py
src/helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/auto_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/free_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/openai_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/token_cost_estimator.py
src/helm/benchmark/presentation/__init__.py
src/helm/benchmark/presentation/contamination.py
src/helm/benchmark/presentation/create_plots.py
src/helm/benchmark/presentation/run_display.py
src/helm/benchmark/presentation/run_entry.py
src/helm/benchmark/presentation/schema.py
src/helm/benchmark/presentation/summarize.py
src/helm/benchmark/presentation/table.py
src/helm/benchmark/presentation/test_contamination.py
src/helm/benchmark/presentation/test_create_plots.py
src/helm/benchmark/presentation/test_run_entry.py
src/helm/benchmark/scenarios/__init__.py
src/helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py
src/helm/benchmark/scenarios/babi_qa_scenario.py
src/helm/benchmark/scenarios/bbq_scenario.py
src/helm/benchmark/scenarios/big_bench_scenario.py
src/helm/benchmark/scenarios/blimp_scenario.py
src/helm/benchmark/scenarios/bold_scenario.py
src/helm/benchmark/scenarios/boolq_scenario.py
src/helm/benchmark/scenarios/civil_comments_scenario.py
src/helm/benchmark/scenarios/code_scenario.py
src/helm/benchmark/scenarios/code_scenario_apps_pinned_file_order.py
src/helm/benchmark/scenarios/code_scenario_helper.py
src/helm/benchmark/scenarios/commonsense_scenario.py
src/helm/benchmark/scenarios/copyright_scenario.py
src/helm/benchmark/scenarios/covid_dialog_scenario.py
src/helm/benchmark/scenarios/dialogue_scenarios.py
src/helm/benchmark/scenarios/disinformation_scenario.py
src/helm/benchmark/scenarios/dyck_language_scenario.py
src/helm/benchmark/scenarios/entity_data_imputation_scenario.py
src/helm/benchmark/scenarios/entity_matching_scenario.py
src/helm/benchmark/scenarios/entity_matching_scenario_fixed_random_state.py
src/helm/benchmark/scenarios/grammar.py
src/helm/benchmark/scenarios/grammar_scenario.py
src/helm/benchmark/scenarios/gsm_scenario.py
src/helm/benchmark/scenarios/ice_scenario.py
src/helm/benchmark/scenarios/ice_scenario_pinned_file_order.py
src/helm/benchmark/scenarios/imdb_scenario.py
src/helm/benchmark/scenarios/imdb_scenario_pinned_file_order.py
src/helm/benchmark/scenarios/interactive_qa_mmlu_scenario.py
src/helm/benchmark/scenarios/koala_scenario.py
src/helm/benchmark/scenarios/legal_summarization_scenario.py
src/helm/benchmark/scenarios/legal_support_scenario.py
src/helm/benchmark/scenarios/lex_glue_scenario.py
src/helm/benchmark/scenarios/lextreme_scenario.py
src/helm/benchmark/scenarios/lsat_qa_scenario.py
src/helm/benchmark/scenarios/math_scenario.py
src/helm/benchmark/scenarios/me_q_sum_scenario.py
src/helm/benchmark/scenarios/med_dialog_scenario.py
src/helm/benchmark/scenarios/med_mcqa_scenario.py
src/helm/benchmark/scenarios/med_paragraph_simplification_scenario.py
src/helm/benchmark/scenarios/med_qa_scenario.py
src/helm/benchmark/scenarios/mmlu_scenario.py
src/helm/benchmark/scenarios/msmarco_scenario.py
src/helm/benchmark/scenarios/narrativeqa_scenario.py
src/helm/benchmark/scenarios/natural_qa_scenario.py
src/helm/benchmark/scenarios/newsqa_scenario.py
src/helm/benchmark/scenarios/numeracy_scenario.py
src/helm/benchmark/scenarios/open_assistant_scenario.py
src/helm/benchmark/scenarios/opinions_qa_scenario.py
src/helm/benchmark/scenarios/pubmed_qa_scenario.py
src/helm/benchmark/scenarios/quac_scenario.py
src/helm/benchmark/scenarios/raft_scenario.py
src/helm/benchmark/scenarios/real_toxicity_prompts_scenario.py
src/helm/benchmark/scenarios/scenario.py
src/helm/benchmark/scenarios/self_instruct_scenario.py
src/helm/benchmark/scenarios/simple_scenarios.py
src/helm/benchmark/scenarios/summarization_scenario.py
src/helm/benchmark/scenarios/synthetic_efficiency_scenario.py
src/helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py
src/helm/benchmark/scenarios/synthetic_reasoning_scenario.py
src/helm/benchmark/scenarios/test_grammar.py
src/helm/benchmark/scenarios/test_scenario.py
src/helm/benchmark/scenarios/the_pile_scenario.py
src/helm/benchmark/scenarios/truthful_qa_scenario.py
src/helm/benchmark/scenarios/twitter_aae_scenario.py
src/helm/benchmark/scenarios/verifiability_judgment_scenario.py
src/helm/benchmark/scenarios/vicuna_scenario.py
src/helm/benchmark/scenarios/wikifact_scenario.py
src/helm/benchmark/scenarios/wikitext_103_scenario.py
src/helm/benchmark/scenarios/wmt_14_scenario.py
src/helm/benchmark/static/benchmarking.css
src/helm/benchmark/static/benchmarking.js
src/helm/benchmark/static/contamination.yaml
src/helm/benchmark/static/general.js
src/helm/benchmark/static/index.html
src/helm/benchmark/static/info-icon.png
src/helm/benchmark/static/json-urls-root.js
src/helm/benchmark/static/json-urls.js
src/helm/benchmark/static/plot-captions.js
src/helm/benchmark/static/schema.yaml
src/helm/benchmark/static/utils.js
src/helm/benchmark/static/images/crfm-logo.png
src/helm/benchmark/static/images/helm-logo-simple.png
src/helm/benchmark/static/images/helm-logo.png
src/helm/benchmark/static/images/language-model-helm.png
src/helm/benchmark/static/images/scenarios-by-metrics.png
src/helm/benchmark/static/images/taxonomy-scenarios.png
src/helm/benchmark/static/images/organizations/ai21.png
src/helm/benchmark/static/images/organizations/anthropic.png
src/helm/benchmark/static/images/organizations/bigscience.png
src/helm/benchmark/static/images/organizations/cohere.png
src/helm/benchmark/static/images/organizations/eleutherai.png
src/helm/benchmark/static/images/organizations/google.png
src/helm/benchmark/static/images/organizations/meta.png
src/helm/benchmark/static/images/organizations/microsoft.png
src/helm/benchmark/static/images/organizations/nvidia.png
src/helm/benchmark/static/images/organizations/openai.png
src/helm/benchmark/static/images/organizations/together.png
src/helm/benchmark/static/images/organizations/tsinghua-keg.png
src/helm/benchmark/static/images/organizations/yandex.png
src/helm/benchmark/window_services/__init__.py
src/helm/benchmark/window_services/ai21_window_service.py
src/helm/benchmark/window_services/anthropic_window_service.py
src/helm/benchmark/window_services/bloom_window_service.py
src/helm/benchmark/window_services/cohere_window_service.py
src/helm/benchmark/window_services/encoder_decoder_window_service.py
src/helm/benchmark/window_services/flan_t5_window_service.py
src/helm/benchmark/window_services/gpt2_window_service.py
src/helm/benchmark/window_services/gptj_window_service.py
src/helm/benchmark/window_services/gptneox_window_service.py
src/helm/benchmark/window_services/huggingface_window_service.py
src/helm/benchmark/window_services/ice_window_service.py
src/helm/benchmark/window_services/llama_window_service.py
src/helm/benchmark/window_services/local_window_service.py
src/helm/benchmark/window_services/luminous_window_service.py
src/helm/benchmark/window_services/megatron_window_service.py
src/helm/benchmark/window_services/mt_nlg_window_service.py
src/helm/benchmark/window_services/openai_window_service.py
src/helm/benchmark/window_services/opt_window_service.py
src/helm/benchmark/window_services/palmyra_window_service.py
src/helm/benchmark/window_services/remote_window_service.py
src/helm/benchmark/window_services/santacoder_window_service.py
src/helm/benchmark/window_services/starcoder_window_service.py
src/helm/benchmark/window_services/t0pp_window_service.py
src/helm/benchmark/window_services/t511b_window_service.py
src/helm/benchmark/window_services/test_ai21_window_service.py
src/helm/benchmark/window_services/test_anthropic_window_service.py
src/helm/benchmark/window_services/test_bloom_window_service.py
src/helm/benchmark/window_services/test_cohere_window_service.py
src/helm/benchmark/window_services/test_cohere_window_service_utils.py
src/helm/benchmark/window_services/test_flan_t5_window_service.py
src/helm/benchmark/window_services/test_gpt2_window_service.py
src/helm/benchmark/window_services/test_gpt4_window_service.py
src/helm/benchmark/window_services/test_gptj_window_service.py
src/helm/benchmark/window_services/test_gptneox_window_service.py
src/helm/benchmark/window_services/test_ice_window_service.py
src/helm/benchmark/window_services/test_mt_nlg_window_service.py
src/helm/benchmark/window_services/test_openai_window_service.py
src/helm/benchmark/window_services/test_opt_window_service.py
src/helm/benchmark/window_services/test_palmyra_window_service.py
src/helm/benchmark/window_services/test_t0pp_window_service.py
src/helm/benchmark/window_services/test_t511b_window_service.py
src/helm/benchmark/window_services/test_ul2_window_service.py
src/helm/benchmark/window_services/test_utils.py
src/helm/benchmark/window_services/test_yalm_window_service.py
src/helm/benchmark/window_services/tokenizer_service.py
src/helm/benchmark/window_services/ul2_window_service.py
src/helm/benchmark/window_services/wider_ai21_window_service.py
src/helm/benchmark/window_services/wider_openai_window_service.py
src/helm/benchmark/window_services/window_service.py
src/helm/benchmark/window_services/window_service_factory.py
src/helm/benchmark/window_services/yalm_window_service.py
src/helm/common/__init__.py
src/helm/common/authentication.py
src/helm/common/cache.py
src/helm/common/codec.py
src/helm/common/critique_request.py
src/helm/common/general.py
src/helm/common/hierarchical_logger.py
src/helm/common/object_spec.py
src/helm/common/perspective_api_request.py
src/helm/common/request.py
src/helm/common/test_cache.py
src/helm/common/test_codec.py
src/helm/common/test_general.py
src/helm/common/tokenization_request.py
src/helm/proxy/__init__.py
src/helm/proxy/accounts.py
src/helm/proxy/cli.py
src/helm/proxy/example_queries.py
src/helm/proxy/models.py
src/helm/proxy/query.py
src/helm/proxy/retry.py
src/helm/proxy/server.py
src/helm/proxy/test_models.py
src/helm/proxy/test_retry.py
src/helm/proxy/clients/__init__.py
src/helm/proxy/clients/ai21_client.py
src/helm/proxy/clients/aleph_alpha_client.py
src/helm/proxy/clients/anthropic_client.py
src/helm/proxy/clients/auto_client.py
src/helm/proxy/clients/chat_gpt_client.py
src/helm/proxy/clients/client.py
src/helm/proxy/clients/client_test_utils.py
src/helm/proxy/clients/cohere_client.py
src/helm/proxy/clients/critique_client.py
src/helm/proxy/clients/google_client.py
src/helm/proxy/clients/goose_ai_client.py
src/helm/proxy/clients/huggingface_client.py
src/helm/proxy/clients/huggingface_model_registry.py
src/helm/proxy/clients/huggingface_tokenizer.py
src/helm/proxy/clients/ice_tokenizer_client.py
src/helm/proxy/clients/mechanical_turk_critique_client.py
src/helm/proxy/clients/mechanical_turk_critique_exporter.py
src/helm/proxy/clients/mechanical_turk_critique_importer.py
src/helm/proxy/clients/megatron_client.py
src/helm/proxy/clients/microsoft_client.py
src/helm/proxy/clients/model_critique_client.py
src/helm/proxy/clients/openai_client.py
src/helm/proxy/clients/palmyra_client.py
src/helm/proxy/clients/perspective_api_client.py
src/helm/proxy/clients/remote_model_registry.py
src/helm/proxy/clients/scale_critique_client.py
src/helm/proxy/clients/simple_client.py
src/helm/proxy/clients/surge_ai_critique_client.py
src/helm/proxy/clients/test_anthropic_client.py
src/helm/proxy/clients/test_auto_client.py
src/helm/proxy/clients/test_client.py
src/helm/proxy/clients/test_huggingface_client.py
src/helm/proxy/clients/test_huggingface_model_registry.py
src/helm/proxy/clients/test_huggingface_tokenizer.py
src/helm/proxy/clients/test_ice_tokenizer_client.py
src/helm/proxy/clients/test_together_client.py
src/helm/proxy/clients/test_yalm_tokenizer_client.py
src/helm/proxy/clients/together_client.py
src/helm/proxy/clients/toxicity_classifier_client.py
src/helm/proxy/clients/yalm_tokenizer_client.py
src/helm/proxy/clients/yalm_tokenizer/__init__.py
src/helm/proxy/clients/yalm_tokenizer/test_yalm_tokenizer.py
src/helm/proxy/clients/yalm_tokenizer/voc_100b.sp
src/helm/proxy/clients/yalm_tokenizer/yalm_tokenizer.py
src/helm/proxy/services/__init__.py
src/helm/proxy/services/remote_service.py
src/helm/proxy/services/server_service.py
src/helm/proxy/services/service.py
src/helm/proxy/services/test_remote_service.py
src/helm/proxy/services/test_service.py
src/helm/proxy/token_counters/__init__.py
src/helm/proxy/token_counters/ai21_token_counter.py
src/helm/proxy/token_counters/auto_token_counter.py
src/helm/proxy/token_counters/cohere_token_counter.py
src/helm/proxy/token_counters/free_token_counter.py
src/helm/proxy/token_counters/gooseai_token_counter.py
src/helm/proxy/token_counters/openai_token_counter.py
src/helm/proxy/token_counters/test_ai21_token_counter.py
src/helm/proxy/token_counters/test_openai_token_counter.py
src/helm/proxy/token_counters/token_counter.py