diff --git a/examples/nanotron/custom_evaluation_tasks.py b/examples/nanotron/custom_evaluation_tasks.py index b079498d8..9ddb2a40d 100644 --- a/examples/nanotron/custom_evaluation_tasks.py +++ b/examples/nanotron/custom_evaluation_tasks.py @@ -275,7 +275,7 @@ def boolq_prompt(line, task_name: str = None): name="quac", prompt_function=quac_prompt, hf_repo="lighteval/quac_helm", - hf_subset="deault", + hf_subset="default", metrics=[ Metrics.exact_match(sample_params={"normalize_gold": helm_normalizer, "normalize_pred": helm_normalizer}) ],