Adding the IFEval scenario (#3122)

stanford-crfm · Nov 12, 2024 · f9c4498 · f9c4498
1 parent 3d28de9
commit f9c4498
Show file tree

Hide file tree

Showing 12 changed files with 2,089 additions and 2 deletions.
diff --git a/setup.cfg b/setup.cfg
@@ -79,6 +79,8 @@ metrics =
     numba~=0.56  # For copyright_metrics
     pytrec_eval==0.5  # For ranking_metrics
     sacrebleu~=2.2.1  # For disinformation_metrics, machine_translation_metrics
+    langdetect~=1.0.9  # For ifeval_metrics
+    immutabledict~=4.2.0  # For ifeval_metrics
 
 summarization =
     summ-eval~=0.892  # For summarization_metrics

diff --git a/src/helm/benchmark/metrics/ifeval/__init__.py b/src/helm/benchmark/metrics/ifeval/__init__.py